using System.Text; namespace NathanMcRae; /// /// Sane Tab-Separated Values /// public class SaneTsv { // Like an enum, but more extensible public class ColumnType { } public class StringType : ColumnType { } public class BooleanType : ColumnType { } public class Float32Type : ColumnType { } public class Float64Type : ColumnType { } public class UInt32Type : ColumnType { } public class UInt64Type : ColumnType { } public class Int32Type : ColumnType { } public class Int64Type : ColumnType { } public class BinaryType : ColumnType { } protected enum FormatType { SIMPLE_TSV = 0, TYPED_TSV = 1, COMMENTED_TSV = 2, } // TODO: We need to be able to update all these in tandem somehow public string[] ColumnNames { get; protected set; } public Type[] ColumnTypes { get; protected set; } public List Records { get; protected set; } public string FileComment { get; protected set; } = null; public static SaneTsv ParseSimpleTsv(byte[] inputBuffer) { return Parse(inputBuffer, FormatType.SIMPLE_TSV); } public static SaneTsv ParseTypedTsv(byte[] inputBuffer) { return Parse(inputBuffer, FormatType.TYPED_TSV); } public static SaneTsv ParseCommentedTsv(byte[] inputBuffer) { return Parse(inputBuffer, FormatType.COMMENTED_TSV); } // TODO: Have parsing errors include line / column # protected static SaneTsv Parse(byte[] inputBuffer, FormatType format) { var parsed = new SaneTsv(); parsed.ColumnNames = new string[] { }; parsed.ColumnTypes = new Type[] { }; parsed.Records = new List(); var fieldBytes = new List(); var fields = new List(); var currentComment = new StringBuilder(); int numFields = -1; int line = 1; int currentLineStart = 0; for (int i = 0; i < inputBuffer.Count(); i++) { if (inputBuffer[i] == '\\') { if (i + 1 == inputBuffer.Count()) { throw new Exception($"Found '\\' at end of input"); } if (inputBuffer[i + 1] == 'n') { fieldBytes.Add((byte)'\n'); i++; } else if (inputBuffer[i + 1] == '\\') { fieldBytes.Add((byte)'\\'); i++; } else if (inputBuffer[i + 1] == 't') { fieldBytes.Add((byte)'\t'); i++; } else if (inputBuffer[i + 1] == '#') { fieldBytes.Add((byte)'#'); i++; } else { throw new Exception($"Expected 'n', 't', '#', or '\\' after '\\' at line {line} column {i - currentLineStart}"); } } else if (inputBuffer[i] == '\t') { // end of field fields.Add(fieldBytes.ToArray()); fieldBytes.Clear(); } else if (inputBuffer[i] == '\n') { fields.Add(fieldBytes.ToArray()); fieldBytes.Clear(); if (numFields < 0) { // This is the header numFields = fields.Count; parsed.ColumnNames = new string[numFields]; parsed.ColumnTypes = new Type[numFields]; int numTypesBlank = 0; for (int j = 0; j < fields.Count; j++) { string columnString; try { columnString = Encoding.UTF8.GetString(fields[j]); } catch (Exception e) { throw new Exception($"Header {fields.Count} is not valid UTF-8", e); } string columnTypeString; string columnName; if (columnString.Contains(':')) { if (format == FormatType.SIMPLE_TSV) { throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names"); } columnTypeString = columnString.Split(":").Last(); columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1); } else { if (format > FormatType.SIMPLE_TSV) { throw new Exception($"Header {fields.Count} has no type"); } columnTypeString = ""; columnName = columnString; } Type type; switch (columnTypeString) { case "": numTypesBlank++; type = typeof(StringType); break; case "string": type = typeof(StringType); break; case "boolean": type = typeof(BooleanType); break; case "float32": type = typeof(Float32Type); break; case "float64": type = typeof(Float64Type); break; case "uint32": type = typeof(UInt32Type); break; case "uint64": type = typeof(UInt64Type); break; case "int32": type = typeof(Int32Type); break; case "int64": type = typeof(Int64Type); break; case "binary": type = typeof(BinaryType); break; default: throw new Exception($"Invalid type '{columnTypeString}' for column {j}"); } // TODO: Check column name uniqueness parsed.ColumnNames[j] = columnName; parsed.ColumnTypes[j] = type; } if (currentComment.Length > 0) { parsed.FileComment = currentComment.ToString(); currentComment.Clear(); } fields.Clear(); } else if (numFields != fields.Count) { throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}"); } else { string comment = null; if (currentComment.Length > 0) { comment = currentComment.ToString(); currentComment.Clear(); } parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line)); fields.Clear(); } line++; currentLineStart = i + 1; } else if (inputBuffer[i] == '#') { if (i == currentLineStart && format >= FormatType.COMMENTED_TSV) { int j = i; for (; j < inputBuffer.Length && inputBuffer[j] != '\n'; j++) { } if (j < inputBuffer.Length) { var commentBytes = new byte[j - i - 1]; Array.Copy(inputBuffer, i + 1, commentBytes, 0, j - i - 1); currentComment.Append(Encoding.UTF8.GetString(commentBytes)); currentComment.Append("\n"); i = j; currentLineStart = i + 1; line++; } else { throw new Exception("Comments at end of file are not allowed"); } } else { throw new Exception($"Found unescaped '#' at line {line}, column {i - currentLineStart}"); } } else { fieldBytes.Add(inputBuffer[i]); } } fields.Add(fieldBytes.ToArray()); if (numFields == 0) { throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record"); } if (numFields != fields.Count) { throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}"); } else { string comment = null; if (currentComment.Length > 0) { comment = currentComment.ToString(); currentComment.Clear(); } parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line)); fields.Clear(); } return parsed; } /// /// Note: this modifies 'parsed' /// protected static object[] ParseCurrentRecord(SaneTsv parsed, List fields, int line) { var parsedFields = new object[fields.Count]; for (int j = 0; j < fields.Count; j++) { // All other types require the content to be UTF-8. Binary fields can ignore that. if (parsed.ColumnTypes[j] == typeof(BinaryType)) { parsedFields[j] = fields[j]; continue; } string fieldString; try { fieldString = Encoding.UTF8.GetString(fields[j]); } catch (Exception e) { throw new Exception($"Field {j} on line {line} is not valid UTF-8", e); } // TODO: Add checking for numeric types format if (parsed.ColumnTypes[j] == typeof(StringType)) { parsedFields[j] = fieldString; } else if (parsed.ColumnTypes[j] == typeof(BooleanType)) { bool parsedBool; if (fieldString == "TRUE") { parsedBool = true; } else if (fieldString == "FALSE") { parsedBool = false; } else { throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly"); } parsedFields[j] = parsedBool; } else if (parsed.ColumnTypes[j] == typeof(Float32Type)) { if (!float.TryParse(fieldString, out float parsedFloat)) { throw new Exception($"Field {j} on line {line} is not valid single-precision float"); } parsedFields[j] = parsedFloat; } else if (parsed.ColumnTypes[j] == typeof(Float64Type)) { if (!double.TryParse(fieldString, out double parsedDouble)) { throw new Exception($"Field {j} on line {line} is not valid double-precision float"); } parsedFields[j] = parsedDouble; } else if (parsed.ColumnTypes[j] == typeof(UInt32Type)) { if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32)) { throw new Exception($"Field {j} on line {line} is not valid UInt32"); } parsedFields[j] = parsedUInt32; } else if (parsed.ColumnTypes[j] == typeof(UInt64Type)) { if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64)) { throw new Exception($"Field {j} on line {line} is not valid UInt64"); } parsedFields[j] = parsedUInt64; } else if (parsed.ColumnTypes[j] == typeof(Int32Type)) { if (!Int32.TryParse(fieldString, out Int32 parsedInt32)) { throw new Exception($"Field {j} on line {line} is not valid Int32"); } parsedFields[j] = parsedInt32; } else if (parsed.ColumnTypes[j] == typeof(Int64Type)) { if (!Int64.TryParse(fieldString, out Int64 parsedInt64)) { throw new Exception($"Field {j} on line {line} is not valid Int64"); } parsedFields[j] = parsedInt64; } else { throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}"); } } return parsedFields; } public static byte[] SerializeSimpleTsv(IList header, IList> data) { var escapedString = new StringBuilder(); // Serialize header for (int i = 0; i < header.Count; i++) { if (header[i].Contains(':')) { throw new Exception($"Column {i} contains the character ':'"); } for (int j = i + 1; j < header.Count; j++) { if (header[i] == header[j]) { throw new Exception("Column names in header must be unique"); } } for (int j = 0; j < header[i].Count(); j++) { if (header[i][j] == '\n') { escapedString.Append("\\n"); } else if (header[i][j] == '\t') { escapedString.Append("\\t"); } else if (header[i][j] == '\\') { escapedString.Append("\\\\"); } else if (header[i][j] == '#') { escapedString.Append("\\#"); } else { escapedString.Append(header[i][j]); } } if (i == header.Count - 1) { escapedString.Append('\n'); } else { escapedString.Append('\t'); } } // Serialize data for (int i = 0; i < data.Count; i++) { for (int j = 0; j < data[i].Count; j++) { for (int k = 0; k < data[i][j].Length; k++) { if (data[i][j][k] == '\n') { escapedString.Append("\\n"); } else if (data[i][j][k] == '\t') { escapedString.Append("\\t"); } else if (data[i][j][k] == '\\') { escapedString.Append("\\\\"); } else if (data[i][j][k] == '#') { escapedString.Append("\\#"); } else { escapedString.Append(data[i][j][k]); } } if (j < data[i].Count - 1) { escapedString.Append('\t'); } else if (i < data.Count - 1) { escapedString.Append('\n'); } } } return Encoding.UTF8.GetBytes(escapedString.ToString()); } public SaneTsvRecord this[int i] => Records[i]; public class SaneTsvRecord { public SaneTsv Parent { get; } public string Comment { get; } public object[] Fields { get; } public int Line { get; } public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)]; public object this[int columnIndex] => Fields[columnIndex]; public SaneTsvRecord(SaneTsv parent, object[] fields, string comment, int line) { Parent = parent; Fields = fields; Comment = comment; Line = line; } } }