725a5b2034
Change the naming so the overall family of formats is Sane TSV, while the simplest format is Simple TSV.
519 lines
14 KiB
C#
519 lines
14 KiB
C#
using System.Text;
|
|
|
|
namespace NathanMcRae;
|
|
|
|
/// <summary>
|
|
/// Sane Tab-Separated Values
|
|
/// </summary>
|
|
public class SaneTsv
|
|
{
|
|
public enum ColumnType
|
|
{
|
|
STRING,
|
|
BOOLEAN,
|
|
FLOAT32,
|
|
FLOAT64,
|
|
UINT32,
|
|
UINT64,
|
|
INT32,
|
|
INT64,
|
|
BINARY,
|
|
}
|
|
|
|
protected enum FormatType
|
|
{
|
|
SIMPLE_TSV = 0,
|
|
TYPED_TSV = 1,
|
|
COMMENTED_TSV = 2,
|
|
}
|
|
|
|
// TODO: We need to be able to update all these in tandem somehow
|
|
public string[] ColumnNames { get; protected set; }
|
|
public ColumnType[] ColumnTypes { get; protected set; }
|
|
public Dictionary<string, List<object>> Columns { get; protected set; }
|
|
public List<SaneTsvRecord> Records { get; protected set; }
|
|
public string FileComment { get; protected set; } = null;
|
|
|
|
public static SaneTsv ParseSimpleTsv(byte[] inputBuffer)
|
|
{
|
|
return Parse(inputBuffer, FormatType.SIMPLE_TSV);
|
|
}
|
|
|
|
public static SaneTsv ParseTypedTsv(byte[] inputBuffer)
|
|
{
|
|
return Parse(inputBuffer, FormatType.TYPED_TSV);
|
|
}
|
|
|
|
public static SaneTsv ParseCommentedTsv(byte[] inputBuffer)
|
|
{
|
|
return Parse(inputBuffer, FormatType.COMMENTED_TSV);
|
|
}
|
|
|
|
// TODO: Have parsing errors include line / column #
|
|
protected static SaneTsv Parse(byte[] inputBuffer, FormatType format)
|
|
{
|
|
var parsed = new SaneTsv();
|
|
parsed.Columns = new Dictionary<string, List<object>>();
|
|
parsed.ColumnNames = new string[] { };
|
|
parsed.ColumnTypes = new ColumnType[] { };
|
|
parsed.Records = new List<SaneTsvRecord>();
|
|
|
|
var fieldBytes = new List<byte>();
|
|
var fields = new List<byte[]>();
|
|
var currentComment = new StringBuilder();
|
|
|
|
int numFields = -1;
|
|
int line = 1;
|
|
int currentLineStart = 0;
|
|
for (int i = 0; i < inputBuffer.Count(); i++)
|
|
{
|
|
if (inputBuffer[i] == '\\')
|
|
{
|
|
if (i + 1 == inputBuffer.Count())
|
|
{
|
|
throw new Exception($"Found '\\' at end of input");
|
|
}
|
|
if (inputBuffer[i + 1] == 'n')
|
|
{
|
|
fieldBytes.Add((byte)'\n');
|
|
i++;
|
|
}
|
|
else if (inputBuffer[i + 1] == '\\')
|
|
{
|
|
fieldBytes.Add((byte)'\\');
|
|
i++;
|
|
}
|
|
else if (inputBuffer[i + 1] == 't')
|
|
{
|
|
fieldBytes.Add((byte)'\t');
|
|
i++;
|
|
}
|
|
else if (inputBuffer[i + 1] == '#')
|
|
{
|
|
fieldBytes.Add((byte)'#');
|
|
i++;
|
|
}
|
|
else
|
|
{
|
|
throw new Exception($"Expected 'n', 't', or '\\' after '\\' at {i}");
|
|
}
|
|
}
|
|
else if (inputBuffer[i] == '\t')
|
|
{
|
|
// end of field
|
|
fields.Add(fieldBytes.ToArray());
|
|
fieldBytes.Clear();
|
|
}
|
|
else if (inputBuffer[i] == '\n')
|
|
{
|
|
fields.Add(fieldBytes.ToArray());
|
|
fieldBytes.Clear();
|
|
|
|
if (numFields < 0)
|
|
{
|
|
// This is the header
|
|
|
|
numFields = fields.Count;
|
|
|
|
parsed.ColumnNames = new string[numFields];
|
|
parsed.ColumnTypes = new ColumnType[numFields];
|
|
|
|
int numTypesBlank = 0;
|
|
|
|
for (int j = 0; j < fields.Count; j++)
|
|
{
|
|
string columnString;
|
|
try
|
|
{
|
|
columnString = Encoding.UTF8.GetString(fields[j]);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
throw new Exception($"Header {fields.Count} is not valid UTF-8", e);
|
|
}
|
|
|
|
string columnTypeString;
|
|
string columnName;
|
|
if (columnString.Contains(':')) {
|
|
if (format == FormatType.SIMPLE_TSV)
|
|
{
|
|
throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
|
|
}
|
|
columnTypeString = columnString.Split(":").Last();
|
|
columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
|
|
}
|
|
else
|
|
{
|
|
if (format > FormatType.SIMPLE_TSV)
|
|
{
|
|
throw new Exception($"Header {fields.Count} has no type");
|
|
}
|
|
columnTypeString = "";
|
|
columnName = columnString;
|
|
}
|
|
|
|
ColumnType type;
|
|
|
|
switch (columnTypeString)
|
|
{
|
|
case "":
|
|
numTypesBlank++;
|
|
type = ColumnType.STRING;
|
|
break;
|
|
case "string":
|
|
type = ColumnType.STRING;
|
|
break;
|
|
case "boolean":
|
|
type = ColumnType.BOOLEAN;
|
|
break;
|
|
case "float32":
|
|
type = ColumnType.FLOAT32;
|
|
break;
|
|
case "float64":
|
|
type = ColumnType.FLOAT64;
|
|
break;
|
|
case "uint32":
|
|
type = ColumnType.UINT32;
|
|
break;
|
|
case "uint64":
|
|
type = ColumnType.UINT64;
|
|
break;
|
|
case "int32":
|
|
type = ColumnType.INT32;
|
|
break;
|
|
case "int64":
|
|
type = ColumnType.INT64;
|
|
break;
|
|
case "binary":
|
|
type = ColumnType.BINARY;
|
|
break;
|
|
default:
|
|
throw new Exception($"Invalid type '{columnTypeString}' for column {j}");
|
|
}
|
|
|
|
try
|
|
{
|
|
parsed.Columns.Add(columnName, new List<object>());
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
throw new Exception($"Column name {columnName} is not unique", e);
|
|
}
|
|
|
|
parsed.ColumnNames[j] = columnName;
|
|
parsed.ColumnTypes[j] = type;
|
|
}
|
|
|
|
if (currentComment.Length > 0)
|
|
{
|
|
parsed.FileComment = currentComment.ToString();
|
|
currentComment.Clear();
|
|
}
|
|
|
|
fields.Clear();
|
|
}
|
|
else if (numFields != fields.Count)
|
|
{
|
|
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
|
|
}
|
|
else
|
|
{
|
|
string comment = null;
|
|
if (currentComment.Length > 0)
|
|
{
|
|
comment = currentComment.ToString();
|
|
currentComment.Clear();
|
|
}
|
|
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment));
|
|
fields.Clear();
|
|
}
|
|
|
|
line++;
|
|
currentLineStart = i + 1;
|
|
}
|
|
else if (inputBuffer[i] == '#')
|
|
{
|
|
if (i == currentLineStart && format >= FormatType.COMMENTED_TSV)
|
|
{
|
|
int j = i;
|
|
for (; j < inputBuffer.Length && inputBuffer[j] != '\n'; j++) { }
|
|
if (j < inputBuffer.Length)
|
|
{
|
|
var commentBytes = new byte[j - i - 1];
|
|
Array.Copy(inputBuffer, i + 1, commentBytes, 0, j - i - 1);
|
|
currentComment.Append(Encoding.UTF8.GetString(commentBytes));
|
|
currentComment.Append("\n");
|
|
i = j;
|
|
currentLineStart = i + 1;
|
|
line++;
|
|
}
|
|
else
|
|
{
|
|
throw new Exception("Comments at end of file are not allowed");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw new Exception($"Found unescaped '#' at line {line}, column {i - currentLineStart}");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
fieldBytes.Add(inputBuffer[i]);
|
|
}
|
|
}
|
|
|
|
fields.Add(fieldBytes.ToArray());
|
|
|
|
if (numFields == 0)
|
|
{
|
|
throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
|
|
}
|
|
if (numFields != fields.Count)
|
|
{
|
|
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
|
|
}
|
|
else
|
|
{
|
|
string comment = null;
|
|
if (currentComment.Length > 0)
|
|
{
|
|
comment = currentComment.ToString();
|
|
currentComment.Clear();
|
|
}
|
|
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment));
|
|
fields.Clear();
|
|
}
|
|
|
|
return parsed;
|
|
}
|
|
|
|
protected static object[] ParseCurrentRecord(SaneTsv parsed, List<byte[]> fields, int line)
|
|
{
|
|
var parsedFields = new object[fields.Count];
|
|
for (int j = 0; j < fields.Count; j++)
|
|
{
|
|
// All other types require the content to be UTF-8. Binary fields can ignore that.
|
|
if (parsed.ColumnTypes[j] == ColumnType.BINARY)
|
|
{
|
|
parsedFields[j] = fields[j];
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(fields[j]);
|
|
continue;
|
|
}
|
|
|
|
string fieldString;
|
|
try
|
|
{
|
|
fieldString = Encoding.UTF8.GetString(fields[j]);
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid UTF-8", e);
|
|
}
|
|
|
|
// TODO: Add checking for numeric types format
|
|
|
|
switch (parsed.ColumnTypes[j])
|
|
{
|
|
case ColumnType.STRING:
|
|
parsedFields[j] = fieldString;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(fieldString);
|
|
break;
|
|
case ColumnType.BOOLEAN:
|
|
bool parsedBool;
|
|
if (fieldString == "TRUE")
|
|
{
|
|
parsedBool = true;
|
|
}
|
|
else if (fieldString == "FALSE")
|
|
{
|
|
parsedBool = false;
|
|
}
|
|
else
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
|
|
}
|
|
|
|
parsedFields[j] = parsedBool;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedBool);
|
|
break;
|
|
case ColumnType.FLOAT32:
|
|
if (!float.TryParse(fieldString, out float parsedFloat))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid single-precision float");
|
|
}
|
|
|
|
parsedFields[j] = parsedFloat;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedFloat);
|
|
break;
|
|
case ColumnType.FLOAT64:
|
|
if (!double.TryParse(fieldString, out double parsedDouble))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid double-precision float");
|
|
}
|
|
|
|
parsedFields[j] = parsedDouble;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedDouble);
|
|
break;
|
|
case ColumnType.UINT32:
|
|
if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid UInt32");
|
|
}
|
|
|
|
parsedFields[j] = parsedUInt32;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedUInt32);
|
|
break;
|
|
case ColumnType.UINT64:
|
|
if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid UInt64");
|
|
}
|
|
|
|
parsedFields[j] = parsedUInt64;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedUInt64);
|
|
break;
|
|
case ColumnType.INT32:
|
|
if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid Int32");
|
|
}
|
|
|
|
parsedFields[j] = parsedInt32;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedInt32);
|
|
break;
|
|
case ColumnType.INT64:
|
|
if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
|
|
{
|
|
throw new Exception($"Field {j} on line {line} is not valid Int64");
|
|
}
|
|
|
|
parsedFields[j] = parsedInt64;
|
|
parsed.Columns[parsed.ColumnNames[j]].Add(parsedInt64);
|
|
break;
|
|
case ColumnType.BINARY:
|
|
throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}");
|
|
default:
|
|
throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}");
|
|
}
|
|
}
|
|
|
|
return parsedFields;
|
|
}
|
|
|
|
public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data)
|
|
{
|
|
var escapedString = new StringBuilder();
|
|
|
|
// Serialize header
|
|
for (int i = 0; i < header.Count; i++)
|
|
{
|
|
if (header[i].Contains(':'))
|
|
{
|
|
throw new Exception($"Column {i} contains the character ':'");
|
|
}
|
|
|
|
for (int j = i + 1; j < header.Count; j++)
|
|
{
|
|
if (header[i] == header[j])
|
|
{
|
|
throw new Exception("Column names in header must be unique");
|
|
}
|
|
}
|
|
|
|
for (int j = 0; j < header[i].Count(); j++)
|
|
{
|
|
if (header[i][j] == '\n')
|
|
{
|
|
escapedString.Append("\\n");
|
|
}
|
|
else if (header[i][j] == '\t')
|
|
{
|
|
escapedString.Append("\\t");
|
|
}
|
|
else if (header[i][j] == '\\')
|
|
{
|
|
escapedString.Append("\\\\");
|
|
}
|
|
else if (header[i][j] == '#')
|
|
{
|
|
escapedString.Append("\\#");
|
|
}
|
|
else
|
|
{
|
|
escapedString.Append(header[i][j]);
|
|
}
|
|
}
|
|
|
|
if (i == header.Count - 1)
|
|
{
|
|
escapedString.Append('\n');
|
|
}
|
|
else
|
|
{
|
|
escapedString.Append('\t');
|
|
}
|
|
}
|
|
|
|
// Serialize data
|
|
for (int i = 0; i < data.Count; i++)
|
|
{
|
|
for (int j = 0; j < data[i].Count; j++)
|
|
{
|
|
for (int k = 0; k < data[i][j].Length; k++)
|
|
{
|
|
if (data[i][j][k] == '\n')
|
|
{
|
|
escapedString.Append("\\n");
|
|
}
|
|
else if (data[i][j][k] == '\t')
|
|
{
|
|
escapedString.Append("\\t");
|
|
}
|
|
else if (data[i][j][k] == '\\')
|
|
{
|
|
escapedString.Append("\\\\");
|
|
}
|
|
else if (data[i][j][k] == '#')
|
|
{
|
|
escapedString.Append("\\#");
|
|
}
|
|
else
|
|
{
|
|
escapedString.Append(data[i][j][k]);
|
|
}
|
|
}
|
|
|
|
if (j < data[i].Count - 1)
|
|
{
|
|
escapedString.Append('\t');
|
|
}
|
|
else if (i < data.Count - 1)
|
|
{
|
|
escapedString.Append('\n');
|
|
}
|
|
}
|
|
}
|
|
|
|
return Encoding.UTF8.GetBytes(escapedString.ToString());
|
|
}
|
|
|
|
public SaneTsvRecord this[int i] => Records[i];
|
|
|
|
public class SaneTsvRecord
|
|
{
|
|
public SaneTsv Parent { get; }
|
|
public string Comment { get; }
|
|
public object[] Fields { get; }
|
|
|
|
public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)];
|
|
|
|
public SaneTsvRecord(SaneTsv parent, object[] fields, string comment)
|
|
{
|
|
Parent = parent;
|
|
Fields = fields;
|
|
Comment = comment;
|
|
}
|
|
}
|
|
}
|