Add Typed TSV serialization

This commit is contained in:
Nathan McRae 2024-02-17 12:54:32 -08:00
parent 8795a2873a
commit 457ef1df01

View File

@ -28,6 +28,9 @@ public class SaneTsv
COMMENTED_TSV = 2, COMMENTED_TSV = 2,
} }
public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE");
public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE");
// TODO: We need to be able to update all these in tandem somehow // TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; } public string[] ColumnNames { get; protected set; }
public Type[] ColumnTypes { get; protected set; } public Type[] ColumnTypes { get; protected set; }
@ -145,7 +148,8 @@ public class SaneTsv
string columnTypeString; string columnTypeString;
string columnName; string columnName;
if (columnString.Contains(':')) { if (columnString.Contains(':'))
{
if (format == FormatType.SIMPLE_TSV) if (format == FormatType.SIMPLE_TSV)
{ {
throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names"); throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
@ -548,6 +552,317 @@ public class SaneTsv
return Encoding.UTF8.GetBytes(escapedString.ToString()); return Encoding.UTF8.GetBytes(escapedString.ToString());
} }
public static Type GetColumnFromType(Type type)
{
if (type == typeof(string))
{
return typeof(StringType);
}
else if (type == typeof(bool))
{
return typeof(BooleanType);
}
else if (type == typeof(float))
{
return typeof(Float32Type);
}
else if (type == typeof(double))
{
return typeof(Float64Type);
}
else if (type == typeof(UInt32))
{
return typeof(UInt32Type);
}
else if (type == typeof(UInt64))
{
return typeof(UInt64Type);
}
else if (type == typeof(Int32))
{
return typeof(Int32Type);
}
else if (type == typeof(Int64))
{
return typeof(Int64Type);
}
else if (type == typeof(byte[]))
{
return typeof(BinaryType);
}
else
{
throw new Exception($"Invalid type: {type}");
}
}
public static string GetNameFromColumn(Type type)
{
if (type == typeof(StringType))
{
return "string";
}
else if (type == typeof(BooleanType))
{
return "boolean";
}
else if (type == typeof(Float32Type))
{
return "float32";
}
else if (type == typeof(Float32LEType))
{
return "float32-le";
}
else if (type == typeof(Float64Type))
{
return "float64";
}
else if (type == typeof(Float64LEType))
{
return "float64-le";
}
else if (type == typeof(UInt32Type))
{
return "uint32";
}
else if (type == typeof(UInt64Type))
{
return "uint64";
}
else if (type == typeof(Int32Type))
{
return "int32";
}
else if (type == typeof(Int64Type))
{
return "int64";
}
else if (type == typeof(BinaryType))
{
return "binary";
}
else
{
throw new Exception($"Invalid type: {type}");
}
}
public static byte[] SerializeTypedTsv(IList<Type> headerTypes, IList<string> headerNames, IList<IList<object>> data)
{
var bytes = new List<byte>();
if (headerNames.Count != headerTypes.Count)
{
throw new ArgumentException($"headerTypes length ({headerTypes.Count}) does not match headerNames length ({headerNames.Count})");
}
// Serialize header
for (int i = 0; i < headerNames.Count; i++)
{
for (int j = i + 1; j < headerNames.Count; j++)
{
if (headerNames[i] == headerNames[j])
{
throw new Exception("Column names in header must be unique");
}
}
byte[] nameEncoded = Encoding.UTF8.GetBytes(headerNames[i]);
for (int j = 0; j < nameEncoded.Length; j++)
{
if (nameEncoded[j] == '\n')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'n');
}
else if (nameEncoded[j] == '\t')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'t');
}
else if (nameEncoded[j] == '\\')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'\\');
}
else if (nameEncoded[j] == '#')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'#');
}
else
{
bytes.Add(nameEncoded[j]);
}
}
bytes.Add((byte)':');
try
{
bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(headerTypes[i])));
}
catch (Exception e)
{
throw new Exception($"Invalid header type for column {i}", e);
}
if (i == headerNames.Count - 1)
{
bytes.Add((byte)'\n');
}
else
{
bytes.Add((byte)'\t');
}
}
// Serialize data
for (int i = 0; i < data.Count; i++)
{
for (int j = 0; j < data[i].Count; j++)
{
try
{
byte[] fieldEncoded;
if (headerTypes[j] == typeof(StringType))
{
fieldEncoded = Encoding.UTF8.GetBytes((string)data[i][j]);
}
else if (headerTypes[j] == typeof(BooleanType))
{
bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded);
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float32Type))
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9")));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float32LEType))
{
if (LittleEndian)
{
fieldEncoded = BitConverter.GetBytes((float)data[i][j]);
}
else
{
byte[] floatBytes = BitConverter.GetBytes((float)data[i][j]);
fieldEncoded = new byte[sizeof(float)];
for (int k = 0; k < sizeof(float); k++)
{
fieldEncoded[k] = floatBytes[sizeof(float) - 1 - k];
}
}
}
else if (headerTypes[j] == typeof(Float64Type))
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((double)data[i][j]).ToString("G17")));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float64LEType))
{
if (LittleEndian)
{
fieldEncoded = BitConverter.GetBytes((double)data[i][j]);
}
else
{
byte[] doubleBytes = BitConverter.GetBytes((double)data[i][j]);
fieldEncoded = new byte[sizeof(double)];
for (int k = 0; k < sizeof(double); k++)
{
fieldEncoded[k] = doubleBytes[sizeof(double) - 1 - k];
}
}
}
else if (headerTypes[j] == typeof(UInt32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(UInt64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Int32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Int64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(BinaryType))
{
fieldEncoded = (byte[])data[i][j];
}
else
{
throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
}
for (int k = 0; k < fieldEncoded.Length; k++)
{
if (fieldEncoded[k] == '\n')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'n');
}
else if (fieldEncoded[k] == '\t')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'t');
}
else if (fieldEncoded[k] == '\\')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'\\');
}
else if (fieldEncoded[k] == '#')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'#');
}
else
{
bytes.Add(fieldEncoded[k]);
}
}
if (j == headerNames.Count - 1)
{
bytes.Add((byte)'\n');
}
else
{
bytes.Add((byte)'\t');
}
}
catch (InvalidCastException e)
{
throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(headerTypes[j])}", e);
}
}
}
return bytes.ToArray();
}
public SaneTsvRecord this[int i] => Records[i]; public SaneTsvRecord this[int i] => Records[i];
public class SaneTsvRecord public class SaneTsvRecord