From 457ef1df010ea61c52629caa0ff43a0e29dde09b Mon Sep 17 00:00:00 2001 From: Nathan McRae Date: Sat, 17 Feb 2024 12:54:32 -0800 Subject: [PATCH] Add Typed TSV serialization --- SaneTsv.cs | 319 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 317 insertions(+), 2 deletions(-) diff --git a/SaneTsv.cs b/SaneTsv.cs index aaefe20..811b981 100644 --- a/SaneTsv.cs +++ b/SaneTsv.cs @@ -28,6 +28,9 @@ public class SaneTsv COMMENTED_TSV = 2, } + public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE"); + public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE"); + // TODO: We need to be able to update all these in tandem somehow public string[] ColumnNames { get; protected set; } public Type[] ColumnTypes { get; protected set; } @@ -145,7 +148,8 @@ public class SaneTsv string columnTypeString; string columnName; - if (columnString.Contains(':')) { + if (columnString.Contains(':')) + { if (format == FormatType.SIMPLE_TSV) { throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names"); @@ -507,7 +511,7 @@ public class SaneTsv // Serialize data for (int i = 0; i < data.Count; i++) - { + { for (int j = 0; j < data[i].Count; j++) { for (int k = 0; k < data[i][j].Length; k++) @@ -548,6 +552,317 @@ public class SaneTsv return Encoding.UTF8.GetBytes(escapedString.ToString()); } + public static Type GetColumnFromType(Type type) + { + if (type == typeof(string)) + { + return typeof(StringType); + } + else if (type == typeof(bool)) + { + return typeof(BooleanType); + } + else if (type == typeof(float)) + { + return typeof(Float32Type); + } + else if (type == typeof(double)) + { + return typeof(Float64Type); + } + else if (type == typeof(UInt32)) + { + return typeof(UInt32Type); + } + else if (type == typeof(UInt64)) + { + return typeof(UInt64Type); + } + else if (type == typeof(Int32)) + { + return typeof(Int32Type); + } + else if (type == typeof(Int64)) + { + return typeof(Int64Type); + } + else if (type == typeof(byte[])) + { + return typeof(BinaryType); + } + else + { + throw new Exception($"Invalid type: {type}"); + } + } + + public static string GetNameFromColumn(Type type) + { + if (type == typeof(StringType)) + { + return "string"; + } + else if (type == typeof(BooleanType)) + { + return "boolean"; + } + else if (type == typeof(Float32Type)) + { + return "float32"; + } + else if (type == typeof(Float32LEType)) + { + return "float32-le"; + } + else if (type == typeof(Float64Type)) + { + return "float64"; + } + else if (type == typeof(Float64LEType)) + { + return "float64-le"; + } + else if (type == typeof(UInt32Type)) + { + return "uint32"; + } + else if (type == typeof(UInt64Type)) + { + return "uint64"; + } + else if (type == typeof(Int32Type)) + { + return "int32"; + } + else if (type == typeof(Int64Type)) + { + return "int64"; + } + else if (type == typeof(BinaryType)) + { + return "binary"; + } + else + { + throw new Exception($"Invalid type: {type}"); + } + } + + public static byte[] SerializeTypedTsv(IList headerTypes, IList headerNames, IList> data) + { + var bytes = new List(); + + if (headerNames.Count != headerTypes.Count) + { + throw new ArgumentException($"headerTypes length ({headerTypes.Count}) does not match headerNames length ({headerNames.Count})"); + } + + // Serialize header + for (int i = 0; i < headerNames.Count; i++) + { + for (int j = i + 1; j < headerNames.Count; j++) + { + if (headerNames[i] == headerNames[j]) + { + throw new Exception("Column names in header must be unique"); + } + } + + byte[] nameEncoded = Encoding.UTF8.GetBytes(headerNames[i]); + + for (int j = 0; j < nameEncoded.Length; j++) + { + if (nameEncoded[j] == '\n') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'n'); + } + else if (nameEncoded[j] == '\t') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'t'); + } + else if (nameEncoded[j] == '\\') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'\\'); + } + else if (nameEncoded[j] == '#') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'#'); + } + else + { + bytes.Add(nameEncoded[j]); + } + } + + bytes.Add((byte)':'); + try + { + bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(headerTypes[i]))); + } + catch (Exception e) + { + throw new Exception($"Invalid header type for column {i}", e); + } + + if (i == headerNames.Count - 1) + { + bytes.Add((byte)'\n'); + } + else + { + bytes.Add((byte)'\t'); + } + } + + // Serialize data + for (int i = 0; i < data.Count; i++) + { + for (int j = 0; j < data[i].Count; j++) + { + try + { + byte[] fieldEncoded; + + if (headerTypes[j] == typeof(StringType)) + { + fieldEncoded = Encoding.UTF8.GetBytes((string)data[i][j]); + } + else if (headerTypes[j] == typeof(BooleanType)) + { + bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(Float32Type)) + { + // See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r + bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9"))); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(Float32LEType)) + { + if (LittleEndian) + { + fieldEncoded = BitConverter.GetBytes((float)data[i][j]); + } + else + { + byte[] floatBytes = BitConverter.GetBytes((float)data[i][j]); + fieldEncoded = new byte[sizeof(float)]; + for (int k = 0; k < sizeof(float); k++) + { + fieldEncoded[k] = floatBytes[sizeof(float) - 1 - k]; + } + } + } + else if (headerTypes[j] == typeof(Float64Type)) + { + // See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r + bytes.AddRange(Encoding.UTF8.GetBytes(((double)data[i][j]).ToString("G17"))); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(Float64LEType)) + { + if (LittleEndian) + { + fieldEncoded = BitConverter.GetBytes((double)data[i][j]); + } + else + { + byte[] doubleBytes = BitConverter.GetBytes((double)data[i][j]); + fieldEncoded = new byte[sizeof(double)]; + for (int k = 0; k < sizeof(double); k++) + { + fieldEncoded[k] = doubleBytes[sizeof(double) - 1 - k]; + } + } + } + else if (headerTypes[j] == typeof(UInt32Type)) + { + bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)data[i][j]).ToString())); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(UInt64Type)) + { + bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)data[i][j]).ToString())); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(Int32Type)) + { + bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)data[i][j]).ToString())); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(Int64Type)) + { + bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)data[i][j]).ToString())); + // In this case we know these values don't need escaping + continue; + } + else if (headerTypes[j] == typeof(BinaryType)) + { + fieldEncoded = (byte[])data[i][j]; + } + else + { + throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}"); + } + + for (int k = 0; k < fieldEncoded.Length; k++) + { + if (fieldEncoded[k] == '\n') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'n'); + } + else if (fieldEncoded[k] == '\t') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'t'); + } + else if (fieldEncoded[k] == '\\') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'\\'); + } + else if (fieldEncoded[k] == '#') + { + bytes.Add((byte)'\\'); + bytes.Add((byte)'#'); + } + else + { + bytes.Add(fieldEncoded[k]); + } + } + + if (j == headerNames.Count - 1) + { + bytes.Add((byte)'\n'); + } + else + { + bytes.Add((byte)'\t'); + } + } + catch (InvalidCastException e) + { + throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(headerTypes[j])}", e); + } + } + } + + return bytes.ToArray(); + } + public SaneTsvRecord this[int i] => Records[i]; public class SaneTsvRecord