Add Typed TSV serialization

SaneTsv/SaneTsv.cs

@@ -28,11 +28,26 @@ public class SaneTsv
     COMMENTED_TSV = 2,
   }
 
+  public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE");
+  public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE");
+
   // TODO: We need to be able to update all these in tandem somehow
   public string[] ColumnNames { get; protected set; }
   public Type[] ColumnTypes { get; protected set; }
   public List<SaneTsvRecord> Records { get; protected set; }
   public string FileComment { get; protected set; } = null;
+  protected static bool? _littleEndian = null;
+  public static bool LittleEndian
+  {
+    get
+    {
+      if (_littleEndian == null)
+      {
+        _littleEndian = BitConverter.GetBytes(double.NegativeInfinity)[7] == 255;
+      }
+      return _littleEndian.Value;
+    }
+  }
 
   public static SaneTsv ParseSimpleTsv(byte[] inputBuffer)
   {
@@ -133,7 +148,8 @@ public class SaneTsv
 
             string columnTypeString;
             string columnName;
-            if (columnString.Contains(':')) {
+            if (columnString.Contains(':'))
+            {
               if (format == FormatType.SIMPLE_TSV)
               {
                 throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
@@ -303,6 +319,44 @@ public class SaneTsv
         parsedFields[j] = fields[j];
         continue;
       }
+      else if (parsed.ColumnTypes[j] == typeof(Float32LEType))
+      {
+        byte[] floatBytes;
+        if (!LittleEndian)
+        {
+          floatBytes = new byte[sizeof(float)];
+          for (int k = 0; k < sizeof(float); k++)
+          {
+            floatBytes[k] = fields[j][sizeof(float) - 1 - k];
+          }
+        }
+        else
+        {
+          floatBytes = fields[j];
+        }
+        parsedFields[j] = BitConverter.ToSingle(floatBytes, 0);
+
+        continue;
+      }
+      else if (parsed.ColumnTypes[j] == typeof(Float64LEType))
+      {
+        byte[] floatBytes;
+        if (!LittleEndian)
+        {
+          floatBytes = new byte[sizeof(double)];
+          for (int k = 0; k < sizeof(double); k++)
+          {
+            floatBytes[k] = fields[j][sizeof(double) - 1 - k];
+          }
+        }
+        else
+        {
+          floatBytes = fields[j];
+        }
+        parsedFields[j] = BitConverter.ToDouble(floatBytes, 0);
+
+        continue;
+      }
 
       string fieldString;
       try
@@ -347,13 +401,6 @@ public class SaneTsv
 
         parsedFields[j] = parsedFloat;
       }
-      else if (parsed.ColumnTypes[j] == typeof(Float32LEType))
-      {
-        throw new NotImplementedException();
-        // TODO: Implement and do byte-swapping if necessary
-
-        //parsedFields[j] = parsedFloat;
-      }
       else if (parsed.ColumnTypes[j] == typeof(Float64Type))
       {
         if (!double.TryParse(fieldString, out double parsedDouble))
@@ -363,13 +410,6 @@ public class SaneTsv
 
         parsedFields[j] = parsedDouble;
       }
-      else if (parsed.ColumnTypes[j] == typeof(Float64LEType))
-      {
-        throw new NotImplementedException();
-        // TODO: Implement and do byte-swapping if necessary
-
-        //parsedFields[j] = parsedFloat;
-      }
       else if (parsed.ColumnTypes[j] == typeof(UInt32Type))
       {
         if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
@@ -471,7 +511,7 @@ public class SaneTsv
 
     // Serialize data
     for (int i = 0; i < data.Count; i++)
-    { 
+    {
       for (int j = 0; j < data[i].Count; j++)
       {
         for (int k = 0; k < data[i][j].Length; k++)
@@ -512,6 +552,317 @@ public class SaneTsv
     return Encoding.UTF8.GetBytes(escapedString.ToString());
   }
 
+  public static Type GetColumnFromType(Type type)
+  {
+    if (type == typeof(string))
+    {
+      return typeof(StringType);
+    }
+    else if (type == typeof(bool))
+    {
+      return typeof(BooleanType);
+    }
+    else if (type == typeof(float))
+    {
+      return typeof(Float32Type);
+    }
+    else if (type == typeof(double))
+    {
+      return typeof(Float64Type);
+    }
+    else if (type == typeof(UInt32))
+    {
+      return typeof(UInt32Type);
+    }
+    else if (type == typeof(UInt64))
+    {
+      return typeof(UInt64Type);
+    }
+    else if (type == typeof(Int32))
+    {
+      return typeof(Int32Type);
+    }
+    else if (type == typeof(Int64))
+    {
+      return typeof(Int64Type);
+    }
+    else if (type == typeof(byte[]))
+    {
+      return typeof(BinaryType);
+    }
+    else
+    {
+      throw new Exception($"Invalid type: {type}");
+    }
+  }
+
+  public static string GetNameFromColumn(Type type)
+  {
+    if (type == typeof(StringType))
+    {
+      return "string";
+    }
+    else if (type == typeof(BooleanType))
+    {
+      return "boolean";
+    }
+    else if (type == typeof(Float32Type))
+    {
+      return "float32";
+    }
+    else if (type == typeof(Float32LEType))
+    {
+      return "float32-le";
+    }
+    else if (type == typeof(Float64Type))
+    {
+      return "float64";
+    }
+    else if (type == typeof(Float64LEType))
+    {
+      return "float64-le";
+    }
+    else if (type == typeof(UInt32Type))
+    {
+      return "uint32";
+    }
+    else if (type == typeof(UInt64Type))
+    {
+      return "uint64";
+    }
+    else if (type == typeof(Int32Type))
+    {
+      return "int32";
+    }
+    else if (type == typeof(Int64Type))
+    {
+      return "int64";
+    }
+    else if (type == typeof(BinaryType))
+    {
+      return "binary";
+    }
+    else
+    {
+      throw new Exception($"Invalid type: {type}");
+    }
+  }
+
+  public static byte[] SerializeTypedTsv(IList<Type> headerTypes, IList<string> headerNames, IList<IList<object>> data)
+  {
+    var bytes = new List<byte>();
+
+    if (headerNames.Count != headerTypes.Count)
+    {
+      throw new ArgumentException($"headerTypes length ({headerTypes.Count}) does not match headerNames length ({headerNames.Count})");
+    }
+
+    // Serialize header
+    for (int i = 0; i < headerNames.Count; i++)
+    {
+      for (int j = i + 1; j < headerNames.Count; j++)
+      {
+        if (headerNames[i] == headerNames[j])
+        {
+          throw new Exception("Column names in header must be unique");
+        }
+      }
+
+      byte[] nameEncoded = Encoding.UTF8.GetBytes(headerNames[i]);
+
+      for (int j = 0; j < nameEncoded.Length; j++)
+      {
+        if (nameEncoded[j] == '\n')
+        {
+          bytes.Add((byte)'\\');
+          bytes.Add((byte)'n');
+        }
+        else if (nameEncoded[j] == '\t')
+        {
+          bytes.Add((byte)'\\');
+          bytes.Add((byte)'t');
+        }
+        else if (nameEncoded[j] == '\\')
+        {
+          bytes.Add((byte)'\\');
+          bytes.Add((byte)'\\');
+        }
+        else if (nameEncoded[j] == '#')
+        {
+          bytes.Add((byte)'\\');
+          bytes.Add((byte)'#');
+        }
+        else
+        {
+          bytes.Add(nameEncoded[j]);
+        }
+      }
+
+      bytes.Add((byte)':');
+      try
+      {
+        bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(headerTypes[i])));
+      }
+      catch (Exception e)
+      {
+        throw new Exception($"Invalid header type for column {i}", e);
+      }
+
+      if (i == headerNames.Count - 1)
+      {
+        bytes.Add((byte)'\n');
+      }
+      else
+      {
+        bytes.Add((byte)'\t');
+      }
+    }
+
+    // Serialize data
+    for (int i = 0; i < data.Count; i++)
+    {
+      for (int j = 0; j < data[i].Count; j++)
+      {
+        try
+        {
+          byte[] fieldEncoded;
+
+          if (headerTypes[j] == typeof(StringType))
+          {
+            fieldEncoded = Encoding.UTF8.GetBytes((string)data[i][j]);
+          }
+          else if (headerTypes[j] == typeof(BooleanType))
+          {
+            bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded);
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(Float32Type))
+          {
+            // See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
+            bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9")));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(Float32LEType))
+          {
+            if (LittleEndian)
+            {
+              fieldEncoded = BitConverter.GetBytes((float)data[i][j]);
+            }
+            else
+            {
+              byte[] floatBytes = BitConverter.GetBytes((float)data[i][j]);
+              fieldEncoded = new byte[sizeof(float)];
+              for (int k = 0; k < sizeof(float); k++)
+              {
+                fieldEncoded[k] = floatBytes[sizeof(float) - 1 - k];
+              }
+            }
+          }
+          else if (headerTypes[j] == typeof(Float64Type))
+          {
+            // See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
+            bytes.AddRange(Encoding.UTF8.GetBytes(((double)data[i][j]).ToString("G17")));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(Float64LEType))
+          {
+            if (LittleEndian)
+            {
+              fieldEncoded = BitConverter.GetBytes((double)data[i][j]);
+            }
+            else
+            {
+              byte[] doubleBytes = BitConverter.GetBytes((double)data[i][j]);
+              fieldEncoded = new byte[sizeof(double)];
+              for (int k = 0; k < sizeof(double); k++)
+              {
+                fieldEncoded[k] = doubleBytes[sizeof(double) - 1 - k];
+              }
+            }
+          }
+          else if (headerTypes[j] == typeof(UInt32Type))
+          {
+            bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)data[i][j]).ToString()));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(UInt64Type))
+          {
+            bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)data[i][j]).ToString()));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(Int32Type))
+          {
+            bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)data[i][j]).ToString()));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(Int64Type))
+          {
+            bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)data[i][j]).ToString()));
+            // In this case we know these values don't need escaping
+            continue;
+          }
+          else if (headerTypes[j] == typeof(BinaryType))
+          {
+            fieldEncoded = (byte[])data[i][j];
+          }
+          else
+          {
+            throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
+          }
+
+          for (int k = 0; k < fieldEncoded.Length; k++)
+          {
+            if (fieldEncoded[k] == '\n')
+            {
+              bytes.Add((byte)'\\');
+              bytes.Add((byte)'n');
+            }
+            else if (fieldEncoded[k] == '\t')
+            {
+              bytes.Add((byte)'\\');
+              bytes.Add((byte)'t');
+            }
+            else if (fieldEncoded[k] == '\\')
+            {
+              bytes.Add((byte)'\\');
+              bytes.Add((byte)'\\');
+            }
+            else if (fieldEncoded[k] == '#')
+            {
+              bytes.Add((byte)'\\');
+              bytes.Add((byte)'#');
+            }
+            else
+            {
+              bytes.Add(fieldEncoded[k]);
+            }
+          }
+
+          if (j == headerNames.Count - 1)
+          {
+            bytes.Add((byte)'\n');
+          }
+          else
+          {
+            bytes.Add((byte)'\t');
+          }
+        }
+        catch (InvalidCastException e)
+        {
+          throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(headerTypes[j])}", e);
+        }
+      }
+    }
+
+    return bytes.ToArray();
+  }
+
   public SaneTsvRecord this[int i] => Records[i];
 
   public class SaneTsvRecord

SaneTsv/SaneTsvTest/Program.cs

@@ -1,4 +1,5 @@
 using NathanMcRae;
+using System.Linq;
 using System.Text;
 
 {
@@ -68,6 +69,24 @@ using System.Text;
   }
 }
 
+{
+  string testName = "Float binary test";
+  var bytes = new List<byte>();
+  bytes.AddRange(Encoding.UTF8.GetBytes("somefloat:float64\tbinfloat:float64-le" +
+    "\n1.5\t")); bytes.AddRange(BitConverter.GetBytes(1.5));
+  bytes.AddRange(Encoding.UTF8.GetBytes("\n-8.0000005E-14\t")); bytes.AddRange(BitConverter.GetBytes(-8.0000005E-14));
+
+  SaneTsv parsed = SaneTsv.ParseTypedTsv(bytes.ToArray());
+  if ((double)parsed.Records[0]["binfloat"] == (double)parsed.Records[0]["somefloat"])
+  {
+    Console.WriteLine($"Passed {testName}");
+  }
+  else
+  {
+    Console.WriteLine($"Failed {testName}");
+  }
+}
+
 Console.WriteLine("Done with tests");