nathanmcrae/extra-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix Typed TSV serialization

Browse Source 
And add inf serializing/parsing to floats
This commit is contained in:
Nathan McRae 2024-02-17 20:44:07 -08:00
parent 457ef1df01
commit aee67e457e
1 changed files with 113 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

168
SaneTsv.cs
View File

@ -394,18 +394,42 @@ public class SaneTsv
} }
else if (parsed.ColumnTypes[j] == typeof(Float32Type)) else if (parsed.ColumnTypes[j] == typeof(Float32Type))
{ {
if (!float.TryParse(fieldString, out float parsedFloat)) float parsedFloat;
if (!float.TryParse(fieldString, out parsedFloat))
{ {
throw new Exception($"Field {j} on line {line} is not valid single-precision float"); if (fieldString == "-inf")
{
parsedFloat = float.NegativeInfinity;
}
else if (fieldString == "+inf")
{
parsedFloat = float.PositiveInfinity;
}
else
{
throw new Exception($"Field {j} on line {line} is not valid single-precision float");
}
} }
parsedFields[j] = parsedFloat; parsedFields[j] = parsedFloat;
} }
else if (parsed.ColumnTypes[j] == typeof(Float64Type)) else if (parsed.ColumnTypes[j] == typeof(Float64Type))
{ {
if (!double.TryParse(fieldString, out double parsedDouble)) double parsedDouble;
if (!double.TryParse(fieldString, out parsedDouble))
{ {
throw new Exception($"Field {j} on line {line} is not valid double-precision float"); if (fieldString == "-inf")
{
parsedDouble = float.NegativeInfinity;
}
else if (fieldString == "+inf")
{
parsedDouble = float.PositiveInfinity;
}
else
{
throw new Exception($"Field {j} on line {line} is not valid double-precision float");
}
} }
parsedFields[j] = parsedDouble; parsedFields[j] = parsedDouble;
@ -725,7 +749,9 @@ public class SaneTsv
{ {
try try
{ {
byte[] fieldEncoded; byte[] fieldEncoded = null;
// Some fields definitely don't need escaping, so we add them directly to bytes
bool skipEscaping = false;
if (headerTypes[j] == typeof(StringType)) if (headerTypes[j] == typeof(StringType))
{ {
@ -734,15 +760,31 @@ public class SaneTsv
else if (headerTypes[j] == typeof(BooleanType)) else if (headerTypes[j] == typeof(BooleanType))
{ {
bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded); bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded);
// In this case we know these values don't need escaping skipEscaping = true;
continue;
} }
else if (headerTypes[j] == typeof(Float32Type)) else if (headerTypes[j] == typeof(Float32Type))
{ {
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r if (data[i][j] is float f)
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9"))); {
// In this case we know these values don't need escaping if (float.IsNegativeInfinity(f))
continue; {
bytes.AddRange(Encoding.UTF8.GetBytes("-inf"));
}
else if (float.IsPositiveInfinity(f))
{
bytes.AddRange(Encoding.UTF8.GetBytes("+inf"));
}
else
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9")));
}
}
else
{
throw new InvalidCastException();
}
skipEscaping = true;
} }
else if (headerTypes[j] == typeof(Float32LEType)) else if (headerTypes[j] == typeof(Float32LEType))
{ {
@ -762,10 +804,27 @@ public class SaneTsv
} }
else if (headerTypes[j] == typeof(Float64Type)) else if (headerTypes[j] == typeof(Float64Type))
{ {
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r if (data[i][j] is double d)
bytes.AddRange(Encoding.UTF8.GetBytes(((double)data[i][j]).ToString("G17"))); {
// In this case we know these values don't need escaping if (double.IsNegativeInfinity(d))
continue; {
bytes.AddRange(Encoding.UTF8.GetBytes("-inf"));
}
else if (double.IsPositiveInfinity(d))
{
bytes.AddRange(Encoding.UTF8.GetBytes("+inf"));
}
else
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes((d).ToString("G17")));
}
}
else
{
throw new InvalidCastException();
}
skipEscaping = true;
} }
else if (headerTypes[j] == typeof(Float64LEType)) else if (headerTypes[j] == typeof(Float64LEType))
{ {
@ -785,27 +844,23 @@ public class SaneTsv
} }
else if (headerTypes[j] == typeof(UInt32Type)) else if (headerTypes[j] == typeof(UInt32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)(int)data[i][j]).ToString()));
// In this case we know these values don't need escaping skipEscaping = true;
continue;
} }
else if (headerTypes[j] == typeof(UInt64Type)) else if (headerTypes[j] == typeof(UInt64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)(int)data[i][j]).ToString()));
// In this case we know these values don't need escaping skipEscaping = true;
continue;
} }
else if (headerTypes[j] == typeof(Int32Type)) else if (headerTypes[j] == typeof(Int32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)(int)data[i][j]).ToString()));
// In this case we know these values don't need escaping skipEscaping = true;
continue;
} }
else if (headerTypes[j] == typeof(Int64Type)) else if (headerTypes[j] == typeof(Int64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)(int)data[i][j]).ToString()));
// In this case we know these values don't need escaping skipEscaping = true;
continue;
} }
else if (headerTypes[j] == typeof(BinaryType)) else if (headerTypes[j] == typeof(BinaryType))
{ {
@ -816,42 +871,45 @@ public class SaneTsv
throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}"); throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
} }
for (int k = 0; k < fieldEncoded.Length; k++) if (!skipEscaping)
{ {
if (fieldEncoded[k] == '\n') for (int k = 0; k < fieldEncoded.Length; k++)
{ {
bytes.Add((byte)'\\'); if (fieldEncoded[k] == '\n')
bytes.Add((byte)'n'); {
} bytes.Add((byte)'\\');
else if (fieldEncoded[k] == '\t') bytes.Add((byte)'n');
{ }
bytes.Add((byte)'\\'); else if (fieldEncoded[k] == '\t')
bytes.Add((byte)'t'); {
} bytes.Add((byte)'\\');
else if (fieldEncoded[k] == '\\') bytes.Add((byte)'t');
{ }
bytes.Add((byte)'\\'); else if (fieldEncoded[k] == '\\')
bytes.Add((byte)'\\'); {
} bytes.Add((byte)'\\');
else if (fieldEncoded[k] == '#') bytes.Add((byte)'\\');
{ }
bytes.Add((byte)'\\'); else if (fieldEncoded[k] == '#')
bytes.Add((byte)'#'); {
} bytes.Add((byte)'\\');
else bytes.Add((byte)'#');
{ }
bytes.Add(fieldEncoded[k]); else
{
bytes.Add(fieldEncoded[k]);
}
} }
} }
if (j == headerNames.Count - 1) if (j < data[i].Count - 1)
{
bytes.Add((byte)'\n');
}
else
{ {
bytes.Add((byte)'\t'); bytes.Add((byte)'\t');
} }
else if (i < data.Count - 1)
{
bytes.Add((byte)'\n');
}
} }
catch (InvalidCastException e) catch (InvalidCastException e)
{ {