nathanmcrae/sane-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nathanmcrae:52ed9495297390ecf13cd313774c905a7557f4da
Branches Tags
nathanmcrae:master
..
compare: nathanmcrae:ea77db46a62adfe134c5ab756ad61e94a84b909c
Branches Tags
nathanmcrae:master

No commits in common. "52ed9495297390ecf13cd313774c905a7557f4da" and "ea77db46a62adfe134c5ab756ad61e94a84b909c" have entirely different histories.
52ed949529 ... ea77db46a6

2 changed files with 56 additions and 166 deletions
Show Stats Expand all files Collapse all files

168
SaneTsv/SaneTsv.cs
View File

@ -394,42 +394,18 @@ public class SaneTsv
}
else if (parsed.ColumnTypes[j] == typeof(Float32Type))
{
float parsedFloat;
if (!float.TryParse(fieldString, out parsedFloat))
if (!float.TryParse(fieldString, out float parsedFloat))
{
if (fieldString == "-inf")
{
parsedFloat = float.NegativeInfinity;
}
else if (fieldString == "+inf")
{
parsedFloat = float.PositiveInfinity;
}
else
{
throw new Exception($"Field {j} on line {line} is not valid single-precision float");
}
throw new Exception($"Field {j} on line {line} is not valid single-precision float");
}
parsedFields[j] = parsedFloat;
}
else if (parsed.ColumnTypes[j] == typeof(Float64Type))
{
double parsedDouble;
if (!double.TryParse(fieldString, out parsedDouble))
if (!double.TryParse(fieldString, out double parsedDouble))
{
if (fieldString == "-inf")
{
parsedDouble = float.NegativeInfinity;
}
else if (fieldString == "+inf")
{
parsedDouble = float.PositiveInfinity;
}
else
{
throw new Exception($"Field {j} on line {line} is not valid double-precision float");
}
throw new Exception($"Field {j} on line {line} is not valid double-precision float");
}
parsedFields[j] = parsedDouble;
@ -749,9 +725,7 @@ public class SaneTsv
{
try
{
byte[] fieldEncoded = null;
// Some fields definitely don't need escaping, so we add them directly to bytes
bool skipEscaping = false;
byte[] fieldEncoded;
if (headerTypes[j] == typeof(StringType))
{
@ -760,31 +734,15 @@ public class SaneTsv
else if (headerTypes[j] == typeof(BooleanType))
{
bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded);
skipEscaping = true;
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float32Type))
{
if (data[i][j] is float f)
{
if (float.IsNegativeInfinity(f))
{
bytes.AddRange(Encoding.UTF8.GetBytes("-inf"));
}
else if (float.IsPositiveInfinity(f))
{
bytes.AddRange(Encoding.UTF8.GetBytes("+inf"));
}
else
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9")));
}
}
else
{
throw new InvalidCastException();
}
skipEscaping = true;
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9")));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float32LEType))
{
@ -804,27 +762,10 @@ public class SaneTsv
}
else if (headerTypes[j] == typeof(Float64Type))
{
if (data[i][j] is double d)
{
if (double.IsNegativeInfinity(d))
{
bytes.AddRange(Encoding.UTF8.GetBytes("-inf"));
}
else if (double.IsPositiveInfinity(d))
{
bytes.AddRange(Encoding.UTF8.GetBytes("+inf"));
}
else
{
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes((d).ToString("G17")));
}
}
else
{
throw new InvalidCastException();
}
skipEscaping = true;
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((double)data[i][j]).ToString("G17")));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Float64LEType))
{
@ -844,23 +785,27 @@ public class SaneTsv
}
else if (headerTypes[j] == typeof(UInt32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)(int)data[i][j]).ToString()));
skipEscaping = true;
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(UInt64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)(int)data[i][j]).ToString()));
skipEscaping = true;
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Int32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)(int)data[i][j]).ToString()));
skipEscaping = true;
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(Int64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)(int)data[i][j]).ToString()));
skipEscaping = true;
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)data[i][j]).ToString()));
// In this case we know these values don't need escaping
continue;
}
else if (headerTypes[j] == typeof(BinaryType))
{
@ -871,45 +816,42 @@ public class SaneTsv
throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
}
if (!skipEscaping)
for (int k = 0; k < fieldEncoded.Length; k++)
{
for (int k = 0; k < fieldEncoded.Length; k++)
if (fieldEncoded[k] == '\n')
{
if (fieldEncoded[k] == '\n')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'n');
}
else if (fieldEncoded[k] == '\t')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'t');
}
else if (fieldEncoded[k] == '\\')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'\\');
}
else if (fieldEncoded[k] == '#')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'#');
}
else
{
bytes.Add(fieldEncoded[k]);
}
bytes.Add((byte)'\\');
bytes.Add((byte)'n');
}
else if (fieldEncoded[k] == '\t')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'t');
}
else if (fieldEncoded[k] == '\\')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'\\');
}
else if (fieldEncoded[k] == '#')
{
bytes.Add((byte)'\\');
bytes.Add((byte)'#');
}
else
{
bytes.Add(fieldEncoded[k]);
}
}
if (j < data[i].Count - 1)
{
bytes.Add((byte)'\t');
}
else if (i < data.Count - 1)
if (j == headerNames.Count - 1)
{
bytes.Add((byte)'\n');
}
else
{
bytes.Add((byte)'\t');
}
}
catch (InvalidCastException e)
{

54
SaneTsv/SaneTsvTest/Program.cs
View File

@ -1,4 +1,5 @@
using NathanMcRae;
using System.Linq;
using System.Text;
{
@ -88,58 +89,5 @@ using System.Text;
Console.WriteLine("Done with tests");
{
string testName = "Serde test";
string[] headerNames =
{
"string-test",
"bool-test",
"float32-test",
"float32-le-test",
"float64-test",
"float64-le-test",
"uint32-test",
"uint64-test",
"int32-test",
"int64-test",
"binary-test",
};
Type[] headerTypes =
{
typeof(SaneTsv.StringType),
typeof(SaneTsv.BooleanType),
typeof(SaneTsv.Float32Type),
typeof(SaneTsv.Float32LEType),
typeof(SaneTsv.Float64Type),
typeof(SaneTsv.Float64LEType),
typeof(SaneTsv.UInt32Type),
typeof(SaneTsv.UInt64Type),
typeof(SaneTsv.Int32Type),
typeof(SaneTsv.Int64Type),
typeof(SaneTsv.BinaryType),
};
object[][] data =
{
new object[] { "test", true, 44.5f, 44.5f, -88e-3, -88e-3, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3 } },
new object[] { "test2", false, 44.5000005f, 44.5000005f, -88e-30, -88e-30, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.NaN, float.NaN, double.NaN, double.NaN, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.NegativeInfinity, float.NegativeInfinity, double.NegativeInfinity, double.NegativeInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.PositiveInfinity, float.PositiveInfinity, double.PositiveInfinity, double.PositiveInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
};
byte[] serialized = SaneTsv.SerializeTypedTsv(headerTypes, headerNames, data);
SaneTsv parsed = SaneTsv.ParseTypedTsv(serialized);
if ((float)parsed.Records[1]["float32-test"] == 44.5000005)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
// TODO: Check qNaN, sNaN, +inf, -inf values for float types