Compare commits

...

15 Commits

Author SHA1 Message Date
2b9bcd1a6d Fix unit name serialization
We just store the unit name as specified and serialize that (instead of using the name as given in UnitsNet)
2024-03-15 23:15:35 -07:00
78574d1872 Add basic implementation of physical units
Serde still isn't working, need to store the exact format of the unit as specified in the attribute
2024-03-15 18:58:07 -07:00
77b679bbdc Remove old ExtraTsv stuff 2024-03-10 23:26:42 -07:00
0807f0ffc2 Add iso8601 parsing 2024-03-10 22:43:21 -07:00
71af3653c4 Change column type management
The column types were tracked just as a Type. This changes them to be an instance
so they can track additional information (such as the specific units of a physical
units type). Becuase of this, the column type attributes need the type to be passed as strings.
(see CS0181).
2024-03-10 22:35:30 -07:00
8e3332b484 Remove unused SaneTsv fields 2024-03-10 12:55:21 -07:00
32393e704d Clarify terminology of 'header'/'column'
The header is just the region containing the column name / types.
2024-03-09 13:13:41 -08:00
d7720d8cde Flesh out tests 2024-03-09 09:58:25 -08:00
aff4b353bb Update some comments 2024-03-09 09:58:09 -08:00
203458fdf7 Fix some end-of-file simple TSV parsing 2024-03-09 09:57:56 -08:00
4148475031 Fix some minor line/column numbering issues 2024-03-09 09:57:27 -08:00
e38baa9167 Fix column attribute bug 2024-03-09 09:57:00 -08:00
a66f6a1368 Make parallel implementations the default 2024-03-08 12:31:40 -08:00
96af5ae82c Complete parallel parsing implementation 2024-03-08 11:43:06 -08:00
11c7daec8e Start parallel versions of general TSV serialization/parsing
They mostly work, but are not actually parallelized yet and likely have some edge cases.
Also, the soon-to-be parallel version of parsing is very slow compared to the original.
2024-02-25 22:35:56 -08:00
9 changed files with 827 additions and 1147 deletions

View File

@ -1,125 +0,0 @@

using System.Globalization;
using System.Text.RegularExpressions;
namespace NathanMcRae;
public class ExtraTsv : SaneTsv
{
public class Iso8601Type : ColumnType { }
public class PhysicalUnitsType : ColumnType
{
public string Units { get; }
public PhysicalUnitsType(string Units) { }
}
public static readonly string[] ValidUnits =
{
"m",
"s",
"A",
"K",
"cd",
"mol",
"kg",
"Hz",
"rad",
"sr",
"N",
"Pa",
"J",
"W",
"C",
"V",
"F",
"Ω",
"S",
"Wb",
"T",
"H",
"°C",
"lm",
"lx",
"Bq",
"Gy",
"Sv",
"kat"
};
public static readonly int MajorVersion = 0;
public static readonly int MinorVersion = 0;
public static readonly int PatchVersion = 1;
public static Regex VersionRegex = new Regex(@"^ ExtraTSV V(\d+)\.(\d+)\.(\d+)");
public static ExtraTsv ParseExtraTsv(byte[] inputBuffer)
{
SaneTsv tsv = ParseCommentedTsv(inputBuffer);
if (tsv.FileComment == null) {
throw new Exception($"ExtraTSV expects the file to start with '# ExtraTSV Vx.y.z' where x.y.z is a version compatible with {MajorVersion}.{MinorVersion}.{PatchVersion}");
}
Match match = VersionRegex.Match(tsv.FileComment);
if (!match.Success)
{
throw new Exception($"ExtraTSV expects the file to start with '# ExtraTSV Vx.y.z' where x.y.z is a version compatible with {MajorVersion}.{MinorVersion}.{PatchVersion}");
}
int fileMajorVersion = int.Parse(match.Groups[1].Value);
if (fileMajorVersion != MajorVersion)
{
throw new Exception($"File has major version ({fileMajorVersion}) which is newer than this parser's version {MajorVersion}");
}
for (int i = 0; i < tsv.ColumnNames.Count(); i++)
{
string[] typeParts = tsv.ColumnNames[i].Split(":");
if (typeParts[typeParts.Length - 1] == "iso8601" && tsv.ColumnTypes[i] == typeof(StringType))
{
string columnName = tsv.ColumnNames[i].Substring(0, tsv.ColumnNames[i].Length - ":iso8601".Length);
tsv.ColumnNames[i] = columnName;
tsv.ColumnTypes[i] = typeof(Iso8601Type);
}
// TODO: ISO8601 time spans
// TODO: ISO8601 time durations
else if (typeParts[typeParts.Length - 1] == "units" && (tsv.ColumnTypes[i] == typeof(Float64Type) || tsv.ColumnTypes[i] == typeof(Float32Type)))
{
if (typeParts.Count() > 1 && ValidUnits.Contains(typeParts[typeParts.Length - 2]))
{
// TODO: How to store type information since the ColumnTypes is of type Type?
}
else
{
throw new Exception($"Invalid units type '{typeParts[typeParts.Length - 2]}' for column {i}");
}
}
}
CultureInfo provider = CultureInfo.InvariantCulture;
for (int i = 0; i < tsv.Records.Count; i++)
{
if (tsv.Records[i].Comment != null)
{
throw new Exception($"Line {tsv.Records[i].Line} has comment above it which is not allowed");
}
for (int j = 0; j < tsv.ColumnNames.Count(); j++)
{
if (tsv.ColumnTypes[j] == typeof(Iso8601Type))
{
if (!DateTime.TryParseExact((string)tsv.Records[i][j], "yyyy-MM-ddTHH:mm:ss.ffff", provider, DateTimeStyles.None, out DateTime parsed))
{
throw new Exception($"ISO 8601 timestamp format error on line {tsv.Records[i].Line}, field {j}");
}
tsv.Records[i].Fields[j] = parsed;
}
}
}
return (ExtraTsv)tsv;
}
}

View File

@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>NathanMcRae</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\SaneTsv.csproj" />
</ItemGroup>
</Project>

View File

@ -1,44 +0,0 @@
Extra TSV adds many convenience types to Sane TSV:
- Timestamps
Just this format for now: yyyy-MM-ddTHH:mm:ss.ffff
- Timespans
- Time durations
- Multiformats
- Multihashes
- Multiprotocols
- ...
- Physical units
To start with, just use SI base and derived units
- Base units
- m
- s
- A
- K
- cd
- mol
- kg
- Derived units
- Hz
- rad
- sr
- N
- Pa
- J
- W
- C
- V
- F
- Ω
- S
- Wb
- T
- H
- °C
- lm
- lx
- Bq
- Gy
- Sv
- kat
How to handle derived units?

View File

@ -1,14 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\ExtraTsv\ExtraTsv.csproj" />
</ItemGroup>
</Project>

View File

@ -1,20 +0,0 @@
using NathanMcRae;
using System.Text;
{
string testName = "Bool test";
string testString1 = "# ExtraTSV V0.0.1\n" +
"column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
ExtraTsv parsed = ExtraTsv.ParseExtraTsv(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0]["column1:ty#pe"] is bool result && result)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}

1275
SaneTsv.cs

File diff suppressed because it is too large Load Diff

View File

@ -21,4 +21,8 @@
<None Remove="SaneTsvTest\**" /> <None Remove="SaneTsvTest\**" />
</ItemGroup> </ItemGroup>
<ItemGroup>
<PackageReference Include="UnitsNet" Version="6.0.0-pre004" />
</ItemGroup>
</Project> </Project>

View File

@ -7,10 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsv", "SaneTsv.csproj",
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsvTest", "SaneTsvTest\SaneTsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsvTest", "SaneTsvTest\SaneTsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}"
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExtraTsv", "ExtraTsv\ExtraTsv.csproj", "{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExtraTsvTest", "ExtraTsvTest\ExtraTsvTest.csproj", "{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}"
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU
@ -25,14 +21,6 @@ Global
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Debug|Any CPU.Build.0 = Debug|Any CPU {43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.ActiveCfg = Release|Any CPU {43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.Build.0 = Release|Any CPU {43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.Build.0 = Release|Any CPU
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Release|Any CPU.Build.0 = Release|Any CPU
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
GlobalSection(SolutionProperties) = preSolution GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE HideSolutionNode = FALSE

View File

@ -1,62 +1,9 @@
using NathanMcRae; using NathanMcRae;
using System.Reflection;
using System.Text; using System.Text;
internal class Program : SaneTsv
internal class Program
{ {
public class TestRecord : SaneTsv.TsvRecord public class DateTest : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("string-test")]
public string StringTest { get; set; }
[SaneTsv.TypedTsvColumn("bool-test")]
public bool BoolTest { get; set; }
[SaneTsv.TypedTsvColumn("float32-test")]
public float Float32Test { get; set; }
[SaneTsv.TypedTsvColumn("float32-le-test", typeof(SaneTsv.Float32LEType))]
public float Float32LETest { get; set; }
[SaneTsv.TypedTsvColumn("float64-test")]
public double Float64Test { get; set; }
[SaneTsv.TypedTsvColumn("float64-le-test", typeof(SaneTsv.Float64LEType))]
public double Float64LETest { get; set; }
[SaneTsv.TypedTsvColumn("uint32-test")]
public UInt32 UInt32Test { get; set; }
[SaneTsv.TypedTsvColumn("uint64-test")]
public UInt64 UInt64Test { get; set; }
[SaneTsv.TypedTsvColumn("int32-test")]
public Int32 Int32Test { get; set; }
[SaneTsv.TypedTsvColumn("int64-test")]
public Int64 Int64Test { get; set; }
[SaneTsv.TypedTsvColumn("binary-test")]
public byte[] BinaryTest { get; set; }
public TestRecord(string stringTest, bool boolTest, float float32Test, float float32LETest, double float64Test, double float64LETest, UInt32 uInt32Test, UInt64 uInt64Test, Int32 int32Test, Int64 int64Test, byte[] binaryTest)
{
StringTest = stringTest;
BoolTest = boolTest;
Float32Test = float32Test;
Float32LETest = float32LETest;
Float64Test = float64Test;
Float64LETest = float64LETest;
UInt32Test = uInt32Test;
UInt64Test = uInt64Test;
Int32Test = int32Test;
Int64Test = int64Test;
BinaryTest = binaryTest;
}
public TestRecord() { }
}
public class BoolTestRecord : SaneTsv.CommentedTsvRecord
{ {
[SaneTsv.TypedTsvColumn("column1:ty#pe")] [SaneTsv.TypedTsvColumn("column1:ty#pe")]
public bool Column1 { get; set; } public bool Column1 { get; set; }
@ -65,63 +12,28 @@ internal class Program
public byte[] column2 { get; set; } public byte[] column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")] [SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; } public DateTime Column3 { get; set; }
} }
public class BoolTestRecord2 : SaneTsv.CommentedTsvRecord public class UnitTest : SaneTsv.CommentedTsvRecord
{ {
[SaneTsv.TypedTsvColumn("column1:type")] [SaneTsv.TypedTsvColumn("id")]
public bool Column1 { get; set; } public UInt32 Id { get; set; }
[SaneTsv.TypedTsvColumn] [SaneTsv.TypedTsvColumn("value", "m/s:ph-unit:float64")]
public byte[] column2 { get; set; } public UnitsNet.Speed Value { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("column1")]
public bool Column1 { get; set; }
[SaneTsv.TypedTsvColumn]
public byte[] column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class FloatTestRecord : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("somefloat")]
public double SomeFloat { get; set; }
[SaneTsv.TypedTsvColumn("binfloat", typeof(SaneTsv.Float64LEType))]
public double BinFloat { get; set; }
}
public class StringTestRecord : SaneTsv.TsvRecord
{
[SaneTsv.TypedTsvColumn("column1")]
public string Column1 { get; set; }
[SaneTsv.TypedTsvColumn]
public string column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
} }
private static void Main(string[] args) private static void Main(string[] args)
{ {
{ {
string testName = "Bool test"; string testName = "Parse date";
string testString1 = "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + string testString1 = "# ExtraTSV V0.0.1\n" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" + "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
"\nFALSE\tnother\tno\\ther"; "\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1)); CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0].Column1) if (parsed.Records[0].Column1)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
@ -133,64 +45,33 @@ internal class Program
} }
{ {
string testName = "Bad bool test"; string testName = "Bad date column name";
string testString1 = "# ExtraTSV V0.0.1\n" +
"column1:ty\\#pe:boolean\tcolumn2:binary\tiso8601:string" +
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
try try
{ {
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
"\nTUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName}");
} }
catch (Exception) catch (Exception e)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
} }
{ {
string testName = "Comment test"; string testName = "Serde date";
string testString1 = "#This is a file comment\n" + string testString1 = "# ExtraTSV V0.0.1\n" +
"#One more file comment line\n" + "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + "\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
"\n#This is a comment" + "\nFALSE\tnother\t2024-02-15T18:03:39.0001";
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1)); CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
} string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeExtraTsv<DateTest>(parsed.Records));
if (serialized == testString1)
//{
// string testName = "Serde test";
// string testString1 = "column1\tcolumn2\tcolumnthree\\nyep" +
// "\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
// "\nFALSE\tnother\tno\\ther";
// Tsv<SerdeTestRecord> parsed = SaneTsv.ParseSimpleTsv<SerdeTestRecord>(Encoding.UTF8.GetBytes(testString1));
// string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeSimpleTsv(parsed.ColumnNames, parsed.Records.Select(r => r.Fields.Select(f => f.ToString()).ToArray()).ToArray()));
// if (testString1 == serialized)
// {
// Console.WriteLine($"Passed {testName}");
// }
// else
// {
// Console.WriteLine($"Failed {testName}");
// }
//}
{
string testName = "Float binary test";
var bytes = new List<byte>();
bytes.AddRange(Encoding.UTF8.GetBytes("somefloat:float64\tbinfloat:float64-le" +
"\n1.5\t")); bytes.AddRange(BitConverter.GetBytes(1.5));
bytes.AddRange(Encoding.UTF8.GetBytes("\n-8.0000005E-14\t")); bytes.AddRange(BitConverter.GetBytes(-8.0000005E-14));
Tsv<FloatTestRecord> parsed = SaneTsv.ParseTypedTsv<FloatTestRecord>(bytes.ToArray());
if (parsed.Records[0].BinFloat == parsed.Records[0].SomeFloat)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -201,254 +82,57 @@ internal class Program
} }
{ {
string testName = "Serde test"; string testName = "Parse unit";
string testString1 = "# ExtraTSV V0.0.1\n" +
"id:uint32\tvalue:m/s:ph-unit:float64\n" +
"0\t1.5\n" +
"1\t5.4e3";
TestRecord[] data = CommentedTsv<UnitTest> parsed = SaneTsv.ParseExtraTsv<UnitTest>(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0].Value.Value == 1.5)
{ {
new TestRecord("test", true, 44.5f, 44.5f, -88e-3, -88e-3, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3 }), Console.WriteLine($"Passed {testName}");
new TestRecord("test2", false, 44.5000005f, 44.5000005f, -88e-30, -88e-30, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }), }
new TestRecord("test2", false, float.NaN, float.NaN, double.NaN, double.NaN, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }), else
new TestRecord("test2", false, float.NegativeInfinity, float.NegativeInfinity, double.NegativeInfinity, double.NegativeInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }), {
new TestRecord("test2", false, float.PositiveInfinity, float.PositiveInfinity, double.PositiveInfinity, double.PositiveInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }), Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Serde unit";
UnitTest[] records = new UnitTest[]
{
new UnitTest
{
Id = 1,
Value = UnitsNet.Speed.FromMetersPerSecond(5.03),
},
new UnitTest
{
Id = 5,
Value = UnitsNet.Speed.FromMetersPerSecond(double.NaN),
},
new UnitTest
{
Id = 1000000,
Value = UnitsNet.Speed.FromMetersPerSecond(9859873.498),
},
}; };
byte[] serialized = SaneTsv.SerializeTypedTsv(data); string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeExtraTsv<UnitTest>(records));
CommentedTsv<UnitTest> parsed = SaneTsv.ParseExtraTsv<UnitTest>(Encoding.UTF8.GetBytes(serialized));
Tsv<TestRecord> parsed = SaneTsv.ParseTypedTsv<TestRecord>(serialized); bool match = true;
for (int i = 0; i < records.Length; i++)
if ((float)parsed.Records[1].Float32Test == 44.5000005f)
{ {
Console.WriteLine($"Passed {testName}"); match = match && records[i].Id == parsed.Records[i].Id &&
} (records[i].Value.Equals(parsed.Records[i].Value, UnitsNet.Speed.FromMetersPerSecond(1E-1))
else || (double.IsNaN(records[i].Value.Value) && double.IsNaN(parsed.Records[i].Value.Value)));
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Trying to parse a not commented record as a Commented TSV test";
// These should not compile:
//byte[] serialized = SaneTsv.SerializeCommentedTsv(data);
// Gives this error: error CS7036: There is no argument given that corresponds to the required parameter 'fileComment' of 'SaneTsv.SerializeCommentedTsv<T>(IList<T>, string)'
//Tsv<TestRecord> parsed = SaneTsv.ParseCommentedTsv<TestRecord>(serialized);
// Gives this error: error CS0311: The type 'Program.TestRecord' cannot be used as type parameter 'T' in the generic type or method 'SaneTsv.ParseCommentedTsv<T>(byte[])'. There is no implicit reference conversion from 'Program.TestRecord' to 'NathanMcRae.SaneTsv.CommentedTsvRecord'.
}
{
string testName = "Try to parsed a Commented TSV as a Simple TSV";
string testString1 = "#This is a file comment\n" +
"#One more file comment line\n" +
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
{
string testName = "Try to parsed a Commented TSV as a Typed TSV";
string testString1 = "#This is a file comment\n" +
"#One more file comment line\n" +
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
{
string testName = "Try to parsed a Typed TSV as a Simple TSV";
string testString1 =
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
{
string testName = "Timing comparison of simple parse methods and comparison of simple serialization methods";
int N = 1000000;
var records = new StringTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
records[i] = new StringTestRecord()
{
Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
} }
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray(); if (match)
DateTime lastTime = DateTime.Now;
byte[] serialized1 = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
TimeSpan speccedSerializationTime = DateTime.Now - lastTime;
Console.WriteLine($"Specced serialization time: {speccedSerializationTime}");
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan unspeccedSerializationTime = DateTime.Now - lastTime;
Console.WriteLine($"Unspecced serialization time: {unspeccedSerializationTime}");
lastTime = DateTime.Now;
Tsv<StringTestRecord> parsed = SaneTsv.ParseSimpleTsv<StringTestRecord>(serialized1);
TimeSpan speccedParseTime = DateTime.Now - lastTime;
Console.WriteLine($"Specced parse time: {speccedParseTime}");
lastTime = DateTime.Now;
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(serialized2);
TimeSpan unspeccedParseTime = DateTime.Now - lastTime;
Console.WriteLine($"Unspecced parse time: {unspeccedParseTime}");
}
{
string testName = "Check parallel serialization";
int N = 100000;
var records = new StringTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
records[i] = new StringTestRecord()
{
Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray();
DateTime lastTime = DateTime.Now;
byte[] serialized1 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeSimpleTsvParallel(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
{
if (serialized1[i] != serialized2[i])
{
matching = false;
break;
}
}
if (matching)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Check parallel parsing";
int N = 100000;
var records = new StringTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
records[i] = new StringTestRecord()
{
Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
byte[] serialized = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
DateTime lastTime = DateTime.Now;
(string[] headers2, string[][] data2) = SaneTsv.ParseSimpleTsv(serialized);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
(string[] headers, string[][] data) = SaneTsv.ParseSimpleTsvParallel(serialized);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int j = 0; j < Math.Min(headers2.Length, headers.Length); j++)
{
if (headers[j] != headers2[j])
{
matching = false;
break;
}
}
for (int i = 0; i < Math.Min(data.Length, data2.Length) && matching; i++)
{
for (int j = 0; j < data[0].Length; j++)
{
if (data[i][j] != data2[i][j])
{
matching = false;
break;
}
}
}
if (matching)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }