Compare commits
15 Commits
a7e6f1c0e9
...
master
Author | SHA1 | Date | |
---|---|---|---|
2b9bcd1a6d | |||
78574d1872 | |||
77b679bbdc | |||
0807f0ffc2 | |||
71af3653c4 | |||
8e3332b484 | |||
32393e704d | |||
d7720d8cde | |||
aff4b353bb | |||
203458fdf7 | |||
4148475031 | |||
e38baa9167 | |||
a66f6a1368 | |||
96af5ae82c | |||
11c7daec8e |
@ -1,125 +0,0 @@
|
|||||||
|
|
||||||
using System.Globalization;
|
|
||||||
using System.Text.RegularExpressions;
|
|
||||||
|
|
||||||
namespace NathanMcRae;
|
|
||||||
|
|
||||||
public class ExtraTsv : SaneTsv
|
|
||||||
{
|
|
||||||
public class Iso8601Type : ColumnType { }
|
|
||||||
public class PhysicalUnitsType : ColumnType
|
|
||||||
{
|
|
||||||
public string Units { get; }
|
|
||||||
public PhysicalUnitsType(string Units) { }
|
|
||||||
}
|
|
||||||
|
|
||||||
public static readonly string[] ValidUnits =
|
|
||||||
{
|
|
||||||
"m",
|
|
||||||
"s",
|
|
||||||
"A",
|
|
||||||
"K",
|
|
||||||
"cd",
|
|
||||||
"mol",
|
|
||||||
"kg",
|
|
||||||
"Hz",
|
|
||||||
"rad",
|
|
||||||
"sr",
|
|
||||||
"N",
|
|
||||||
"Pa",
|
|
||||||
"J",
|
|
||||||
"W",
|
|
||||||
"C",
|
|
||||||
"V",
|
|
||||||
"F",
|
|
||||||
"Ω",
|
|
||||||
"S",
|
|
||||||
"Wb",
|
|
||||||
"T",
|
|
||||||
"H",
|
|
||||||
"°C",
|
|
||||||
"lm",
|
|
||||||
"lx",
|
|
||||||
"Bq",
|
|
||||||
"Gy",
|
|
||||||
"Sv",
|
|
||||||
"kat"
|
|
||||||
};
|
|
||||||
|
|
||||||
public static readonly int MajorVersion = 0;
|
|
||||||
public static readonly int MinorVersion = 0;
|
|
||||||
public static readonly int PatchVersion = 1;
|
|
||||||
|
|
||||||
public static Regex VersionRegex = new Regex(@"^ ExtraTSV V(\d+)\.(\d+)\.(\d+)");
|
|
||||||
|
|
||||||
public static ExtraTsv ParseExtraTsv(byte[] inputBuffer)
|
|
||||||
{
|
|
||||||
SaneTsv tsv = ParseCommentedTsv(inputBuffer);
|
|
||||||
|
|
||||||
if (tsv.FileComment == null) {
|
|
||||||
throw new Exception($"ExtraTSV expects the file to start with '# ExtraTSV Vx.y.z' where x.y.z is a version compatible with {MajorVersion}.{MinorVersion}.{PatchVersion}");
|
|
||||||
}
|
|
||||||
|
|
||||||
Match match = VersionRegex.Match(tsv.FileComment);
|
|
||||||
if (!match.Success)
|
|
||||||
{
|
|
||||||
throw new Exception($"ExtraTSV expects the file to start with '# ExtraTSV Vx.y.z' where x.y.z is a version compatible with {MajorVersion}.{MinorVersion}.{PatchVersion}");
|
|
||||||
}
|
|
||||||
|
|
||||||
int fileMajorVersion = int.Parse(match.Groups[1].Value);
|
|
||||||
|
|
||||||
if (fileMajorVersion != MajorVersion)
|
|
||||||
{
|
|
||||||
throw new Exception($"File has major version ({fileMajorVersion}) which is newer than this parser's version {MajorVersion}");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < tsv.ColumnNames.Count(); i++)
|
|
||||||
{
|
|
||||||
string[] typeParts = tsv.ColumnNames[i].Split(":");
|
|
||||||
if (typeParts[typeParts.Length - 1] == "iso8601" && tsv.ColumnTypes[i] == typeof(StringType))
|
|
||||||
{
|
|
||||||
string columnName = tsv.ColumnNames[i].Substring(0, tsv.ColumnNames[i].Length - ":iso8601".Length);
|
|
||||||
tsv.ColumnNames[i] = columnName;
|
|
||||||
tsv.ColumnTypes[i] = typeof(Iso8601Type);
|
|
||||||
}
|
|
||||||
// TODO: ISO8601 time spans
|
|
||||||
// TODO: ISO8601 time durations
|
|
||||||
else if (typeParts[typeParts.Length - 1] == "units" && (tsv.ColumnTypes[i] == typeof(Float64Type) || tsv.ColumnTypes[i] == typeof(Float32Type)))
|
|
||||||
{
|
|
||||||
if (typeParts.Count() > 1 && ValidUnits.Contains(typeParts[typeParts.Length - 2]))
|
|
||||||
{
|
|
||||||
// TODO: How to store type information since the ColumnTypes is of type Type?
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
throw new Exception($"Invalid units type '{typeParts[typeParts.Length - 2]}' for column {i}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
CultureInfo provider = CultureInfo.InvariantCulture;
|
|
||||||
|
|
||||||
for (int i = 0; i < tsv.Records.Count; i++)
|
|
||||||
{
|
|
||||||
if (tsv.Records[i].Comment != null)
|
|
||||||
{
|
|
||||||
throw new Exception($"Line {tsv.Records[i].Line} has comment above it which is not allowed");
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int j = 0; j < tsv.ColumnNames.Count(); j++)
|
|
||||||
{
|
|
||||||
if (tsv.ColumnTypes[j] == typeof(Iso8601Type))
|
|
||||||
{
|
|
||||||
if (!DateTime.TryParseExact((string)tsv.Records[i][j], "yyyy-MM-ddTHH:mm:ss.ffff", provider, DateTimeStyles.None, out DateTime parsed))
|
|
||||||
{
|
|
||||||
throw new Exception($"ISO 8601 timestamp format error on line {tsv.Records[i].Line}, field {j}");
|
|
||||||
}
|
|
||||||
|
|
||||||
tsv.Records[i].Fields[j] = parsed;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return (ExtraTsv)tsv;
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,14 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
<RootNamespace>NathanMcRae</RootNamespace>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\SaneTsv.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
@ -1,44 +0,0 @@
|
|||||||
Extra TSV adds many convenience types to Sane TSV:
|
|
||||||
|
|
||||||
- Timestamps
|
|
||||||
Just this format for now: yyyy-MM-ddTHH:mm:ss.ffff
|
|
||||||
- Timespans
|
|
||||||
- Time durations
|
|
||||||
- Multiformats
|
|
||||||
- Multihashes
|
|
||||||
- Multiprotocols
|
|
||||||
- ...
|
|
||||||
- Physical units
|
|
||||||
To start with, just use SI base and derived units
|
|
||||||
- Base units
|
|
||||||
- m
|
|
||||||
- s
|
|
||||||
- A
|
|
||||||
- K
|
|
||||||
- cd
|
|
||||||
- mol
|
|
||||||
- kg
|
|
||||||
- Derived units
|
|
||||||
- Hz
|
|
||||||
- rad
|
|
||||||
- sr
|
|
||||||
- N
|
|
||||||
- Pa
|
|
||||||
- J
|
|
||||||
- W
|
|
||||||
- C
|
|
||||||
- V
|
|
||||||
- F
|
|
||||||
- Ω
|
|
||||||
- S
|
|
||||||
- Wb
|
|
||||||
- T
|
|
||||||
- H
|
|
||||||
- °C
|
|
||||||
- lm
|
|
||||||
- lx
|
|
||||||
- Bq
|
|
||||||
- Gy
|
|
||||||
- Sv
|
|
||||||
- kat
|
|
||||||
How to handle derived units?
|
|
@ -1,14 +0,0 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
|
||||||
|
|
||||||
<PropertyGroup>
|
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<TargetFramework>net6.0</TargetFramework>
|
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
|
||||||
<Nullable>enable</Nullable>
|
|
||||||
</PropertyGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
|
||||||
<ProjectReference Include="..\ExtraTsv\ExtraTsv.csproj" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
</Project>
|
|
@ -1,20 +0,0 @@
|
|||||||
using NathanMcRae;
|
|
||||||
using System.Text;
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Bool test";
|
|
||||||
string testString1 = "# ExtraTSV V0.0.1\n" +
|
|
||||||
"column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
|
|
||||||
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
|
|
||||||
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
|
|
||||||
|
|
||||||
ExtraTsv parsed = ExtraTsv.ParseExtraTsv(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
if (parsed.Records[0]["column1:ty#pe"] is bool result && result)
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Passed {testName}");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
1275
SaneTsv.cs
1275
SaneTsv.cs
File diff suppressed because it is too large
Load Diff
@ -21,4 +21,8 @@
|
|||||||
<None Remove="SaneTsvTest\**" />
|
<None Remove="SaneTsvTest\**" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="UnitsNet" Version="6.0.0-pre004" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
12
SaneTsv.sln
12
SaneTsv.sln
@ -7,10 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsv", "SaneTsv.csproj",
|
|||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsvTest", "SaneTsvTest\SaneTsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}"
|
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsvTest", "SaneTsvTest\SaneTsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExtraTsv", "ExtraTsv\ExtraTsv.csproj", "{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}"
|
|
||||||
EndProject
|
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExtraTsvTest", "ExtraTsvTest\ExtraTsvTest.csproj", "{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}"
|
|
||||||
EndProject
|
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
Debug|Any CPU = Debug|Any CPU
|
Debug|Any CPU = Debug|Any CPU
|
||||||
@ -25,14 +21,6 @@ Global
|
|||||||
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.Build.0 = Release|Any CPU
|
{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{D9F2E9C8-4F52-4BB7-9BBD-AE9A0C6168E7}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
|
||||||
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
|
||||||
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
|
||||||
{A545B0DB-F799-43E2-9DFA-C18BDF3535F1}.Release|Any CPU.Build.0 = Release|Any CPU
|
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
@ -1,62 +1,9 @@
|
|||||||
using NathanMcRae;
|
using NathanMcRae;
|
||||||
|
using System.Reflection;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
internal class Program : SaneTsv
|
||||||
internal class Program
|
|
||||||
{
|
{
|
||||||
public class TestRecord : SaneTsv.TsvRecord
|
public class DateTest : SaneTsv.CommentedTsvRecord
|
||||||
{
|
|
||||||
[SaneTsv.TypedTsvColumn("string-test")]
|
|
||||||
public string StringTest { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("bool-test")]
|
|
||||||
public bool BoolTest { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("float32-test")]
|
|
||||||
public float Float32Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("float32-le-test", typeof(SaneTsv.Float32LEType))]
|
|
||||||
public float Float32LETest { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("float64-test")]
|
|
||||||
public double Float64Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("float64-le-test", typeof(SaneTsv.Float64LEType))]
|
|
||||||
public double Float64LETest { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("uint32-test")]
|
|
||||||
public UInt32 UInt32Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("uint64-test")]
|
|
||||||
public UInt64 UInt64Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("int32-test")]
|
|
||||||
public Int32 Int32Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("int64-test")]
|
|
||||||
public Int64 Int64Test { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("binary-test")]
|
|
||||||
public byte[] BinaryTest { get; set; }
|
|
||||||
|
|
||||||
public TestRecord(string stringTest, bool boolTest, float float32Test, float float32LETest, double float64Test, double float64LETest, UInt32 uInt32Test, UInt64 uInt64Test, Int32 int32Test, Int64 int64Test, byte[] binaryTest)
|
|
||||||
{
|
|
||||||
StringTest = stringTest;
|
|
||||||
BoolTest = boolTest;
|
|
||||||
Float32Test = float32Test;
|
|
||||||
Float32LETest = float32LETest;
|
|
||||||
Float64Test = float64Test;
|
|
||||||
Float64LETest = float64LETest;
|
|
||||||
UInt32Test = uInt32Test;
|
|
||||||
UInt64Test = uInt64Test;
|
|
||||||
Int32Test = int32Test;
|
|
||||||
Int64Test = int64Test;
|
|
||||||
BinaryTest = binaryTest;
|
|
||||||
}
|
|
||||||
|
|
||||||
public TestRecord() { }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class BoolTestRecord : SaneTsv.CommentedTsvRecord
|
|
||||||
{
|
{
|
||||||
[SaneTsv.TypedTsvColumn("column1:ty#pe")]
|
[SaneTsv.TypedTsvColumn("column1:ty#pe")]
|
||||||
public bool Column1 { get; set; }
|
public bool Column1 { get; set; }
|
||||||
@ -65,63 +12,28 @@ internal class Program
|
|||||||
public byte[] column2 { get; set; }
|
public byte[] column2 { get; set; }
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
|
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
|
||||||
public string Column3 { get; set; }
|
public DateTime Column3 { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class BoolTestRecord2 : SaneTsv.CommentedTsvRecord
|
public class UnitTest : SaneTsv.CommentedTsvRecord
|
||||||
{
|
{
|
||||||
[SaneTsv.TypedTsvColumn("column1:type")]
|
[SaneTsv.TypedTsvColumn("id")]
|
||||||
public bool Column1 { get; set; }
|
public UInt32 Id { get; set; }
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn]
|
[SaneTsv.TypedTsvColumn("value", "m/s:ph-unit:float64")]
|
||||||
public byte[] column2 { get; set; }
|
public UnitsNet.Speed Value { get; set; }
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
|
|
||||||
public string Column3 { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
|
|
||||||
{
|
|
||||||
[SaneTsv.TypedTsvColumn("column1")]
|
|
||||||
public bool Column1 { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn]
|
|
||||||
public byte[] column2 { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
|
|
||||||
public string Column3 { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class FloatTestRecord : SaneTsv.CommentedTsvRecord
|
|
||||||
{
|
|
||||||
[SaneTsv.TypedTsvColumn("somefloat")]
|
|
||||||
public double SomeFloat { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("binfloat", typeof(SaneTsv.Float64LEType))]
|
|
||||||
public double BinFloat { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class StringTestRecord : SaneTsv.TsvRecord
|
|
||||||
{
|
|
||||||
[SaneTsv.TypedTsvColumn("column1")]
|
|
||||||
public string Column1 { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn]
|
|
||||||
public string column2 { get; set; }
|
|
||||||
|
|
||||||
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
|
|
||||||
public string Column3 { get; set; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void Main(string[] args)
|
private static void Main(string[] args)
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
string testName = "Bool test";
|
string testName = "Parse date";
|
||||||
string testString1 = "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
string testString1 = "# ExtraTSV V0.0.1\n" +
|
||||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
"column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
|
||||||
"\nFALSE\tnother\tno\\ther";
|
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
|
||||||
|
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
|
||||||
|
|
||||||
Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1));
|
CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
|
||||||
if (parsed.Records[0].Column1)
|
if (parsed.Records[0].Column1)
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Passed {testName}");
|
Console.WriteLine($"Passed {testName}");
|
||||||
@ -133,64 +45,33 @@ internal class Program
|
|||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
string testName = "Bad bool test";
|
string testName = "Bad date column name";
|
||||||
|
string testString1 = "# ExtraTSV V0.0.1\n" +
|
||||||
|
"column1:ty\\#pe:boolean\tcolumn2:binary\tiso8601:string" +
|
||||||
|
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
|
||||||
|
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
|
||||||
"\nTUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
|
||||||
"\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
Console.WriteLine($"Failed {testName}");
|
||||||
}
|
}
|
||||||
catch (Exception)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Passed {testName}");
|
Console.WriteLine($"Passed {testName}");
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
string testName = "Comment test";
|
string testName = "Serde date";
|
||||||
string testString1 = "#This is a file comment\n" +
|
string testString1 = "# ExtraTSV V0.0.1\n" +
|
||||||
"#One more file comment line\n" +
|
"column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:iso8601:string" +
|
||||||
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
"\nTRUE\tvalue\\\\t\0woo\t2024-02-15T18:03:30.0000" +
|
||||||
"\n#This is a comment" +
|
"\nFALSE\tnother\t2024-02-15T18:03:39.0001";
|
||||||
"\n#Another comment line" +
|
|
||||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
|
||||||
"\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
CommentedTsv<DateTest> parsed = SaneTsv.ParseExtraTsv<DateTest>(Encoding.UTF8.GetBytes(testString1));
|
||||||
}
|
string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeExtraTsv<DateTest>(parsed.Records));
|
||||||
|
if (serialized == testString1)
|
||||||
//{
|
|
||||||
// string testName = "Serde test";
|
|
||||||
// string testString1 = "column1\tcolumn2\tcolumnthree\\nyep" +
|
|
||||||
// "\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
|
|
||||||
// "\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
// Tsv<SerdeTestRecord> parsed = SaneTsv.ParseSimpleTsv<SerdeTestRecord>(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
// string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeSimpleTsv(parsed.ColumnNames, parsed.Records.Select(r => r.Fields.Select(f => f.ToString()).ToArray()).ToArray()));
|
|
||||||
|
|
||||||
// if (testString1 == serialized)
|
|
||||||
// {
|
|
||||||
// Console.WriteLine($"Passed {testName}");
|
|
||||||
// }
|
|
||||||
// else
|
|
||||||
// {
|
|
||||||
// Console.WriteLine($"Failed {testName}");
|
|
||||||
// }
|
|
||||||
//}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Float binary test";
|
|
||||||
var bytes = new List<byte>();
|
|
||||||
bytes.AddRange(Encoding.UTF8.GetBytes("somefloat:float64\tbinfloat:float64-le" +
|
|
||||||
"\n1.5\t")); bytes.AddRange(BitConverter.GetBytes(1.5));
|
|
||||||
bytes.AddRange(Encoding.UTF8.GetBytes("\n-8.0000005E-14\t")); bytes.AddRange(BitConverter.GetBytes(-8.0000005E-14));
|
|
||||||
|
|
||||||
Tsv<FloatTestRecord> parsed = SaneTsv.ParseTypedTsv<FloatTestRecord>(bytes.ToArray());
|
|
||||||
if (parsed.Records[0].BinFloat == parsed.Records[0].SomeFloat)
|
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Passed {testName}");
|
Console.WriteLine($"Passed {testName}");
|
||||||
}
|
}
|
||||||
@ -201,254 +82,57 @@ internal class Program
|
|||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
string testName = "Serde test";
|
string testName = "Parse unit";
|
||||||
|
string testString1 = "# ExtraTSV V0.0.1\n" +
|
||||||
|
"id:uint32\tvalue:m/s:ph-unit:float64\n" +
|
||||||
|
"0\t1.5\n" +
|
||||||
|
"1\t5.4e3";
|
||||||
|
|
||||||
TestRecord[] data =
|
CommentedTsv<UnitTest> parsed = SaneTsv.ParseExtraTsv<UnitTest>(Encoding.UTF8.GetBytes(testString1));
|
||||||
|
if (parsed.Records[0].Value.Value == 1.5)
|
||||||
{
|
{
|
||||||
new TestRecord("test", true, 44.5f, 44.5f, -88e-3, -88e-3, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3 }),
|
Console.WriteLine($"Passed {testName}");
|
||||||
new TestRecord("test2", false, 44.5000005f, 44.5000005f, -88e-30, -88e-30, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
|
}
|
||||||
new TestRecord("test2", false, float.NaN, float.NaN, double.NaN, double.NaN, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
|
else
|
||||||
new TestRecord("test2", false, float.NegativeInfinity, float.NegativeInfinity, double.NegativeInfinity, double.NegativeInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
|
{
|
||||||
new TestRecord("test2", false, float.PositiveInfinity, float.PositiveInfinity, double.PositiveInfinity, double.PositiveInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
|
Console.WriteLine($"Failed {testName}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
string testName = "Serde unit";
|
||||||
|
|
||||||
|
UnitTest[] records = new UnitTest[]
|
||||||
|
{
|
||||||
|
new UnitTest
|
||||||
|
{
|
||||||
|
Id = 1,
|
||||||
|
Value = UnitsNet.Speed.FromMetersPerSecond(5.03),
|
||||||
|
},
|
||||||
|
new UnitTest
|
||||||
|
{
|
||||||
|
Id = 5,
|
||||||
|
Value = UnitsNet.Speed.FromMetersPerSecond(double.NaN),
|
||||||
|
},
|
||||||
|
new UnitTest
|
||||||
|
{
|
||||||
|
Id = 1000000,
|
||||||
|
Value = UnitsNet.Speed.FromMetersPerSecond(9859873.498),
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
byte[] serialized = SaneTsv.SerializeTypedTsv(data);
|
string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeExtraTsv<UnitTest>(records));
|
||||||
|
CommentedTsv<UnitTest> parsed = SaneTsv.ParseExtraTsv<UnitTest>(Encoding.UTF8.GetBytes(serialized));
|
||||||
|
|
||||||
Tsv<TestRecord> parsed = SaneTsv.ParseTypedTsv<TestRecord>(serialized);
|
bool match = true;
|
||||||
|
for (int i = 0; i < records.Length; i++)
|
||||||
if ((float)parsed.Records[1].Float32Test == 44.5000005f)
|
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Passed {testName}");
|
match = match && records[i].Id == parsed.Records[i].Id &&
|
||||||
}
|
(records[i].Value.Equals(parsed.Records[i].Value, UnitsNet.Speed.FromMetersPerSecond(1E-1))
|
||||||
else
|
|| (double.IsNaN(records[i].Value.Value) && double.IsNaN(parsed.Records[i].Value.Value)));
|
||||||
{
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Trying to parse a not commented record as a Commented TSV test";
|
|
||||||
|
|
||||||
// These should not compile:
|
|
||||||
|
|
||||||
//byte[] serialized = SaneTsv.SerializeCommentedTsv(data);
|
|
||||||
// Gives this error: error CS7036: There is no argument given that corresponds to the required parameter 'fileComment' of 'SaneTsv.SerializeCommentedTsv<T>(IList<T>, string)'
|
|
||||||
|
|
||||||
//Tsv<TestRecord> parsed = SaneTsv.ParseCommentedTsv<TestRecord>(serialized);
|
|
||||||
// Gives this error: error CS0311: The type 'Program.TestRecord' cannot be used as type parameter 'T' in the generic type or method 'SaneTsv.ParseCommentedTsv<T>(byte[])'. There is no implicit reference conversion from 'Program.TestRecord' to 'NathanMcRae.SaneTsv.CommentedTsvRecord'.
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Try to parsed a Commented TSV as a Simple TSV";
|
|
||||||
|
|
||||||
string testString1 = "#This is a file comment\n" +
|
|
||||||
"#One more file comment line\n" +
|
|
||||||
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
|
||||||
"\n#This is a comment" +
|
|
||||||
"\n#Another comment line" +
|
|
||||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
|
||||||
"\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Passed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Try to parsed a Commented TSV as a Typed TSV";
|
|
||||||
|
|
||||||
string testString1 = "#This is a file comment\n" +
|
|
||||||
"#One more file comment line\n" +
|
|
||||||
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
|
||||||
"\n#This is a comment" +
|
|
||||||
"\n#Another comment line" +
|
|
||||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
|
||||||
"\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Passed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Try to parsed a Typed TSV as a Simple TSV";
|
|
||||||
|
|
||||||
string testString1 =
|
|
||||||
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
|
||||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
|
||||||
"\nFALSE\tnother\tno\\ther";
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
|
||||||
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Passed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Timing comparison of simple parse methods and comparison of simple serialization methods";
|
|
||||||
|
|
||||||
int N = 1000000;
|
|
||||||
var records = new StringTestRecord[N];
|
|
||||||
var rand = new Random(1);
|
|
||||||
|
|
||||||
for (int i = 0; i < N; i++)
|
|
||||||
{
|
|
||||||
records[i] = new StringTestRecord()
|
|
||||||
{
|
|
||||||
Column1 = rand.Next().ToString(),
|
|
||||||
column2 = rand.Next().ToString(),
|
|
||||||
Column3 = rand.Next().ToString(),
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray();
|
if (match)
|
||||||
|
|
||||||
DateTime lastTime = DateTime.Now;
|
|
||||||
byte[] serialized1 = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
|
|
||||||
|
|
||||||
TimeSpan speccedSerializationTime = DateTime.Now - lastTime;
|
|
||||||
Console.WriteLine($"Specced serialization time: {speccedSerializationTime}");
|
|
||||||
lastTime = DateTime.Now;
|
|
||||||
|
|
||||||
byte[] serialized2 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
|
|
||||||
|
|
||||||
TimeSpan unspeccedSerializationTime = DateTime.Now - lastTime;
|
|
||||||
Console.WriteLine($"Unspecced serialization time: {unspeccedSerializationTime}");
|
|
||||||
lastTime = DateTime.Now;
|
|
||||||
|
|
||||||
Tsv<StringTestRecord> parsed = SaneTsv.ParseSimpleTsv<StringTestRecord>(serialized1);
|
|
||||||
|
|
||||||
TimeSpan speccedParseTime = DateTime.Now - lastTime;
|
|
||||||
Console.WriteLine($"Specced parse time: {speccedParseTime}");
|
|
||||||
lastTime = DateTime.Now;
|
|
||||||
|
|
||||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(serialized2);
|
|
||||||
|
|
||||||
TimeSpan unspeccedParseTime = DateTime.Now - lastTime;
|
|
||||||
Console.WriteLine($"Unspecced parse time: {unspeccedParseTime}");
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Check parallel serialization";
|
|
||||||
|
|
||||||
int N = 100000;
|
|
||||||
var records = new StringTestRecord[N];
|
|
||||||
var rand = new Random(1);
|
|
||||||
|
|
||||||
for (int i = 0; i < N; i++)
|
|
||||||
{
|
|
||||||
records[i] = new StringTestRecord()
|
|
||||||
{
|
|
||||||
Column1 = rand.Next().ToString(),
|
|
||||||
column2 = rand.Next().ToString(),
|
|
||||||
Column3 = rand.Next().ToString(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray();
|
|
||||||
|
|
||||||
DateTime lastTime = DateTime.Now;
|
|
||||||
byte[] serialized1 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
|
|
||||||
TimeSpan unparallelTime = DateTime.Now - lastTime;
|
|
||||||
lastTime = DateTime.Now;
|
|
||||||
byte[] serialized2 = SaneTsv.SerializeSimpleTsvParallel(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
|
|
||||||
TimeSpan parallelTime = DateTime.Now - lastTime;
|
|
||||||
|
|
||||||
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
|
|
||||||
Console.WriteLine($"Parallel serialization time: {parallelTime}");
|
|
||||||
|
|
||||||
bool matching = true;
|
|
||||||
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
|
|
||||||
{
|
|
||||||
if (serialized1[i] != serialized2[i])
|
|
||||||
{
|
|
||||||
matching = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (matching)
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Passed {testName}");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
Console.WriteLine($"Failed {testName}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
string testName = "Check parallel parsing";
|
|
||||||
|
|
||||||
int N = 100000;
|
|
||||||
var records = new StringTestRecord[N];
|
|
||||||
var rand = new Random(1);
|
|
||||||
|
|
||||||
for (int i = 0; i < N; i++)
|
|
||||||
{
|
|
||||||
records[i] = new StringTestRecord()
|
|
||||||
{
|
|
||||||
Column1 = rand.Next().ToString(),
|
|
||||||
column2 = rand.Next().ToString(),
|
|
||||||
Column3 = rand.Next().ToString(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
byte[] serialized = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
|
|
||||||
|
|
||||||
DateTime lastTime = DateTime.Now;
|
|
||||||
(string[] headers2, string[][] data2) = SaneTsv.ParseSimpleTsv(serialized);
|
|
||||||
TimeSpan unparallelTime = DateTime.Now - lastTime;
|
|
||||||
lastTime = DateTime.Now;
|
|
||||||
(string[] headers, string[][] data) = SaneTsv.ParseSimpleTsvParallel(serialized);
|
|
||||||
TimeSpan parallelTime = DateTime.Now - lastTime;
|
|
||||||
|
|
||||||
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
|
|
||||||
Console.WriteLine($"Parallel serialization time: {parallelTime}");
|
|
||||||
|
|
||||||
bool matching = true;
|
|
||||||
for (int j = 0; j < Math.Min(headers2.Length, headers.Length); j++)
|
|
||||||
{
|
|
||||||
if (headers[j] != headers2[j])
|
|
||||||
{
|
|
||||||
matching = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < Math.Min(data.Length, data2.Length) && matching; i++)
|
|
||||||
{
|
|
||||||
for (int j = 0; j < data[0].Length; j++)
|
|
||||||
{
|
|
||||||
if (data[i][j] != data2[i][j])
|
|
||||||
{
|
|
||||||
matching = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (matching)
|
|
||||||
{
|
{
|
||||||
Console.WriteLine($"Passed {testName}");
|
Console.WriteLine($"Passed {testName}");
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user