nathanmcrae/extra-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nathanmcrae:a7e6f1c0e91a4a186879ef3c1726adc74ca21abd
Branches Tags
nathanmcrae:master
...
compare: nathanmcrae:8e3332b484966982d31d52ff06f8f5330d47ac42
Branches Tags
nathanmcrae:master

10 Commits
a7e6f1c0e9 ... 8e3332b484

Author SHA1 Message Date
Nathan McRae
8e3332b484 Remove unused SaneTsv fields 2024-03-10 12:55:21 -07:00
Nathan McRae
32393e704d Clarify terminology of 'header'/'column' 
The header is just the region containing the column name / types.
 2024-03-09 13:13:41 -08:00
Nathan McRae
d7720d8cde Flesh out tests 2024-03-09 09:58:25 -08:00
Nathan McRae
aff4b353bb Update some comments 2024-03-09 09:58:09 -08:00
Nathan McRae
203458fdf7 Fix some end-of-file simple TSV parsing 2024-03-09 09:57:56 -08:00
Nathan McRae
4148475031 Fix some minor line/column numbering issues 2024-03-09 09:57:27 -08:00
Nathan McRae
e38baa9167 Fix column attribute bug 2024-03-09 09:57:00 -08:00
Nathan McRae
a66f6a1368 Make parallel implementations the default 2024-03-08 12:31:40 -08:00
Nathan McRae
96af5ae82c Complete parallel parsing implementation 2024-03-08 11:43:06 -08:00
Nathan McRae
11c7daec8e Start parallel versions of general TSV serialization/parsing 
They mostly work, but are not actually parallelized yet and likely have some edge cases.
Also, the soon-to-be parallel version of parsing is very slow compared to the original.
 2024-02-25 22:35:56 -08:00
2 changed files with 731 additions and 511 deletions
Show Stats Expand all files Collapse all files

785
SaneTsv.cs
View File

File diff suppressed because it is too large Load Diff

457
SaneTsvTest/Program.cs
View File

@ -1,7 +1,8 @@
using NathanMcRae; using NathanMcRae;
using System.Reflection;
using System.Text; using System.Text;
internal class Program internal class Program : SaneTsv
{ {
public class TestRecord : SaneTsv.TsvRecord public class TestRecord : SaneTsv.TsvRecord
{ {
@ -80,6 +81,18 @@ internal class Program
public string Column3 { get; set; } public string Column3 { get; set; }
} }
public class BoolTestRecord3 : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TsvColumn("column1")]
public string Column1 { get; set; }
[SaneTsv.TsvColumn]
public string column2 { get; set; }
[SaneTsv.TsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
{ {
[SaneTsv.TypedTsvColumn("column1")] [SaneTsv.TypedTsvColumn("column1")]
@ -349,45 +362,26 @@ internal class Program
} }
{ {
string testName = "Check parallel serialization"; string testName = "With and without file comment";
int N = 100000; string testString1 = "#This is a file comment\n" +
var records = new StringTestRecord[N]; "#One more file comment line\n" +
var rand = new Random(1); "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
for (int i = 0; i < N; i++) string testString2 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
{ "\n#This is a comment" +
records[i] = new StringTestRecord() "\n#Another comment line" +
{ "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
Column1 = rand.Next().ToString(), "\nFALSE\tnother\tno\\ther";
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray(); CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
CommentedTsv<BoolTestRecord2> parsed2 = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2));
DateTime lastTime = DateTime.Now; if (parsed.FileComment == "This is a file comment\nOne more file comment line" && parsed2.FileComment == null)
byte[] serialized1 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeSimpleTsvParallel(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
{
if (serialized1[i] != serialized2[i])
{
matching = false;
break;
}
}
if (matching)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -398,63 +392,370 @@ internal class Program
} }
{ {
string testName = "Check parallel parsing"; string testName = "With and without types";
int N = 100000; string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
var records = new StringTestRecord[N]; "\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
var rand = new Random(1); "\nFALSE\tnother\tno\\ther";
for (int i = 0; i < N; i++) try
{ {
records[i] = new StringTestRecord() Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
{ Console.WriteLine($"Passed {testName} 1A");
Column1 = rand.Next().ToString(), }
column2 = rand.Next().ToString(), catch (Exception e)
Column3 = rand.Next().ToString(), {
}; Console.WriteLine($"Failed {testName} 1A");
} }
byte[] serialized = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records); try
DateTime lastTime = DateTime.Now;
(string[] headers2, string[][] data2) = SaneTsv.ParseSimpleTsv(serialized);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
(string[] headers, string[][] data) = SaneTsv.ParseSimpleTsvParallel(serialized);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int j = 0; j < Math.Min(headers2.Length, headers.Length); j++)
{ {
if (headers[j] != headers2[j]) Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 2A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 2A");
}
try
{
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 2B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 2B");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 2C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 2C");
}
}
{
string testName = "With and without line comment";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Passed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Failed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
try
{
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
}
}
{
string testName = "End of file comment";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
}
}
{
string testName = "Partial parsing";
string line1 = "column1\tcolumn2\tcolumnthree\\nyep";
string line2 = "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee";
string line3 = "\nFALSE\tnother\tno\\ther";
byte[] inputBuffer = Encoding.UTF8.GetBytes(line1 + line2 + line3);
var headerTypes = new List<Type>();
var headerNames = new List<string>();
var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
foreach (PropertyInfo property in typeof(BoolTestRecord3).GetProperties())
{
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
if (attribute == null)
{ {
matching = false; continue;
break;
} }
headerNames.Add(attribute.ColumnName ?? property.Name);
headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible
columnCount++;
} }
for (int i = 0; i < Math.Min(data.Length, data2.Length) && matching; i++) BoolTestRecord3[] records = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
{ FormatType.SIMPLE_TSV,
for (int j = 0; j < data[0].Length; j++) headerPropertyInfos.ToArray(),
{ headerTypes.ToArray(),
if (data[i][j] != data2[i][j]) line1.Length + line2.Length + 1,
{ inputBuffer.Length);
matching = false;
break;
}
}
}
if (matching) if (records.Length == 0 )
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName} 1");
} }
else else
{ {
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName} 1");
}
BoolTestRecord3[] records2 = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
FormatType.SIMPLE_TSV,
headerPropertyInfos.ToArray(),
headerTypes.ToArray(),
line1.Length,
line1.Length + 3);
if (records2[0].Column3 == "valuetrhee")
{
Console.WriteLine($"Passed {testName} 2");
}
else
{
Console.WriteLine($"Failed {testName} 2");
}
string[][] data = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length + line2.Length + 1, inputBuffer.Length);
if (data[0][1] == "nother")
{
Console.WriteLine($"Passed {testName} 3");
}
else
{
Console.WriteLine($"Failed {testName} 3");
}
string[][] data2 = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length, line1.Length + 3);
if (data2.Length == 0)
{
Console.WriteLine($"Passed {testName} 4");
}
else
{
Console.WriteLine($"Failed {testName} 4");
}
}
{
string testName = "End of file \\n";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
}
}
{
string testName = "End of file partial record";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\nTRUE\t";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\nTRUE\t";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
} }
} }