nathanmcrae/extra-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make parallel implementations the default

Browse Source
This commit is contained in:
Nathan McRae
2024-03-08 12:31:40 -08:00
parent 96af5ae82c
commit a66f6a1368
2 changed files with 11 additions and 1031 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
SaneTsvTest/Program.cs
View File

@ -348,222 +348,6 @@ internal class Program : SaneTsv
Console.WriteLine($"Unspecced parse time: {unspeccedParseTime}");
}
{
string testName = "Check parallel Simple TSV serialization";
int N = 100000;
var records = new StringTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
records[i] = new StringTestRecord()
{
Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray();
DateTime lastTime = DateTime.Now;
byte[] serialized1 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeSimpleTsvParallel(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
{
if (serialized1[i] != serialized2[i])
{
matching = false;
break;
}
}
if (matching)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Check Simple TSV parallel parsing";
int N = 100000;
var records = new StringTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
records[i] = new StringTestRecord()
{
Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
byte[] serialized = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
DateTime lastTime = DateTime.Now;
(string[] headers2, string[][] data2) = SaneTsv.ParseSimpleTsv(serialized);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
(string[] headers, string[][] data) = SaneTsv.ParseSimpleTsvParallel(serialized);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel parse time: {unparallelTime}");
Console.WriteLine($"Parallel parse time: {parallelTime}");
bool matching = true;
for (int j = 0; j < Math.Min(headers2.Length, headers.Length); j++)
{
if (headers[j] != headers2[j])
{
matching = false;
break;
}
}
for (int i = 0; i < Math.Min(data.Length, data2.Length) && matching; i++)
{
for (int j = 0; j < data[0].Length; j++)
{
if (data[i][j] != data2[i][j])
{
matching = false;
break;
}
}
}
if (matching)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Check parallel serialization";
int N = 1000;
var records = new BoolTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
byte[] bytes = new byte[rand.Next(50)];
rand.NextBytes(bytes);
records[i] = new BoolTestRecord()
{
Column1 = rand.NextDouble() > 0.5,
column2 = bytes,
Column3 = rand.Next().ToString(),
};
}
DateTime lastTime = DateTime.Now;
byte[] serialized1 = SaneTsv.SerializeTsv<BoolTestRecord>(records, FormatType.COMMENTED_TSV);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeTsvParallel<BoolTestRecord>(records, FormatType.COMMENTED_TSV);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
{
if (serialized1[i] != serialized2[i])
{
matching = false;
break;
}
}
if (matching)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Check parallel parsing";
int N = 1000000;
var records = new BoolTestRecord[N];
var rand = new Random(1);
for (int i = 0; i < N; i++)
{
byte[] bytes = new byte[rand.Next(50)];
rand.NextBytes(bytes);
records[i] = new BoolTestRecord()
{
Column1 = rand.NextDouble() > 0.5,
column2 = bytes,
Column3 = rand.Next().ToString(),
};
}
byte[] serialized2 = SaneTsv.SerializeTsvParallel<BoolTestRecord>(records, FormatType.COMMENTED_TSV);
DateTime lastTime = DateTime.Now;
CommentedTsv<BoolTestRecord> parsed = (CommentedTsv<BoolTestRecord>)SaneTsv.Parse<BoolTestRecord>(serialized2, FormatType.COMMENTED_TSV);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
CommentedTsv<BoolTestRecord> parsed2 = (CommentedTsv<BoolTestRecord>)SaneTsv.ParseParallel<BoolTestRecord>(serialized2, FormatType.COMMENTED_TSV);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel parsing time: {unparallelTime}");
Console.WriteLine($"Parallel parsing time: {parallelTime}");
bool matching = parsed.FileComment == parsed2.FileComment;
matching &= parsed.Records.Count == parsed2.Records.Count;
for (int i = 0; matching && i < parsed.Records.Count; i++)
{
matching &= parsed.Records[i].Comment == parsed2.Records[i].Comment;
matching &= parsed.Records[i].Column1 == parsed2.Records[i].Column1;
matching &= parsed.Records[i].column2.Length == parsed2.Records[i].column2.Length;
for (int j = 0; matching && j < parsed.Records[i].column2.Length; j++)
{
matching &= parsed.Records[i].column2[j] == parsed2.Records[i].column2[j];
}
}
if (matching)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
Console.WriteLine("Done with tests");
}
}