nathanmcrae/sane-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nathanmcrae:ab72d875bf233e2e42e12dc428274c33e03cd67d
Branches Tags
nathanmcrae:master
..
compare: nathanmcrae:4dd3a4878ebae90cd1a5d1a8725388f859c80cf9
Branches Tags
nathanmcrae:master

No commits in common. "ab72d875bf233e2e42e12dc428274c33e03cd67d" and "4dd3a4878ebae90cd1a5d1a8725388f859c80cf9" have entirely different histories.
ab72d875bf ... 4dd3a4878e

9 changed files with 195 additions and 394 deletions
Show Stats Expand all files Collapse all files

333
SaneTsv/SaneTsv.cs
View File

@ -1,333 +0,0 @@
using System.Text;
namespace NathanMcRae;
/// <summary>
/// Sane Tab-Separated Values
/// </summary>
public class SaneTsv
{
public enum ColumnType
{
STRING,
BOOLEAN,
FLOAT32,
FLOAT64,
UINT32,
UINT64,
INT32,
INT64,
BINARY,
}
// TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; }
public ColumnType[] ColumnTypes { get; protected set; }
public Dictionary<string, List<object>> Columns { get; protected set; }
public List<SaneTsvRecord> Records { get; protected set; }
public static SaneTsv Parse(byte[] inputBuffer)
{
var parsed = new SaneTsv();
parsed.Columns = new Dictionary<string, List<object>>();
parsed.ColumnNames = new string[] { };
parsed.ColumnTypes = new ColumnType[] { };
parsed.Records = new List<SaneTsvRecord>();
var fieldBytes = new List<byte>();
var fields = new List<byte[]>();
int numFields = -1;
for (int i = 0; i < inputBuffer.Count(); i++)
{
if (inputBuffer[i] == '\\')
{
if (i + 1 == inputBuffer.Count())
{
throw new Exception($"Found '\\' at end of input");
}
if (inputBuffer[i + 1] == 'n')
{
fieldBytes.Add((byte)'\n');
i++;
}
else if (inputBuffer[i + 1] == '\\')
{
fieldBytes.Add((byte)'\\');
i++;
}
else if (inputBuffer[i + 1] == 't')
{
fieldBytes.Add((byte)'\t');
i++;
}
else
{
throw new Exception($"Expected 'n', 't', or '\\' after '\\' at {i}");
}
}
else if (inputBuffer[i] == '\t')
{
// end of field
fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
}
else if (inputBuffer[i] == '\n')
{
fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
if (numFields < 0)
{
// This is the header
numFields = fields.Count;
parsed.ColumnNames = new string[numFields];
parsed.ColumnTypes = new ColumnType[numFields];
int numTypesBlank = 0;
for (int j = 0; j < fields.Count; j++)
{
string columnString;
try
{
columnString = Encoding.UTF8.GetString(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Header {fields.Count} is not valid UTF-8", e);
}
string columnTypeString;
string columnName;
if (columnString.Contains(":")) {
columnTypeString = columnString.Split(":").Last();
columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
}
else
{
columnTypeString = "";
columnName = columnString;
}
ColumnType type;
switch (columnTypeString)
{
case "":
numTypesBlank++;
type = ColumnType.STRING;
break;
case "string":
type = ColumnType.STRING;
break;
case "boolean":
type = ColumnType.BOOLEAN;
break;
case "float32":
type = ColumnType.FLOAT32;
break;
case "float64":
type = ColumnType.FLOAT64;
break;
case "uint32":
type = ColumnType.UINT32;
break;
case "uint64":
type = ColumnType.UINT64;
break;
case "int32":
type = ColumnType.INT32;
break;
case "int64":
type = ColumnType.INT64;
break;
case "binary":
type = ColumnType.BINARY;
break;
default:
throw new Exception($"Invalid type '{columnTypeString}' for column {j}");
}
try
{
parsed.Columns.Add(columnName, new List<object>());
}
catch (Exception e)
{
throw new Exception($"Column name {columnName} is not unique", e);
}
parsed.ColumnNames[j] = columnName;
parsed.ColumnTypes[j] = type;
}
if (numTypesBlank != 0 && numTypesBlank != fields.Count)
{
throw new Exception("Types must be provided for all columns or none. Use 'string' for columns missing types.");
}
fields.Clear();
}
else if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
}
else
{
AddRecord(parsed, fields);
}
}
else
{
fieldBytes.Add(inputBuffer[i]);
}
}
fields.Add(fieldBytes.ToArray());
if (numFields == 0)
{
throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
}
if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
}
else
{
AddRecord(parsed, fields);
}
return parsed;
}
protected static void AddRecord(SaneTsv parsed, List<byte[]> fields)
{
var parsedFields = new object[fields.Count];
for (int j = 0; j < fields.Count; j++)
{
// All other types require the content to be UTF-8. Binary fields can ignore that.
if (parsed.ColumnTypes[j] == ColumnType.BINARY)
{
parsedFields[j] = fields[j];
parsed.Columns[parsed.ColumnNames[j]].Add(fields[j]);
continue;
}
string fieldString;
try
{
fieldString = Encoding.UTF8.GetString(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UTF-8", e);
}
switch (parsed.ColumnTypes[j])
{
case ColumnType.STRING:
parsedFields[j] = fieldString;
parsed.Columns[parsed.ColumnNames[j]].Add(fieldString);
break;
case ColumnType.BOOLEAN:
bool parsedBool;
if (fieldString == "TRUE")
{
parsedBool = true;
}
else if (fieldString == "FALSE")
{
parsedBool = false;
}
else
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
}
parsedFields[j] = parsedBool;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedBool);
break;
case ColumnType.FLOAT32:
if (!float.TryParse(fieldString, out float parsedFloat))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid single-precision float");
}
parsedFields[j] = parsedFloat;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedFloat);
break;
case ColumnType.FLOAT64:
if (!double.TryParse(fieldString, out double parsedDouble))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid double-precision float");
}
parsedFields[j] = parsedDouble;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedDouble);
break;
case ColumnType.UINT32:
if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UInt32");
}
parsedFields[j] = parsedUInt32;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedUInt32);
break;
case ColumnType.UINT64:
if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UInt64");
}
parsedFields[j] = parsedUInt64;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedUInt64);
break;
case ColumnType.INT32:
if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid Int32");
}
parsedFields[j] = parsedInt32;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedInt32);
break;
case ColumnType.INT64:
if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid Int64");
}
parsedFields[j] = parsedInt64;
parsed.Columns[parsed.ColumnNames[j]].Add(parsedInt64);
break;
case ColumnType.BINARY:
throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}");
default:
throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}");
}
}
parsed.Records.Add(new SaneTsvRecord(parsed, parsedFields));
fields.Clear();
}
public SaneTsvRecord this[int i] => Records[i];
public class SaneTsvRecord
{
public SaneTsv Parent { get; }
public object[] Fields { get; }
public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)];
public SaneTsvRecord(SaneTsv parent, object[] fields)
{
Parent = parent;
Fields = fields;
}
}
}

18
SaneTsv/SaneTsv.csproj
View File

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>disable</Nullable>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
<RootNamespace>NathanMcRae</RootNamespace>
</PropertyGroup>
<ItemGroup>
<Compile Remove="SaneTsvTest\**" />
<EmbeddedResource Remove="SaneTsvTest\**" />
<None Remove="SaneTsvTest\**" />
</ItemGroup>
</Project>

38
SaneTsv/SaneTsvTest/Program.cs
View File

@ -1,38 +0,0 @@
using NathanMcRae;
using System.Text;
{
string testName = "Bool test";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.Parse(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0]["column1:type"] is bool result && result)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
{
string testName = "Bad bool test";
try
{
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.Parse(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception)
{
Console.WriteLine($"Passed {testName}");
}
Console.WriteLine("Done with tests");
}

0
SaneTsv/.editorconfig → Stsv/.editorconfig
View File

174
Stsv/Stsv.cs Normal file
View File

@ -0,0 +1,174 @@
using System.Text;
namespace NathanMcRae;
/// <summary>
/// Sane Tab-Separated Values
/// </summary>
public class Stsv
{
// TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; }
public Dictionary<string, List<string>> Columns { get; protected set; }
public List<StsvRecord> Records { get; protected set; }
public static Stsv Parse(byte[] inputBuffer)
{
var parsed = new Stsv();
parsed.Columns = new Dictionary<string, List<string>>();
parsed.ColumnNames = new string[] { };
parsed.Records = new List<StsvRecord>();
var fieldBytes = new List<byte>();
var fields = new List<string>();
int numFields = -1;
for (int i = 0; i < inputBuffer.Count(); i++)
{
if (inputBuffer[i] == '\\')
{
if (i + 1 == inputBuffer.Count())
{
throw new Exception($"Found '\\' at end of input");
}
if (inputBuffer[i + 1] == 'n')
{
fieldBytes.Add((byte)'\n');
i++;
}
else if (inputBuffer[i + 1] == '\\')
{
fieldBytes.Add((byte)'\\');
i++;
}
else if (inputBuffer[i + 1] == 't')
{
fieldBytes.Add((byte)'\t');
i++;
}
else
{
throw new Exception($"Expected 'n', 't', or '\\' after '\\' at {i}");
}
}
else if (inputBuffer[i] == '\t')
{
// end of field
try
{
fields.Add(Encoding.UTF8.GetString(fieldBytes.ToArray()));
}
catch (Exception e)
{
throw new Exception($"Field {fields.Count} on line {parsed.Records.Count + 1} is not valid UTF-8", e);
}
fieldBytes.Clear();
}
else if (inputBuffer[i] == '\n')
{
try
{
fields.Add(Encoding.UTF8.GetString(fieldBytes.ToArray()));
}
catch (Exception e)
{
throw new Exception($"Field {fields.Count} on line {parsed.Records.Count + 1} is not valid UTF-8", e);
}
fieldBytes.Clear();
if (numFields < 0)
{
// This is the header
numFields = fields.Count;
parsed.ColumnNames = new string[numFields];
for (int j = 0; j < fields.Count; j++)
{
string columnName = fields[j];
try
{
parsed.Columns.Add(columnName, new List<string>());
}
catch (Exception e)
{
throw new Exception($"Column name {columnName} is not unique", e);
}
parsed.ColumnNames[j] = columnName;
}
fields.Clear();
}
else if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
}
else
{
for (int j = 0; j < fields.Count; j++)
{
parsed.Columns[parsed.ColumnNames[j]].Add(fields[j]);
}
parsed.Records.Add(new StsvRecord(parsed, fields.ToArray()));
fields.Clear();
}
}
else
{
fieldBytes.Add(inputBuffer[i]);
}
}
try
{
fields.Add(Encoding.UTF8.GetString(fieldBytes.ToArray()));
}
catch (Exception e)
{
throw new Exception($"Field {fields.Count} on line {parsed.Records.Count + 1} is not valid UTF-8", e);
}
if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
}
else
{
for (int j = 0; j < fields.Count; j++)
{
try
{
parsed.Columns[parsed.ColumnNames[j]].Add(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Field {j} on line {parsed.Records.Count + 1} is not valid UTF-8", e);
}
}
parsed.Records.Add(new StsvRecord(parsed, fields.ToArray()));
fields.Clear();
}
return parsed;
}
public StsvRecord this[int i] => Records[i];
public class StsvRecord
{
public Stsv Parent { get; }
public string[] Fields { get; }
public string this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)];
public StsvRecord(Stsv parent, string[] fields)
{
Parent = parent;
Fields = fields;
}
}
}

9
Stsv/Stsv.csproj Normal file
View File

@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>disable</Nullable>
</PropertyGroup>
</Project>

4
SaneTsv/SaneTsv.sln → Stsv/Stsv.sln
View File

@ -3,9 +3,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.7.34024.191
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SaneTsv", "SaneTsv.csproj", "{DBC5CE44-361C-4387-B1E2-409C1CAE2B4C}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Stsv", "Stsv.csproj", "{DBC5CE44-361C-4387-B1E2-409C1CAE2B4C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SaneTsvTest", "SaneTsvTest\SaneTsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StsvTest", "..\StsvTest\StsvTest.csproj", "{43B1B09C-19BD-4B45-B41B-7C00DB3F7E9C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

8
StsvTest/Program.cs Normal file
View File

@ -0,0 +1,8 @@
using NathanMcRae;
using System.Text;
string testString1 = "column1\tcolumn2\tcolumnthree\\nyep\nvalue1\tvalue\\\\twoo\tvaluetrhee\nthis\\nis\\na\\nvalue\tnother\tno\\ther";
Stsv parsed = Stsv.Parse(Encoding.UTF8.GetBytes(testString1));
// See https://aka.ms/new-console-template for more information
Console.WriteLine("Hello, World!");

5
SaneTsv/SaneTsvTest/SaneTsvTest.csproj → StsvTest/StsvTest.csproj
View File

@ -1,15 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net6.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<RootNamespace>NathanMcRae</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\SaneTsv.csproj" />
<ProjectReference Include="..\Stsv\Stsv.csproj" />
</ItemGroup>
</Project>