Compare commits

..

3 Commits

Author SHA1 Message Date
Nathan McRae
7368ac816b Get commented typing working 2024-02-22 22:33:08 -08:00
Nathan McRae
0a45f541a4 Implement static typing column-spec
By creating a class inheriting from SaneTsvRecord with properties
decorated with SaneTsvColumn attributes
2024-02-20 14:30:01 -08:00
Nathan McRae
b593fb9613 Only keep column names in SaneTsvRecord
We don't need to reference the parent
2024-02-19 21:03:48 -08:00
2 changed files with 548 additions and 216 deletions

View File

@ -1,7 +1,19 @@
using System.Text; using System.Reflection;
using System.Text;
namespace NathanMcRae; namespace NathanMcRae;
public class Tsv<T> where T : SaneTsv.TsvRecord
{
public virtual List<T> Records { get; set; }
}
public class CommentedTsv<T>: Tsv<T> where T : SaneTsv.TsvRecord
{
public override List<T> Records { get; set; }
public string FileComment { get; set; } = null;
}
/// <summary> /// <summary>
/// Sane Tab-Separated Values /// Sane Tab-Separated Values
/// </summary> /// </summary>
@ -34,8 +46,6 @@ public class SaneTsv
// TODO: We need to be able to update all these in tandem somehow // TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; } public string[] ColumnNames { get; protected set; }
public Type[] ColumnTypes { get; protected set; } public Type[] ColumnTypes { get; protected set; }
public List<SaneTsvRecord> Records { get; protected set; }
public string FileComment { get; protected set; } = null;
protected static bool? _littleEndian = null; protected static bool? _littleEndian = null;
public static bool LittleEndian public static bool LittleEndian
{ {
@ -49,28 +59,55 @@ public class SaneTsv
} }
} }
public static SaneTsv ParseSimpleTsv(byte[] inputBuffer) public static Tsv<T> ParseSimpleTsv<T>(byte[] inputBuffer) where T : TsvRecord, new()
{ {
return Parse(inputBuffer, FormatType.SIMPLE_TSV); return Parse<T>(inputBuffer, FormatType.SIMPLE_TSV);
} }
public static SaneTsv ParseTypedTsv(byte[] inputBuffer) public static Tsv<T> ParseTypedTsv<T>(byte[] inputBuffer) where T : TsvRecord, new()
{ {
return Parse(inputBuffer, FormatType.TYPED_TSV); return Parse<T>(inputBuffer, FormatType.TYPED_TSV);
} }
public static SaneTsv ParseCommentedTsv(byte[] inputBuffer) public static CommentedTsv<T> ParseCommentedTsv<T>(byte[] inputBuffer) where T : CommentedTsvRecord, new()
{ {
return Parse(inputBuffer, FormatType.COMMENTED_TSV); // TODO: add the file comment?
return (CommentedTsv<T>)Parse<T>(inputBuffer, FormatType.COMMENTED_TSV);
} }
// TODO: Have parsing errors include line / column # // TODO: Have parsing errors include line / column #
protected static SaneTsv Parse(byte[] inputBuffer, FormatType format) protected static Tsv<T> Parse<T>(byte[] inputBuffer, FormatType format) where T : TsvRecord, new()
{ {
var parsed = new SaneTsv(); Tsv<T> parsed;
parsed.ColumnNames = new string[] { }; if (format == FormatType.COMMENTED_TSV)
parsed.ColumnTypes = new Type[] { }; {
parsed.Records = new List<SaneTsvRecord>(); parsed = new CommentedTsv<T>();
}
else
{
parsed = new Tsv<T>();
}
parsed.Records = new List<T>();
var headerTypes = new List<Type>();
var headerNames = new List<string>();
var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
foreach (PropertyInfo property in typeof(T).GetProperties())
{
TypedTsvColumnAttribute attribute = (TypedTsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TypedTsvColumnAttribute));
if (attribute == null)
{
continue;
}
headerNames.Add(attribute.ColumnName ?? property.Name);
headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible
columnCount++;
}
var fieldBytes = new List<byte>(); var fieldBytes = new List<byte>();
var fields = new List<byte[]>(); var fields = new List<byte[]>();
@ -129,9 +166,6 @@ public class SaneTsv
numFields = fields.Count; numFields = fields.Count;
parsed.ColumnNames = new string[numFields];
parsed.ColumnTypes = new Type[numFields];
int numTypesBlank = 0; int numTypesBlank = 0;
for (int j = 0; j < fields.Count; j++) for (int j = 0; j < fields.Count; j++)
@ -213,16 +247,32 @@ public class SaneTsv
} }
// TODO: Check column name uniqueness // TODO: Check column name uniqueness
// TODO: Allow lax parsing (only worry about parsing columns that are given in the specifying type
parsed.ColumnNames[j] = columnName; if (headerNames[j] != columnName)
parsed.ColumnTypes[j] = type; {
throw new Exception($"Column {j} has name {columnName}, but expected {headerNames[j]}");
}
if (headerTypes[j] != type)
{
throw new Exception($"Column {j} has type {type}, but expected {headerTypes[j]}");
}
} }
if (currentComment.Length > 0) if (currentComment.Length > 0)
{ {
parsed.FileComment = currentComment.ToString(); if (parsed is CommentedTsv<T> commentedParsed)
{
commentedParsed.FileComment = currentComment.ToString();
currentComment.Clear(); currentComment.Clear();
} }
else
{
throw new Exception("Found a file comment, but parser wasn't expecting a comment");
}
}
fields.Clear(); fields.Clear();
} }
@ -238,7 +288,7 @@ public class SaneTsv
comment = currentComment.ToString(); comment = currentComment.ToString();
currentComment.Clear(); currentComment.Clear();
} }
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line)); parsed.Records.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, line));
fields.Clear(); fields.Clear();
} }
@ -298,28 +348,46 @@ public class SaneTsv
comment = currentComment.ToString(); comment = currentComment.ToString();
currentComment.Clear(); currentComment.Clear();
} }
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line)); parsed.Records.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, line));
fields.Clear(); fields.Clear();
} }
return parsed; return parsed;
} }
/// <summary> protected static T ParseCurrentCommentedRecord<T> (Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : CommentedTsvRecord, new()
/// Note: this modifies 'parsed'
/// </summary>
protected static object[] ParseCurrentRecord(SaneTsv parsed, List<byte[]> fields, int line)
{ {
var parsedFields = new object[fields.Count]; return (T)ParseCurrentRecord<T>(columnTypes, properties, fields, comment, line);
}
protected static T ParseCurrentRecord<T>(Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : TsvRecord, new()
{
T record = new T();
if (record is CommentedTsvRecord commentedRecord)
{
commentedRecord.Comment = comment;
}
else if (comment != null)
{
throw new Exception($"Found comment for line {line}, but format does not support comments");
}
record.Line = line;
for (int j = 0; j < fields.Count; j++) for (int j = 0; j < fields.Count; j++)
{ {
// All other types require the content to be UTF-8. Binary fields can ignore that. // All other types require the content to be UTF-8. Binary fields can ignore that.
if (parsed.ColumnTypes[j] == typeof(BinaryType)) if (columnTypes[j] == typeof(BinaryType))
{ {
parsedFields[j] = fields[j]; // TODO: Use faster method for property setting
// e.g. https://blog.marcgravell.com/2012/01/playing-with-your-member.html
// or https://stackoverflow.com/questions/1027980/improving-performance-reflection-what-alternatives-should-i-consider
// or https://stackoverflow.com/questions/12767091/why-are-propertyinfo-setvalue-and-getvalue-so-slow
properties[j].SetValue(record, fields[j]);
continue; continue;
} }
else if (parsed.ColumnTypes[j] == typeof(Float32LEType)) else if (columnTypes[j] == typeof(Float32LEType))
{ {
byte[] floatBytes; byte[] floatBytes;
if (!LittleEndian) if (!LittleEndian)
@ -334,11 +402,11 @@ public class SaneTsv
{ {
floatBytes = fields[j]; floatBytes = fields[j];
} }
parsedFields[j] = BitConverter.ToSingle(floatBytes, 0); properties[j].SetValue(record, BitConverter.ToSingle(floatBytes, 0));
continue; continue;
} }
else if (parsed.ColumnTypes[j] == typeof(Float64LEType)) else if (columnTypes[j] == typeof(Float64LEType))
{ {
byte[] floatBytes; byte[] floatBytes;
if (!LittleEndian) if (!LittleEndian)
@ -353,7 +421,7 @@ public class SaneTsv
{ {
floatBytes = fields[j]; floatBytes = fields[j];
} }
parsedFields[j] = BitConverter.ToDouble(floatBytes, 0); properties[j].SetValue(record, BitConverter.ToDouble(floatBytes, 0));
continue; continue;
} }
@ -370,11 +438,11 @@ public class SaneTsv
// TODO: Add checking for numeric types format // TODO: Add checking for numeric types format
if (parsed.ColumnTypes[j] == typeof(StringType)) if (columnTypes[j] == typeof(StringType))
{ {
parsedFields[j] = fieldString; properties[j].SetValue(record, fieldString);
} }
else if (parsed.ColumnTypes[j] == typeof(BooleanType)) else if (columnTypes[j] == typeof(BooleanType))
{ {
bool parsedBool; bool parsedBool;
if (fieldString == "TRUE") if (fieldString == "TRUE")
@ -390,9 +458,9 @@ public class SaneTsv
throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly"); throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
} }
parsedFields[j] = parsedBool; properties[j].SetValue(record, parsedBool);
} }
else if (parsed.ColumnTypes[j] == typeof(Float32Type)) else if (columnTypes[j] == typeof(Float32Type))
{ {
float parsedFloat; float parsedFloat;
if (!float.TryParse(fieldString, out parsedFloat)) if (!float.TryParse(fieldString, out parsedFloat))
@ -411,9 +479,9 @@ public class SaneTsv
} }
} }
parsedFields[j] = parsedFloat; properties[j].SetValue(record, parsedFloat);
} }
else if (parsed.ColumnTypes[j] == typeof(Float64Type)) else if (columnTypes[j] == typeof(Float64Type))
{ {
double parsedDouble; double parsedDouble;
if (!double.TryParse(fieldString, out parsedDouble)) if (!double.TryParse(fieldString, out parsedDouble))
@ -432,51 +500,51 @@ public class SaneTsv
} }
} }
parsedFields[j] = parsedDouble; properties[j].SetValue(record, parsedDouble);
} }
else if (parsed.ColumnTypes[j] == typeof(UInt32Type)) else if (columnTypes[j] == typeof(UInt32Type))
{ {
if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32)) if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
{ {
throw new Exception($"Field {j} on line {line} is not valid UInt32"); throw new Exception($"Field {j} on line {line} is not valid UInt32");
} }
parsedFields[j] = parsedUInt32; properties[j].SetValue(record, parsedUInt32);
} }
else if (parsed.ColumnTypes[j] == typeof(UInt64Type)) else if (columnTypes[j] == typeof(UInt64Type))
{ {
if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64)) if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
{ {
throw new Exception($"Field {j} on line {line} is not valid UInt64"); throw new Exception($"Field {j} on line {line} is not valid UInt64");
} }
parsedFields[j] = parsedUInt64; properties[j].SetValue(record, parsedUInt64);
} }
else if (parsed.ColumnTypes[j] == typeof(Int32Type)) else if (columnTypes[j] == typeof(Int32Type))
{ {
if (!Int32.TryParse(fieldString, out Int32 parsedInt32)) if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
{ {
throw new Exception($"Field {j} on line {line} is not valid Int32"); throw new Exception($"Field {j} on line {line} is not valid Int32");
} }
parsedFields[j] = parsedInt32; properties[j].SetValue(record, parsedInt32);
} }
else if (parsed.ColumnTypes[j] == typeof(Int64Type)) else if (columnTypes[j] == typeof(Int64Type))
{ {
if (!Int64.TryParse(fieldString, out Int64 parsedInt64)) if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
{ {
throw new Exception($"Field {j} on line {line} is not valid Int64"); throw new Exception($"Field {j} on line {line} is not valid Int64");
} }
parsedFields[j] = parsedInt64; properties[j].SetValue(record, parsedInt64);
} }
else else
{ {
throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}"); throw new Exception($"Unexpected type {columnTypes[j]}");
} }
} }
return parsedFields; return record;
} }
public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data) public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data)
@ -672,13 +740,49 @@ public class SaneTsv
} }
} }
public static byte[] SerializeTypedTsv(IList<Type> headerTypes, IList<string> headerNames, IList<IList<object>> data) public static byte[] SerializeSimpleTsv<T>(IList<T> data) where T : TsvRecord
{
return SerializeTsv<T>(data, FormatType.TYPED_TSV);
}
public static byte[] SerializeTypedTsv<T>(IList<T> data) where T : TsvRecord
{
return SerializeTsv<T>(data, FormatType.TYPED_TSV);
}
public static byte[] SerializeCommentedTsv<T>(IList<T> data, string fileComment) where T : CommentedTsvRecord
{
return SerializeTsv<T>(data, FormatType.COMMENTED_TSV);
}
protected static byte[] SerializeTsv<T>(IList<T> data, FormatType tsvFormat) where T : TsvRecord
{ {
var bytes = new List<byte>(); var bytes = new List<byte>();
if (headerNames.Count != headerTypes.Count) var headerTypes = new List<Type>();
var headerNames = new List<string>();
var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
foreach (PropertyInfo property in typeof(T).GetProperties())
{ {
throw new ArgumentException($"headerTypes length ({headerTypes.Count}) does not match headerNames length ({headerNames.Count})"); TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
if (attribute == null)
{
continue;
}
string headerName = attribute.ColumnName ?? property.Name;
headerNames.Add(headerName);
Type headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType);
if (tsvFormat == FormatType.SIMPLE_TSV && headerType != typeof(StringType))
{
throw new Exception($"Serializing Simple TSV requires all columns be of type string, but column '{headerName}' has type '{headerType}'");
}
headerTypes.Add(headerType);
headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible
columnCount++;
} }
// Serialize header // Serialize header
@ -745,8 +849,10 @@ public class SaneTsv
// Serialize data // Serialize data
for (int i = 0; i < data.Count; i++) for (int i = 0; i < data.Count; i++)
{ {
for (int j = 0; j < data[i].Count; j++) for (int j = 0; j < columnCount; j++)
{ {
object datum = headerPropertyInfos[j].GetValue(data[i]);
try try
{ {
byte[] fieldEncoded = null; byte[] fieldEncoded = null;
@ -755,16 +861,16 @@ public class SaneTsv
if (headerTypes[j] == typeof(StringType)) if (headerTypes[j] == typeof(StringType))
{ {
fieldEncoded = Encoding.UTF8.GetBytes((string)data[i][j]); fieldEncoded = Encoding.UTF8.GetBytes((string)datum);
} }
else if (headerTypes[j] == typeof(BooleanType)) else if (headerTypes[j] == typeof(BooleanType))
{ {
bytes.AddRange((bool)data[i][j] ? TrueEncoded : FalseEncoded); bytes.AddRange((bool)datum ? TrueEncoded : FalseEncoded);
skipEscaping = true; skipEscaping = true;
} }
else if (headerTypes[j] == typeof(Float32Type)) else if (headerTypes[j] == typeof(Float32Type))
{ {
if (data[i][j] is float f) if (datum is float f)
{ {
if (float.IsNegativeInfinity(f)) if (float.IsNegativeInfinity(f))
{ {
@ -777,7 +883,7 @@ public class SaneTsv
else else
{ {
// See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r // See https://learn.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#round-trip-format-specifier-r
bytes.AddRange(Encoding.UTF8.GetBytes(((float)data[i][j]).ToString("G9"))); bytes.AddRange(Encoding.UTF8.GetBytes(((float)datum).ToString("G9")));
} }
} }
else else
@ -790,11 +896,11 @@ public class SaneTsv
{ {
if (LittleEndian) if (LittleEndian)
{ {
fieldEncoded = BitConverter.GetBytes((float)data[i][j]); fieldEncoded = BitConverter.GetBytes((float)datum);
} }
else else
{ {
byte[] floatBytes = BitConverter.GetBytes((float)data[i][j]); byte[] floatBytes = BitConverter.GetBytes((float)datum);
fieldEncoded = new byte[sizeof(float)]; fieldEncoded = new byte[sizeof(float)];
for (int k = 0; k < sizeof(float); k++) for (int k = 0; k < sizeof(float); k++)
{ {
@ -804,7 +910,7 @@ public class SaneTsv
} }
else if (headerTypes[j] == typeof(Float64Type)) else if (headerTypes[j] == typeof(Float64Type))
{ {
if (data[i][j] is double d) if (datum is double d)
{ {
if (double.IsNegativeInfinity(d)) if (double.IsNegativeInfinity(d))
{ {
@ -830,11 +936,11 @@ public class SaneTsv
{ {
if (LittleEndian) if (LittleEndian)
{ {
fieldEncoded = BitConverter.GetBytes((double)data[i][j]); fieldEncoded = BitConverter.GetBytes((double)datum);
} }
else else
{ {
byte[] doubleBytes = BitConverter.GetBytes((double)data[i][j]); byte[] doubleBytes = BitConverter.GetBytes((double)datum);
fieldEncoded = new byte[sizeof(double)]; fieldEncoded = new byte[sizeof(double)];
for (int k = 0; k < sizeof(double); k++) for (int k = 0; k < sizeof(double); k++)
{ {
@ -844,27 +950,27 @@ public class SaneTsv
} }
else if (headerTypes[j] == typeof(UInt32Type)) else if (headerTypes[j] == typeof(UInt32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)(int)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (headerTypes[j] == typeof(UInt64Type)) else if (headerTypes[j] == typeof(UInt64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)(int)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (headerTypes[j] == typeof(Int32Type)) else if (headerTypes[j] == typeof(Int32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)(int)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (headerTypes[j] == typeof(Int64Type)) else if (headerTypes[j] == typeof(Int64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)(int)data[i][j]).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (headerTypes[j] == typeof(BinaryType)) else if (headerTypes[j] == typeof(BinaryType))
{ {
fieldEncoded = (byte[])data[i][j]; fieldEncoded = (byte[])datum;
} }
else else
{ {
@ -902,7 +1008,7 @@ public class SaneTsv
} }
} }
if (j < data[i].Count - 1) if (j < columnCount - 1)
{ {
bytes.Add((byte)'\t'); bytes.Add((byte)'\t');
} }
@ -921,25 +1027,97 @@ public class SaneTsv
return bytes.ToArray(); return bytes.ToArray();
} }
public SaneTsvRecord this[int i] => Records[i]; public class SimpleTsvRecord
public class SaneTsvRecord
{ {
public SaneTsv Parent { get; } public string[] ColumnNames { get; }
public string Comment { get; } public string Comment { get; }
public object[] Fields { get; } public string[] Fields { get; }
public int Line { get; } public int? Line { get; }
public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)]; public string this[string columnName] => Fields[Array.IndexOf(ColumnNames, columnName)];
public string this[int columnIndex] => Fields[columnIndex];
public object this[int columnIndex] => Fields[columnIndex]; public SimpleTsvRecord(string[] columnNames, string[] fields, string comment, int line)
public SaneTsvRecord(SaneTsv parent, object[] fields, string comment, int line)
{ {
Parent = parent; ColumnNames = columnNames;
Fields = fields; Fields = fields;
Comment = comment; Comment = comment;
Line = line; Line = line;
} }
} }
public class TsvRecord
{
public int? Line { get; set; }
public TsvRecord(int? line)
{
Line = line;
}
public TsvRecord() { }
}
public class CommentedTsvRecord : TsvRecord
{
public string Comment { get; set; }
public CommentedTsvRecord(string comment, int? line)
{
Comment = comment;
Line = line;
}
public CommentedTsvRecord() { }
}
public class TestRecord : CommentedTsvRecord
{
[TypedTsvColumn("my-column")]
public string MyColumn { get; set; }
}
// TODO: Add column ordering
public class TsvColumnAttribute : Attribute
{
public string ColumnName { get; }
public virtual Type ColumnType { get; }
public TsvColumnAttribute()
{
ColumnType = typeof(StringType);
}
public TsvColumnAttribute(string columnName)
{
ColumnType = typeof(StringType);
ColumnName = columnName;
}
}
// TODO: Add column ordering
public class TypedTsvColumnAttribute : TsvColumnAttribute
{
public override Type ColumnType { get; }
public TypedTsvColumnAttribute() { }
public TypedTsvColumnAttribute(string columnName) : base(columnName) { }
public TypedTsvColumnAttribute(string columnName, Type columnType) : base(columnName)
{
if (columnType.BaseType != typeof(ColumnType))
{
throw new Exception("Column type must inherit from SaneTsv.ColumnType");
}
ColumnType = columnType;
}
public TypedTsvColumnAttribute(Type columnType)
{
if (columnType.BaseType != typeof(ColumnType))
{
throw new Exception("Column type must inherit from SaneTsv.ColumnType");
}
ColumnType = columnType;
}
}
} }

View File

@ -1,14 +1,116 @@
using NathanMcRae; using NathanMcRae;
using System.Text; using System.Text;
internal class Program
{ {
public class TestRecord : SaneTsv.TsvRecord
{
[SaneTsv.TypedTsvColumn("string-test")]
public string StringTest { get; set; }
[SaneTsv.TypedTsvColumn("bool-test")]
public bool BoolTest { get; set; }
[SaneTsv.TypedTsvColumn("float32-test")]
public float Float32Test { get; set; }
[SaneTsv.TypedTsvColumn("float32-le-test", typeof(SaneTsv.Float32LEType))]
public float Float32LETest { get; set; }
[SaneTsv.TypedTsvColumn("float64-test")]
public double Float64Test { get; set; }
[SaneTsv.TypedTsvColumn("float64-le-test", typeof(SaneTsv.Float64LEType))]
public double Float64LETest { get; set; }
[SaneTsv.TypedTsvColumn("uint32-test")]
public UInt32 UInt32Test { get; set; }
[SaneTsv.TypedTsvColumn("uint64-test")]
public UInt64 UInt64Test { get; set; }
[SaneTsv.TypedTsvColumn("int32-test")]
public Int32 Int32Test { get; set; }
[SaneTsv.TypedTsvColumn("int64-test")]
public Int64 Int64Test { get; set; }
[SaneTsv.TypedTsvColumn("binary-test")]
public byte[] BinaryTest { get; set; }
public TestRecord(string stringTest, bool boolTest, float float32Test, float float32LETest, double float64Test, double float64LETest, UInt32 uInt32Test, UInt64 uInt64Test, Int32 int32Test, Int64 int64Test, byte[] binaryTest)
{
StringTest = stringTest;
BoolTest = boolTest;
Float32Test = float32Test;
Float32LETest = float32LETest;
Float64Test = float64Test;
Float64LETest = float64LETest;
UInt32Test = uInt32Test;
UInt64Test = uInt64Test;
Int32Test = int32Test;
Int64Test = int64Test;
BinaryTest = binaryTest;
}
public TestRecord() { }
}
public class BoolTestRecord : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("column1:ty#pe")]
public bool Column1 { get; set; }
[SaneTsv.TypedTsvColumn]
public byte[] column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class BoolTestRecord2 : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("column1:type")]
public bool Column1 { get; set; }
[SaneTsv.TypedTsvColumn]
public byte[] column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("column1")]
public bool Column1 { get; set; }
[SaneTsv.TypedTsvColumn]
public byte[] column2 { get; set; }
[SaneTsv.TypedTsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class FloatTestRecord : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TypedTsvColumn("somefloat")]
public double SomeFloat { get; set; }
[SaneTsv.TypedTsvColumn("binfloat", typeof(SaneTsv.Float64LEType))]
public double BinFloat { get; set; }
}
private static void Main(string[] args)
{
{
string testName = "Bool test"; string testName = "Bool test";
string testString1 = "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + string testString1 = "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" + "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther"; "\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseTypedTsv(Encoding.UTF8.GetBytes(testString1)); Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0]["column1:ty#pe"] is bool result && result) if (parsed.Records[0].Column1)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -16,9 +118,9 @@ using System.Text;
{ {
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName}");
} }
} }
{ {
string testName = "Bad bool test"; string testName = "Bad bool test";
try try
{ {
@ -26,7 +128,7 @@ using System.Text;
"\nTUE\tvalue\\\\t\0woo\tvaluetrhee" + "\nTUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther"; "\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseTypedTsv(Encoding.UTF8.GetBytes(testString1)); Tsv<BoolTestRecord> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName}");
} }
catch (Exception) catch (Exception)
@ -34,9 +136,9 @@ using System.Text;
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
} }
{ {
string testName = "Comment test"; string testName = "Comment test";
string testString1 = "#This is a file comment\n" + string testString1 = "#This is a file comment\n" +
"#One more file comment line\n" + "#One more file comment line\n" +
@ -46,37 +148,37 @@ using System.Text;
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" + "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther"; "\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseCommentedTsv(Encoding.UTF8.GetBytes(testString1)); CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
}
{
string testName = "Serde test";
string testString1 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeSimpleTsv(parsed.ColumnNames, parsed.Records.Select(r => r.Fields.Select(f => f.ToString()).ToArray()).ToArray()));
if (testString1 == serialized)
{
Console.WriteLine($"Passed {testName}");
} }
else
{
Console.WriteLine($"Failed {testName}");
}
}
{ //{
// string testName = "Serde test";
// string testString1 = "column1\tcolumn2\tcolumnthree\\nyep" +
// "\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
// "\nFALSE\tnother\tno\\ther";
// Tsv<SerdeTestRecord> parsed = SaneTsv.ParseSimpleTsv<SerdeTestRecord>(Encoding.UTF8.GetBytes(testString1));
// string serialized = Encoding.UTF8.GetString(SaneTsv.SerializeSimpleTsv(parsed.ColumnNames, parsed.Records.Select(r => r.Fields.Select(f => f.ToString()).ToArray()).ToArray()));
// if (testString1 == serialized)
// {
// Console.WriteLine($"Passed {testName}");
// }
// else
// {
// Console.WriteLine($"Failed {testName}");
// }
//}
{
string testName = "Float binary test"; string testName = "Float binary test";
var bytes = new List<byte>(); var bytes = new List<byte>();
bytes.AddRange(Encoding.UTF8.GetBytes("somefloat:float64\tbinfloat:float64-le" + bytes.AddRange(Encoding.UTF8.GetBytes("somefloat:float64\tbinfloat:float64-le" +
"\n1.5\t")); bytes.AddRange(BitConverter.GetBytes(1.5)); "\n1.5\t")); bytes.AddRange(BitConverter.GetBytes(1.5));
bytes.AddRange(Encoding.UTF8.GetBytes("\n-8.0000005E-14\t")); bytes.AddRange(BitConverter.GetBytes(-8.0000005E-14)); bytes.AddRange(Encoding.UTF8.GetBytes("\n-8.0000005E-14\t")); bytes.AddRange(BitConverter.GetBytes(-8.0000005E-14));
SaneTsv parsed = SaneTsv.ParseTypedTsv(bytes.ToArray()); Tsv<FloatTestRecord> parsed = SaneTsv.ParseTypedTsv<FloatTestRecord>(bytes.ToArray());
if ((double)parsed.Records[0]["binfloat"] == (double)parsed.Records[0]["somefloat"]) if (parsed.Records[0].BinFloat == parsed.Records[0].SomeFloat)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -84,56 +186,25 @@ using System.Text;
{ {
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName}");
} }
} }
Console.WriteLine("Done with tests"); {
{
string testName = "Serde test"; string testName = "Serde test";
string[] headerNames =
TestRecord[] data =
{ {
"string-test", new TestRecord("test", true, 44.5f, 44.5f, -88e-3, -88e-3, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3 }),
"bool-test", new TestRecord("test2", false, 44.5000005f, 44.5000005f, -88e-30, -88e-30, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
"float32-test", new TestRecord("test2", false, float.NaN, float.NaN, double.NaN, double.NaN, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
"float32-le-test", new TestRecord("test2", false, float.NegativeInfinity, float.NegativeInfinity, double.NegativeInfinity, double.NegativeInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
"float64-test", new TestRecord("test2", false, float.PositiveInfinity, float.PositiveInfinity, double.PositiveInfinity, double.PositiveInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 }),
"float64-le-test",
"uint32-test",
"uint64-test",
"int32-test",
"int64-test",
"binary-test",
}; };
Type[] headerTypes = byte[] serialized = SaneTsv.SerializeTypedTsv(data);
{
typeof(SaneTsv.StringType),
typeof(SaneTsv.BooleanType),
typeof(SaneTsv.Float32Type),
typeof(SaneTsv.Float32LEType),
typeof(SaneTsv.Float64Type),
typeof(SaneTsv.Float64LEType),
typeof(SaneTsv.UInt32Type),
typeof(SaneTsv.UInt64Type),
typeof(SaneTsv.Int32Type),
typeof(SaneTsv.Int64Type),
typeof(SaneTsv.BinaryType),
};
object[][] data = Tsv<TestRecord> parsed = SaneTsv.ParseTypedTsv<TestRecord>(serialized);
{
new object[] { "test", true, 44.5f, 44.5f, -88e-3, -88e-3, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3 } },
new object[] { "test2", false, 44.5000005f, 44.5000005f, -88e-30, -88e-30, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.NaN, float.NaN, double.NaN, double.NaN, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.NegativeInfinity, float.NegativeInfinity, double.NegativeInfinity, double.NegativeInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
new object[] { "test2", false, float.PositiveInfinity, float.PositiveInfinity, double.PositiveInfinity, double.PositiveInfinity, 7773, 88888888, -7773, -88888888, new byte[] { 0, 1, 2, 3, 4 } },
};
byte[] serialized = SaneTsv.SerializeTypedTsv(headerTypes, headerNames, data); if ((float)parsed.Records[1].Float32Test == 44.5000005f)
SaneTsv parsed = SaneTsv.ParseTypedTsv(serialized);
if ((float)parsed.Records[1]["float32-test"] == 44.5000005)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -141,5 +212,88 @@ Console.WriteLine("Done with tests");
{ {
Console.WriteLine($"Failed {testName}"); Console.WriteLine($"Failed {testName}");
} }
}
{
string testName = "Trying to parse a not commented record as a Commented TSV test";
// These should not compile:
//byte[] serialized = SaneTsv.SerializeCommentedTsv(data);
// Gives this error: error CS7036: There is no argument given that corresponds to the required parameter 'fileComment' of 'SaneTsv.SerializeCommentedTsv<T>(IList<T>, string)'
//Tsv<TestRecord> parsed = SaneTsv.ParseCommentedTsv<TestRecord>(serialized);
// Gives this error: error CS0311: The type 'Program.TestRecord' cannot be used as type parameter 'T' in the generic type or method 'SaneTsv.ParseCommentedTsv<T>(byte[])'. There is no implicit reference conversion from 'Program.TestRecord' to 'NathanMcRae.SaneTsv.CommentedTsvRecord'.
}
{
string testName = "Try to parsed a Commented TSV as a Simple TSV";
string testString1 = "#This is a file comment\n" +
"#One more file comment line\n" +
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
{
string testName = "Try to parsed a Commented TSV as a Typed TSV";
string testString1 = "#This is a file comment\n" +
"#One more file comment line\n" +
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
{
string testName = "Try to parsed a Typed TSV as a Simple TSV";
string testString1 =
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName}");
}
}
Console.WriteLine("Done with tests");
}
} }
// TODO: Check qNaN, sNaN, +inf, -inf values for float types