nathanmcrae/sane-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nathanmcrae:a80206767e6afa338d6c037a54225534d1bfb67c
Branches Tags
nathanmcrae:master
..
compare: nathanmcrae:0fd092685d9bb2d24437466e44669172a227dc91
Branches Tags
nathanmcrae:master

No commits in common. "a80206767e6afa338d6c037a54225534d1bfb67c" and "0fd092685d9bb2d24437466e44669172a227dc91" have entirely different histories.
a80206767e ... 0fd092685d

2 changed files with 85 additions and 163 deletions
Show Stats Expand all files Collapse all files

224
SaneTsv/SaneTsv.cs
View File

@ -84,7 +84,7 @@ public class SaneTsv
}
parsed.Records = new List<T>();
var columnTypes = new List<ColumnType>();
var columnTypes = new List<Type>();
var columnNames = new List<string>();
var columnPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
@ -190,46 +190,46 @@ public class SaneTsv
columnName = columnString;
}
ColumnType type;
Type type;
switch (columnTypeString)
{
case "":
numTypesBlank++;
type = new StringType();
type = typeof(StringType);
break;
case "string":
type = new StringType();
type = typeof(StringType);
break;
case "boolean":
type = new BooleanType();
type = typeof(BooleanType);
break;
case "float32":
type = new Float32Type();
type = typeof(Float32Type);
break;
case "float32-le":
type = new Float32LEType();
type = typeof(Float32LEType);
break;
case "float64":
type = new Float64Type();
type = typeof(Float64Type);
break;
case "float64-le":
type = new Float64LEType();
type = typeof(Float64LEType);
break;
case "uint32":
type = new UInt32Type();
type = typeof(UInt32Type);
break;
case "uint64":
type = new UInt64Type();
type = typeof(UInt64Type);
break;
case "int32":
type = new Int32Type();
type = typeof(Int32Type);
break;
case "int64":
type = new Int64Type();
type = typeof(Int64Type);
break;
case "binary":
type = new BinaryType();
type = typeof(BinaryType);
break;
default:
throw new Exception($"Invalid type '{columnTypeString}' for column {j}");
@ -242,7 +242,7 @@ public class SaneTsv
throw new Exception($"Column {j} has name {columnName}, but expected {columnNames[j]}");
}
if (columnTypes[j].GetType() != type.GetType())
if (columnTypes[j] != type)
{
throw new Exception($"Column {j} has type {type}, but expected {columnTypes[j]}");
}
@ -348,7 +348,7 @@ public class SaneTsv
// startIndex is in we'd have to go back to the start of the record's comment, and to know
// exactly where that comment started we'd have to go back to the start of the record before that
// (not including that other record's comment).
protected static T[] Parse<T>(byte[] inputBuffer, FormatType format, PropertyInfo[] columnPropertyInfos, ColumnType[] columnTypes, int startIndex, int endIndex) where T : TsvRecord, new()
protected static T[] Parse<T>(byte[] inputBuffer, FormatType format, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, int startIndex, int endIndex) where T : TsvRecord, new()
{
var fieldBytes = new List<byte>();
var fields = new List<byte[]>();
@ -509,12 +509,12 @@ public class SaneTsv
return parsed.ToArray();
}
protected static T ParseCurrentCommentedRecord<T>(ColumnType[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : CommentedTsvRecord, new()
protected static T ParseCurrentCommentedRecord<T>(Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : CommentedTsvRecord, new()
{
return (T)ParseCurrentRecord<T>(columnTypes, properties, fields, comment, line);
}
protected static T ParseCurrentRecord<T>(ColumnType[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : TsvRecord, new()
protected static T ParseCurrentRecord<T>(Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : TsvRecord, new()
{
T record = new T();
@ -532,7 +532,7 @@ public class SaneTsv
for (int j = 0; j < fields.Count; j++)
{
// All other types require the content to be UTF-8. Binary fields can ignore that.
if (columnTypes[j].GetType() == typeof(BinaryType))
if (columnTypes[j] == typeof(BinaryType))
{
// TODO: Use faster method for property setting
// e.g. https://blog.marcgravell.com/2012/01/playing-with-your-member.html
@ -541,7 +541,7 @@ public class SaneTsv
properties[j].SetValue(record, fields[j]);
continue;
}
else if (columnTypes[j].GetType() == typeof(Float32LEType))
else if (columnTypes[j] == typeof(Float32LEType))
{
byte[] floatBytes;
if (!LittleEndian)
@ -560,7 +560,7 @@ public class SaneTsv
continue;
}
else if (columnTypes[j].GetType() == typeof(Float64LEType))
else if (columnTypes[j] == typeof(Float64LEType))
{
byte[] floatBytes;
if (!LittleEndian)
@ -592,11 +592,11 @@ public class SaneTsv
// TODO: Add checking for numeric types format
if (columnTypes[j].GetType() == typeof(StringType))
if (columnTypes[j] == typeof(StringType))
{
properties[j].SetValue(record, fieldString);
}
else if (columnTypes[j].GetType() == typeof(BooleanType))
else if (columnTypes[j] == typeof(BooleanType))
{
bool parsedBool;
if (fieldString == "TRUE")
@ -614,7 +614,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedBool);
}
else if (columnTypes[j].GetType() == typeof(Float32Type))
else if (columnTypes[j] == typeof(Float32Type))
{
float parsedFloat;
if (!float.TryParse(fieldString, out parsedFloat))
@ -635,7 +635,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedFloat);
}
else if (columnTypes[j].GetType() == typeof(Float64Type))
else if (columnTypes[j] == typeof(Float64Type))
{
double parsedDouble;
if (!double.TryParse(fieldString, out parsedDouble))
@ -656,7 +656,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedDouble);
}
else if (columnTypes[j].GetType() == typeof(UInt32Type))
else if (columnTypes[j] == typeof(UInt32Type))
{
if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
{
@ -665,7 +665,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedUInt32);
}
else if (columnTypes[j].GetType() == typeof(UInt64Type))
else if (columnTypes[j] == typeof(UInt64Type))
{
if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
{
@ -674,7 +674,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedUInt64);
}
else if (columnTypes[j].GetType() == typeof(Int32Type))
else if (columnTypes[j] == typeof(Int32Type))
{
if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
{
@ -683,7 +683,7 @@ public class SaneTsv
properties[j].SetValue(record, parsedInt32);
}
else if (columnTypes[j].GetType() == typeof(Int64Type))
else if (columnTypes[j] == typeof(Int64Type))
{
if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
{
@ -1066,151 +1066,99 @@ public class SaneTsv
return records.ToArray();
}
public static ColumnType GetColumnFromString(string type)
{
if (type == "string")
{
return new StringType();
}
else if (type == "boolean")
{
return new BooleanType();
}
else if (type == "float32")
{
return new Float32Type();
}
else if (type == "float32-le")
{
return new Float32LEType();
}
else if (type == "float64")
{
return new Float64Type();
}
else if (type == "float64-le")
{
return new Float64LEType();
}
else if (type == "uint32")
{
return new UInt32Type();
}
else if (type == "uint64")
{
return new UInt64Type();
}
else if (type == "int32")
{
return new Int32Type();
}
else if (type == "int64")
{
return new Int64Type();
}
else if (type == "binary")
{
return new BinaryType();
}
else
{
throw new Exception($"Invalid type: {type.GetType()}");
}
}
public static ColumnType GetColumnFromType(Type type)
public static Type GetColumnFromType(Type type)
{
if (type == typeof(string))
{
return new StringType();
return typeof(StringType);
}
else if (type == typeof(bool))
{
return new BooleanType();
return typeof(BooleanType);
}
else if (type == typeof(float))
{
return new Float32Type();
return typeof(Float32Type);
}
else if (type == typeof(double))
{
return new Float64Type();
return typeof(Float64Type);
}
else if (type == typeof(UInt32))
{
return new UInt32Type();
return typeof(UInt32Type);
}
else if (type == typeof(UInt64))
{
return new UInt64Type();
return typeof(UInt64Type);
}
else if (type == typeof(Int32))
{
return new Int32Type();
return typeof(Int32Type);
}
else if (type == typeof(Int64))
{
return new Int64Type();
return typeof(Int64Type);
}
else if (type == typeof(byte[]))
{
return new BinaryType();
return typeof(BinaryType);
}
else
{
throw new Exception($"Invalid type: {type.GetType()}");
throw new Exception($"Invalid type: {type}");
}
}
public static string GetNameFromColumn(ColumnType type)
public static string GetNameFromColumn(Type type)
{
if (type.GetType() == typeof(StringType))
if (type == typeof(StringType))
{
return "string";
}
else if (type.GetType() == typeof(BooleanType))
else if (type == typeof(BooleanType))
{
return "boolean";
}
else if (type.GetType() == typeof(Float32Type))
else if (type == typeof(Float32Type))
{
return "float32";
}
else if (type.GetType() == typeof(Float32LEType))
else if (type == typeof(Float32LEType))
{
return "float32-le";
}
else if (type.GetType() == typeof(Float64Type))
else if (type == typeof(Float64Type))
{
return "float64";
}
else if (type.GetType() == typeof(Float64LEType))
else if (type == typeof(Float64LEType))
{
return "float64-le";
}
else if (type.GetType() == typeof(UInt32Type))
else if (type == typeof(UInt32Type))
{
return "uint32";
}
else if (type.GetType() == typeof(UInt64Type))
else if (type == typeof(UInt64Type))
{
return "uint64";
}
else if (type.GetType() == typeof(Int32Type))
else if (type == typeof(Int32Type))
{
return "int32";
}
else if (type.GetType() == typeof(Int64Type))
else if (type == typeof(Int64Type))
{
return "int64";
}
else if (type.GetType() == typeof(BinaryType))
else if (type == typeof(BinaryType))
{
return "binary";
}
else
{
throw new Exception($"Invalid type: {type.GetType()}");
throw new Exception($"Invalid type: {type}");
}
}
@ -1226,24 +1174,14 @@ public class SaneTsv
public static byte[] SerializeCommentedTsv<T>(IList<T> data, string fileComment) where T : CommentedTsvRecord
{
return SerializeTsv<T>(data, FormatType.COMMENTED_TSV, fileComment);
return SerializeTsv<T>(data, FormatType.COMMENTED_TSV);
}
protected static byte[] SerializeTsv<T>(IList<T> data, FormatType tsvFormat, string fileComment = null)
protected static byte[] SerializeTsv<T>(IList<T> data, FormatType tsvFormat)
{
var bytes = new List<byte>();
if (fileComment != null)
{
if (tsvFormat != FormatType.COMMENTED_TSV)
{
throw new Exception($"File comments are not valid for {tsvFormat}");
}
bytes.AddRange(Encoding.UTF8.GetBytes("#" + fileComment.Replace("\n", "\n#") + "\n"));
}
var columnTypes = new List<ColumnType>();
var columnTypes = new List<Type>();
var columnNames = new List<string>();
var columnPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
@ -1259,8 +1197,8 @@ public class SaneTsv
string headerName = attribute.ColumnName ?? property.Name;
columnNames.Add(headerName);
ColumnType headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType);
if (tsvFormat == FormatType.SIMPLE_TSV && headerType.GetType() != typeof(StringType))
Type headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType);
if (tsvFormat == FormatType.SIMPLE_TSV && headerType != typeof(StringType))
{
throw new Exception($"Serializing Simple TSV requires all columns be of type string, but column '{headerName}' has type '{headerType}'");
}
@ -1340,7 +1278,7 @@ public class SaneTsv
return bytes.ToArray();
}
protected static void SerializeTsv<T>(IList<T> data, List<byte> bytes, PropertyInfo[] columnPropertyInfos, ColumnType[] columnTypes, FormatType tsvFormat, int startIndex, int endIndex)
protected static void SerializeTsv<T>(IList<T> data, List<byte> bytes, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, FormatType tsvFormat, int startIndex, int endIndex)
{
// Serialize data
for (int i = 0; i < data.Count; i++)
@ -1355,16 +1293,16 @@ public class SaneTsv
// Some fields definitely don't need escaping, so we add them directly to bytes
bool skipEscaping = false;
if (columnTypes[j].GetType() == typeof(StringType))
if (columnTypes[j] == typeof(StringType))
{
fieldEncoded = Encoding.UTF8.GetBytes((string)datum);
}
else if (columnTypes[j].GetType() == typeof(BooleanType))
else if (columnTypes[j] == typeof(BooleanType))
{
bytes.AddRange((bool)datum ? TrueEncoded : FalseEncoded);
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(Float32Type))
else if (columnTypes[j] == typeof(Float32Type))
{
if (datum is float f)
{
@ -1388,7 +1326,7 @@ public class SaneTsv
}
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(Float32LEType))
else if (columnTypes[j] == typeof(Float32LEType))
{
if (LittleEndian)
{
@ -1404,7 +1342,7 @@ public class SaneTsv
}
}
}
else if (columnTypes[j].GetType() == typeof(Float64Type))
else if (columnTypes[j] == typeof(Float64Type))
{
if (datum is double d)
{
@ -1428,7 +1366,7 @@ public class SaneTsv
}
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(Float64LEType))
else if (columnTypes[j] == typeof(Float64LEType))
{
if (LittleEndian)
{
@ -1444,27 +1382,27 @@ public class SaneTsv
}
}
}
else if (columnTypes[j].GetType() == typeof(UInt32Type))
else if (columnTypes[j] == typeof(UInt32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)datum).ToString()));
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(UInt64Type))
else if (columnTypes[j] == typeof(UInt64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)datum).ToString()));
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(Int32Type))
else if (columnTypes[j] == typeof(Int32Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)datum).ToString()));
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(Int64Type))
else if (columnTypes[j] == typeof(Int64Type))
{
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)datum).ToString()));
skipEscaping = true;
}
else if (columnTypes[j].GetType() == typeof(BinaryType))
else if (columnTypes[j] == typeof(BinaryType))
{
fieldEncoded = (byte[])datum;
}
@ -1574,15 +1512,15 @@ public class SaneTsv
public class TsvColumnAttribute : Attribute
{
public string ColumnName { get; }
public virtual ColumnType ColumnType { get; }
public virtual Type ColumnType { get; }
public TsvColumnAttribute()
{
ColumnType = new StringType();
ColumnType = typeof(StringType);
}
public TsvColumnAttribute(string columnName)
{
ColumnType = new StringType();
ColumnType = typeof(StringType);
ColumnName = columnName;
}
}
@ -1590,19 +1528,27 @@ public class SaneTsv
// TODO: Add column ordering
public class TypedTsvColumnAttribute : TsvColumnAttribute
{
public override ColumnType ColumnType { get; }
public override Type ColumnType { get; }
public TypedTsvColumnAttribute() { }
public TypedTsvColumnAttribute(string columnName) : base(columnName) { }
public TypedTsvColumnAttribute(string columnName, string columnType) : base(columnName)
public TypedTsvColumnAttribute(string columnName, Type columnType) : base(columnName)
{
ColumnType = GetColumnFromString(columnType);
if (columnType.BaseType != typeof(ColumnType))
{
throw new Exception("Column type must inherit from SaneTsv.ColumnType");
}
ColumnType = columnType;
}
public TypedTsvColumnAttribute(ColumnType columnType)
public TypedTsvColumnAttribute(Type columnType)
{
if (columnType.BaseType != typeof(ColumnType))
{
throw new Exception("Column type must inherit from SaneTsv.ColumnType");
}
ColumnType = columnType;
}
}

24
SaneTsv/SaneTsvTest/Program.cs
View File

@ -759,30 +759,6 @@ internal class Program : SaneTsv
}
}
{
string testName = "File comment serde";
string testString1 = "#this is a file comment" +
"\n# and one more line since you're such a good customer" +
"\ncolumn1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
string reserialized = Encoding.UTF8.GetString(SaneTsv.SerializeCommentedTsv<BoolTestRecord2>(parsed.Records, parsed.FileComment));
if (reserialized == testString1)
{
Console.WriteLine($"Passed {testName}");
}
else
{
Console.WriteLine($"Failed {testName}");
}
}
Console.WriteLine("Done with tests");
}
}