Compare commits
No commits in common. "0fd092685d9bb2d24437466e44669172a227dc91" and "f98a40a173fb773d42d48cffc54620f711a5b2c8" have entirely different histories.
0fd092685d
...
f98a40a173
@ -43,6 +43,9 @@ public class SaneTsv
|
||||
public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE");
|
||||
public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE");
|
||||
|
||||
// TODO: We need to be able to update all these in tandem somehow
|
||||
public string[] ColumnNames { get; protected set; }
|
||||
public Type[] ColumnTypes { get; protected set; }
|
||||
protected static bool? _littleEndian = null;
|
||||
public static bool LittleEndian
|
||||
{
|
||||
@ -68,6 +71,7 @@ public class SaneTsv
|
||||
|
||||
public static CommentedTsv<T> ParseCommentedTsv<T>(byte[] inputBuffer) where T : CommentedTsvRecord, new()
|
||||
{
|
||||
// TODO: add the file comment?
|
||||
return (CommentedTsv<T>)Parse<T>(inputBuffer, FormatType.COMMENTED_TSV);
|
||||
}
|
||||
|
||||
@ -84,22 +88,22 @@ public class SaneTsv
|
||||
}
|
||||
parsed.Records = new List<T>();
|
||||
|
||||
var columnTypes = new List<Type>();
|
||||
var columnNames = new List<string>();
|
||||
var columnPropertyInfos = new List<PropertyInfo>();
|
||||
var headerTypes = new List<Type>();
|
||||
var headerNames = new List<string>();
|
||||
var headerPropertyInfos = new List<PropertyInfo>();
|
||||
int columnCount = 0;
|
||||
|
||||
foreach (PropertyInfo property in typeof(T).GetProperties())
|
||||
{
|
||||
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
|
||||
TypedTsvColumnAttribute attribute = (TypedTsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TypedTsvColumnAttribute));
|
||||
if (attribute == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
columnNames.Add(attribute.ColumnName ?? property.Name);
|
||||
columnTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
|
||||
columnPropertyInfos.Add(property);
|
||||
headerNames.Add(attribute.ColumnName ?? property.Name);
|
||||
headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
|
||||
headerPropertyInfos.Add(property);
|
||||
// TODO: Check that the property type and given column type are compatible
|
||||
columnCount++;
|
||||
}
|
||||
@ -166,7 +170,7 @@ public class SaneTsv
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
throw new Exception($"Header field {fields.Count} is not valid UTF-8", e);
|
||||
throw new Exception($"Header {fields.Count} is not valid UTF-8", e);
|
||||
}
|
||||
|
||||
string columnTypeString;
|
||||
@ -175,7 +179,7 @@ public class SaneTsv
|
||||
{
|
||||
if (format == FormatType.SIMPLE_TSV)
|
||||
{
|
||||
throw new Exception($"Header field {j} contains ':', which is not allowed for column names");
|
||||
throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
|
||||
}
|
||||
columnTypeString = columnString.Split(":").Last();
|
||||
columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
|
||||
@ -184,7 +188,7 @@ public class SaneTsv
|
||||
{
|
||||
if (format > FormatType.SIMPLE_TSV)
|
||||
{
|
||||
throw new Exception($"Header field {fields.Count} has no type");
|
||||
throw new Exception($"Header {fields.Count} has no type");
|
||||
}
|
||||
columnTypeString = "";
|
||||
columnName = columnString;
|
||||
@ -237,14 +241,14 @@ public class SaneTsv
|
||||
|
||||
// TODO: Allow lax parsing (only worry about parsing columns that are given in the specifying type
|
||||
|
||||
if (columnNames[j] != columnName)
|
||||
if (headerNames[j] != columnName)
|
||||
{
|
||||
throw new Exception($"Column {j} has name {columnName}, but expected {columnNames[j]}");
|
||||
throw new Exception($"Column {j} has name {columnName}, but expected {headerNames[j]}");
|
||||
}
|
||||
|
||||
if (columnTypes[j] != type)
|
||||
if (headerTypes[j] != type)
|
||||
{
|
||||
throw new Exception($"Column {j} has type {type}, but expected {columnTypes[j]}");
|
||||
throw new Exception($"Column {j} has type {type}, but expected {headerTypes[j]}");
|
||||
}
|
||||
}
|
||||
|
||||
@ -308,7 +312,7 @@ public class SaneTsv
|
||||
// Complication: it probably depends on processor count
|
||||
if (inputBuffer.Length < 10000)
|
||||
{
|
||||
parsed.Records.AddRange(Parse<T>(inputBuffer, format, columnPropertyInfos.ToArray(), columnTypes.ToArray(), currentLineStart - 1, inputBuffer.Length));
|
||||
parsed.Records.AddRange(Parse<T>(inputBuffer, format, headerPropertyInfos.ToArray(), headerTypes.ToArray(), currentLineStart - 1, inputBuffer.Length));
|
||||
return parsed;
|
||||
}
|
||||
else
|
||||
@ -330,10 +334,9 @@ public class SaneTsv
|
||||
endIndex = (i + 1) * splitCount + parseStart;
|
||||
}
|
||||
|
||||
parsedValues[i] = Parse<T>(inputBuffer, format, columnPropertyInfos.ToArray(), columnTypes.ToArray(), startIndex, endIndex);
|
||||
parsedValues[i] = Parse<T>(inputBuffer, format, headerPropertyInfos.ToArray(), headerTypes.ToArray(), startIndex, endIndex);
|
||||
});
|
||||
|
||||
// TODO: Handle relative line numbers
|
||||
for (int i = 0; i < tasks; i++)
|
||||
{
|
||||
parsed.Records.AddRange(parsedValues[i]);
|
||||
@ -348,7 +351,7 @@ public class SaneTsv
|
||||
// startIndex is in we'd have to go back to the start of the record's comment, and to know
|
||||
// exactly where that comment started we'd have to go back to the start of the record before that
|
||||
// (not including that other record's comment).
|
||||
protected static T[] Parse<T>(byte[] inputBuffer, FormatType format, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, int startIndex, int endIndex) where T : TsvRecord, new()
|
||||
protected static T[] Parse<T>(byte[] inputBuffer, FormatType format, PropertyInfo[] headerPropertyInfos, Type[] headerTypes, int startIndex, int endIndex) where T : TsvRecord, new()
|
||||
{
|
||||
var fieldBytes = new List<byte>();
|
||||
var fields = new List<byte[]>();
|
||||
@ -418,9 +421,9 @@ public class SaneTsv
|
||||
fields.Add(fieldBytes.ToArray());
|
||||
fieldBytes.Clear();
|
||||
|
||||
if (columnTypes.Length != fields.Count)
|
||||
if (headerTypes.Length != fields.Count)
|
||||
{
|
||||
throw new Exception($"Expected {columnTypes.Length} fields on line {relativeLine}, but found {fields.Count}");
|
||||
throw new Exception($"Expected {headerTypes.Length} fields on line {relativeLine}, but found {fields.Count}");
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -430,7 +433,7 @@ public class SaneTsv
|
||||
comment = currentComment.ToString();
|
||||
currentComment.Clear();
|
||||
}
|
||||
parsed.Add(ParseCurrentRecord<T>(columnTypes.ToArray(), columnPropertyInfos.ToArray(), fields, comment, relativeLine));
|
||||
parsed.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, relativeLine));
|
||||
fields.Clear();
|
||||
}
|
||||
|
||||
@ -490,9 +493,9 @@ public class SaneTsv
|
||||
// TODO
|
||||
throw new Exception("Not sure when this will happen. THis might actuall be fine");
|
||||
}
|
||||
if (fields.Count != columnTypes.Length)
|
||||
if (fields.Count != headerTypes.Length)
|
||||
{
|
||||
throw new Exception($"Expected {columnTypes} fields on line {relativeLine}, but found {fields.Count}");
|
||||
throw new Exception($"Expected {headerTypes} fields on line {relativeLine}, but found {fields.Count}");
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -502,7 +505,7 @@ public class SaneTsv
|
||||
comment = currentComment.ToString();
|
||||
currentComment.Clear();
|
||||
}
|
||||
parsed.Add(ParseCurrentRecord<T>(columnTypes.ToArray(), columnPropertyInfos.ToArray(), fields, comment, relativeLine));
|
||||
parsed.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, relativeLine));
|
||||
fields.Clear();
|
||||
}
|
||||
|
||||
@ -843,7 +846,7 @@ public class SaneTsv
|
||||
public static (string[] columns, string[][] data) ParseSimpleTsv(byte[] inputBuffer)
|
||||
{
|
||||
string[] columnNames = null;
|
||||
var headerFields = new List<byte[]>();
|
||||
var headers = new List<byte[]>();
|
||||
var fieldBytes = new List<byte>();
|
||||
int startOfData = -1;
|
||||
for (int i = 0; i < inputBuffer.Count(); i++)
|
||||
@ -882,33 +885,33 @@ public class SaneTsv
|
||||
else if (inputBuffer[i] == '\t')
|
||||
{
|
||||
// end of field
|
||||
headerFields.Add(fieldBytes.ToArray());
|
||||
headers.Add(fieldBytes.ToArray());
|
||||
fieldBytes.Clear();
|
||||
}
|
||||
else if (inputBuffer[i] == '\n')
|
||||
{
|
||||
// This is the end of the header
|
||||
headerFields.Add(fieldBytes.ToArray());
|
||||
headers.Add(fieldBytes.ToArray());
|
||||
startOfData = i + 1;
|
||||
|
||||
columnNames = new string[headerFields.Count];
|
||||
columnNames = new string[headers.Count];
|
||||
fieldBytes.Clear();
|
||||
|
||||
for (int j = 0; j < headerFields.Count; j++)
|
||||
for (int j = 0; j < headers.Count; j++)
|
||||
{
|
||||
string columnString;
|
||||
try
|
||||
{
|
||||
columnString = Encoding.UTF8.GetString(headerFields[j]);
|
||||
columnString = Encoding.UTF8.GetString(headers[j]);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
throw new Exception($"Column {headerFields.Count} name is not valid UTF-8", e);
|
||||
throw new Exception($"Column {headers.Count} name is not valid UTF-8", e);
|
||||
}
|
||||
|
||||
if (columnString.Contains(':'))
|
||||
{
|
||||
throw new Exception($"Header field {headerFields.Count} contain ':', which is not allowed for column names");
|
||||
throw new Exception($"Header {headers.Count} contain ':', which is not allowed for column names");
|
||||
}
|
||||
|
||||
columnNames[j] = columnString;
|
||||
@ -936,7 +939,7 @@ public class SaneTsv
|
||||
var fields = new List<byte[]>();
|
||||
var records = new List<string[]>();
|
||||
|
||||
int line = 2;
|
||||
int line = 1;
|
||||
int currentLineStart = 0;
|
||||
|
||||
// Go back to the start of the current line
|
||||
@ -1030,20 +1033,13 @@ public class SaneTsv
|
||||
|
||||
fields.Add(fieldBytes.ToArray());
|
||||
|
||||
if (fields.Count == 0 && endIndex == inputBuffer.Length)
|
||||
if (fields.Count == 0)
|
||||
{
|
||||
throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
|
||||
}
|
||||
if (numFields != fields.Count)
|
||||
{
|
||||
if (endIndex == inputBuffer.Length)
|
||||
{
|
||||
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
|
||||
}
|
||||
else
|
||||
{
|
||||
return records.ToArray();
|
||||
}
|
||||
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -1181,9 +1177,9 @@ public class SaneTsv
|
||||
{
|
||||
var bytes = new List<byte>();
|
||||
|
||||
var columnTypes = new List<Type>();
|
||||
var columnNames = new List<string>();
|
||||
var columnPropertyInfos = new List<PropertyInfo>();
|
||||
var headerTypes = new List<Type>();
|
||||
var headerNames = new List<string>();
|
||||
var headerPropertyInfos = new List<PropertyInfo>();
|
||||
int columnCount = 0;
|
||||
|
||||
// Serialize header
|
||||
@ -1196,30 +1192,30 @@ public class SaneTsv
|
||||
}
|
||||
|
||||
string headerName = attribute.ColumnName ?? property.Name;
|
||||
columnNames.Add(headerName);
|
||||
headerNames.Add(headerName);
|
||||
Type headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType);
|
||||
if (tsvFormat == FormatType.SIMPLE_TSV && headerType != typeof(StringType))
|
||||
{
|
||||
throw new Exception($"Serializing Simple TSV requires all columns be of type string, but column '{headerName}' has type '{headerType}'");
|
||||
}
|
||||
columnTypes.Add(headerType);
|
||||
columnPropertyInfos.Add(property);
|
||||
headerTypes.Add(headerType);
|
||||
headerPropertyInfos.Add(property);
|
||||
// TODO: Check that the property type and given column type are compatible
|
||||
columnCount++;
|
||||
}
|
||||
|
||||
// Serialize header
|
||||
for (int i = 0; i < columnNames.Count; i++)
|
||||
for (int i = 0; i < headerNames.Count; i++)
|
||||
{
|
||||
for (int j = i + 1; j < columnNames.Count; j++)
|
||||
for (int j = i + 1; j < headerNames.Count; j++)
|
||||
{
|
||||
if (columnNames[i] == columnNames[j])
|
||||
if (headerNames[i] == headerNames[j])
|
||||
{
|
||||
throw new Exception("Column names in header must be unique");
|
||||
}
|
||||
}
|
||||
|
||||
byte[] nameEncoded = Encoding.UTF8.GetBytes(columnNames[i]);
|
||||
byte[] nameEncoded = Encoding.UTF8.GetBytes(headerNames[i]);
|
||||
|
||||
for (int j = 0; j < nameEncoded.Length; j++)
|
||||
{
|
||||
@ -1254,15 +1250,15 @@ public class SaneTsv
|
||||
bytes.Add((byte)':');
|
||||
try
|
||||
{
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(columnTypes[i])));
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(headerTypes[i])));
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
throw new Exception($"Invalid column type for column {i}", e);
|
||||
throw new Exception($"Invalid header type for column {i}", e);
|
||||
}
|
||||
}
|
||||
|
||||
if (i == columnNames.Count - 1)
|
||||
if (i == headerNames.Count - 1)
|
||||
{
|
||||
bytes.Add((byte)'\n');
|
||||
}
|
||||
@ -1273,19 +1269,19 @@ public class SaneTsv
|
||||
}
|
||||
|
||||
// Serialize data
|
||||
SerializeTsv<T>(data, bytes, columnPropertyInfos.ToArray(), columnTypes.ToArray(), tsvFormat, 0, data.Count);
|
||||
SerializeTsv<T>(data, bytes, headerPropertyInfos.ToArray(), headerTypes.ToArray(), tsvFormat, 0, data.Count);
|
||||
|
||||
return bytes.ToArray();
|
||||
}
|
||||
|
||||
protected static void SerializeTsv<T>(IList<T> data, List<byte> bytes, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, FormatType tsvFormat, int startIndex, int endIndex)
|
||||
protected static void SerializeTsv<T>(IList<T> data, List<byte> bytes, PropertyInfo[] headerPropertyInfos, Type[] headerTypes, FormatType tsvFormat, int startIndex, int endIndex)
|
||||
{
|
||||
// Serialize data
|
||||
for (int i = 0; i < data.Count; i++)
|
||||
{
|
||||
for (int j = 0; j < columnTypes.Length; j++)
|
||||
for (int j = 0; j < headerTypes.Length; j++)
|
||||
{
|
||||
object datum = columnPropertyInfos[j].GetValue(data[i]);
|
||||
object datum = headerPropertyInfos[j].GetValue(data[i]);
|
||||
|
||||
try
|
||||
{
|
||||
@ -1293,16 +1289,16 @@ public class SaneTsv
|
||||
// Some fields definitely don't need escaping, so we add them directly to bytes
|
||||
bool skipEscaping = false;
|
||||
|
||||
if (columnTypes[j] == typeof(StringType))
|
||||
if (headerTypes[j] == typeof(StringType))
|
||||
{
|
||||
fieldEncoded = Encoding.UTF8.GetBytes((string)datum);
|
||||
}
|
||||
else if (columnTypes[j] == typeof(BooleanType))
|
||||
else if (headerTypes[j] == typeof(BooleanType))
|
||||
{
|
||||
bytes.AddRange((bool)datum ? TrueEncoded : FalseEncoded);
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Float32Type))
|
||||
else if (headerTypes[j] == typeof(Float32Type))
|
||||
{
|
||||
if (datum is float f)
|
||||
{
|
||||
@ -1326,7 +1322,7 @@ public class SaneTsv
|
||||
}
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Float32LEType))
|
||||
else if (headerTypes[j] == typeof(Float32LEType))
|
||||
{
|
||||
if (LittleEndian)
|
||||
{
|
||||
@ -1342,7 +1338,7 @@ public class SaneTsv
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Float64Type))
|
||||
else if (headerTypes[j] == typeof(Float64Type))
|
||||
{
|
||||
if (datum is double d)
|
||||
{
|
||||
@ -1366,7 +1362,7 @@ public class SaneTsv
|
||||
}
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Float64LEType))
|
||||
else if (headerTypes[j] == typeof(Float64LEType))
|
||||
{
|
||||
if (LittleEndian)
|
||||
{
|
||||
@ -1382,33 +1378,33 @@ public class SaneTsv
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (columnTypes[j] == typeof(UInt32Type))
|
||||
else if (headerTypes[j] == typeof(UInt32Type))
|
||||
{
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)datum).ToString()));
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(UInt64Type))
|
||||
else if (headerTypes[j] == typeof(UInt64Type))
|
||||
{
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)datum).ToString()));
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Int32Type))
|
||||
else if (headerTypes[j] == typeof(Int32Type))
|
||||
{
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)datum).ToString()));
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(Int64Type))
|
||||
else if (headerTypes[j] == typeof(Int64Type))
|
||||
{
|
||||
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)datum).ToString()));
|
||||
skipEscaping = true;
|
||||
}
|
||||
else if (columnTypes[j] == typeof(BinaryType))
|
||||
else if (headerTypes[j] == typeof(BinaryType))
|
||||
{
|
||||
fieldEncoded = (byte[])datum;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception($"Unexpected column type {columnTypes[j]} for column {j}");
|
||||
throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
|
||||
}
|
||||
|
||||
if (!skipEscaping)
|
||||
@ -1442,7 +1438,7 @@ public class SaneTsv
|
||||
}
|
||||
}
|
||||
|
||||
if (j < columnTypes.Length - 1)
|
||||
if (j < headerTypes.Length - 1)
|
||||
{
|
||||
bytes.Add((byte)'\t');
|
||||
}
|
||||
@ -1453,7 +1449,7 @@ public class SaneTsv
|
||||
}
|
||||
catch (InvalidCastException e)
|
||||
{
|
||||
throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(columnTypes[j])}", e);
|
||||
throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(headerTypes[j])}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
using NathanMcRae;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
|
||||
internal class Program : SaneTsv
|
||||
@ -81,18 +80,6 @@ internal class Program : SaneTsv
|
||||
public string Column3 { get; set; }
|
||||
}
|
||||
|
||||
public class BoolTestRecord3 : SaneTsv.CommentedTsvRecord
|
||||
{
|
||||
[SaneTsv.TsvColumn("column1")]
|
||||
public string Column1 { get; set; }
|
||||
|
||||
[SaneTsv.TsvColumn]
|
||||
public string column2 { get; set; }
|
||||
|
||||
[SaneTsv.TsvColumn("columnthree\nyep")]
|
||||
public string Column3 { get; set; }
|
||||
}
|
||||
|
||||
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
|
||||
{
|
||||
[SaneTsv.TypedTsvColumn("column1")]
|
||||
@ -361,404 +348,6 @@ internal class Program : SaneTsv
|
||||
Console.WriteLine($"Unspecced parse time: {unspeccedParseTime}");
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "With and without file comment";
|
||||
|
||||
string testString1 = "#This is a file comment\n" +
|
||||
"#One more file comment line\n" +
|
||||
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\n#This is a comment" +
|
||||
"\n#Another comment line" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther";
|
||||
|
||||
string testString2 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\n#This is a comment" +
|
||||
"\n#Another comment line" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther";
|
||||
|
||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
CommentedTsv<BoolTestRecord2> parsed2 = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2));
|
||||
|
||||
if (parsed.FileComment == "This is a file comment\nOne more file comment line" && parsed2.FileComment == null)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Failed {testName}");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "With and without types";
|
||||
|
||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther";
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Passed {testName} 1A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 1A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1B");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1C");
|
||||
}
|
||||
|
||||
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
|
||||
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther";
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 2A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 2A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 2B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 2B");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 2C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 2C");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "With and without line comment";
|
||||
|
||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\n#This is a comment" +
|
||||
"\n#Another comment line" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther";
|
||||
|
||||
try
|
||||
{
|
||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Passed {testName} 1A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 1A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1B");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1C");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1D");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1D");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "End of file comment";
|
||||
|
||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
|
||||
|
||||
try
|
||||
{
|
||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1B");
|
||||
}
|
||||
|
||||
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1C");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1D");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1D");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "Partial parsing";
|
||||
|
||||
string line1 = "column1\tcolumn2\tcolumnthree\\nyep";
|
||||
string line2 = "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee";
|
||||
string line3 = "\nFALSE\tnother\tno\\ther";
|
||||
|
||||
byte[] inputBuffer = Encoding.UTF8.GetBytes(line1 + line2 + line3);
|
||||
|
||||
var headerTypes = new List<Type>();
|
||||
var headerNames = new List<string>();
|
||||
var headerPropertyInfos = new List<PropertyInfo>();
|
||||
int columnCount = 0;
|
||||
|
||||
foreach (PropertyInfo property in typeof(BoolTestRecord3).GetProperties())
|
||||
{
|
||||
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
|
||||
if (attribute == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
headerNames.Add(attribute.ColumnName ?? property.Name);
|
||||
headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
|
||||
headerPropertyInfos.Add(property);
|
||||
// TODO: Check that the property type and given column type are compatible
|
||||
columnCount++;
|
||||
}
|
||||
|
||||
BoolTestRecord3[] records = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
|
||||
FormatType.SIMPLE_TSV,
|
||||
headerPropertyInfos.ToArray(),
|
||||
headerTypes.ToArray(),
|
||||
line1.Length + line2.Length + 1,
|
||||
inputBuffer.Length);
|
||||
|
||||
if (records.Length == 0 )
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 1");
|
||||
}
|
||||
|
||||
BoolTestRecord3[] records2 = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
|
||||
FormatType.SIMPLE_TSV,
|
||||
headerPropertyInfos.ToArray(),
|
||||
headerTypes.ToArray(),
|
||||
line1.Length,
|
||||
line1.Length + 3);
|
||||
|
||||
if (records2[0].Column3 == "valuetrhee")
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 2");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 2");
|
||||
}
|
||||
|
||||
string[][] data = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length + line2.Length + 1, inputBuffer.Length);
|
||||
|
||||
if (data[0][1] == "nother")
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 3");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 3");
|
||||
}
|
||||
|
||||
string[][] data2 = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length, line1.Length + 3);
|
||||
|
||||
if (data2.Length == 0)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 4");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"Failed {testName} 4");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "End of file \\n";
|
||||
|
||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\n";
|
||||
|
||||
try
|
||||
{
|
||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1B");
|
||||
}
|
||||
|
||||
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\n";
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1C");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1D");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1D");
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
string testName = "End of file partial record";
|
||||
|
||||
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\nTRUE\t";
|
||||
|
||||
try
|
||||
{
|
||||
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1A");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1A");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
|
||||
Console.WriteLine($"Failed {testName} 1B");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1B");
|
||||
}
|
||||
|
||||
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
|
||||
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
|
||||
"\nFALSE\tnother\tno\\ther" +
|
||||
"\nTRUE\t";
|
||||
|
||||
try
|
||||
{
|
||||
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1C");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1C");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
|
||||
Console.WriteLine($"Failed {testName} 1D");
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine($"Passed {testName} 1D");
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Done with tests");
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user