Compare commits

..

No commits in common. "8e3332b484966982d31d52ff06f8f5330d47ac42" and "a7e6f1c0e91a4a186879ef3c1726adc74ca21abd" have entirely different histories.

2 changed files with 511 additions and 731 deletions

View File

@ -43,6 +43,9 @@ public class SaneTsv
public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE"); public static readonly byte[] TrueEncoded = Encoding.UTF8.GetBytes("TRUE");
public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE"); public static readonly byte[] FalseEncoded = Encoding.UTF8.GetBytes("FALSE");
// TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; }
public Type[] ColumnTypes { get; protected set; }
protected static bool? _littleEndian = null; protected static bool? _littleEndian = null;
public static bool LittleEndian public static bool LittleEndian
{ {
@ -68,9 +71,11 @@ public class SaneTsv
public static CommentedTsv<T> ParseCommentedTsv<T>(byte[] inputBuffer) where T : CommentedTsvRecord, new() public static CommentedTsv<T> ParseCommentedTsv<T>(byte[] inputBuffer) where T : CommentedTsvRecord, new()
{ {
// TODO: add the file comment?
return (CommentedTsv<T>)Parse<T>(inputBuffer, FormatType.COMMENTED_TSV); return (CommentedTsv<T>)Parse<T>(inputBuffer, FormatType.COMMENTED_TSV);
} }
// TODO: Have parsing errors include line / column #
protected static Tsv<T> Parse<T>(byte[] inputBuffer, FormatType format) where T : TsvRecord, new() protected static Tsv<T> Parse<T>(byte[] inputBuffer, FormatType format) where T : TsvRecord, new()
{ {
Tsv<T> parsed; Tsv<T> parsed;
@ -84,22 +89,22 @@ public class SaneTsv
} }
parsed.Records = new List<T>(); parsed.Records = new List<T>();
var columnTypes = new List<Type>(); var headerTypes = new List<Type>();
var columnNames = new List<string>(); var headerNames = new List<string>();
var columnPropertyInfos = new List<PropertyInfo>(); var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0; int columnCount = 0;
foreach (PropertyInfo property in typeof(T).GetProperties()) foreach (PropertyInfo property in typeof(T).GetProperties())
{ {
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute)); TypedTsvColumnAttribute attribute = (TypedTsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TypedTsvColumnAttribute));
if (attribute == null) if (attribute == null)
{ {
continue; continue;
} }
columnNames.Add(attribute.ColumnName ?? property.Name); headerNames.Add(attribute.ColumnName ?? property.Name);
columnTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType)); headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
columnPropertyInfos.Add(property); headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible // TODO: Check that the property type and given column type are compatible
columnCount++; columnCount++;
} }
@ -155,6 +160,12 @@ public class SaneTsv
fields.Add(fieldBytes.ToArray()); fields.Add(fieldBytes.ToArray());
fieldBytes.Clear(); fieldBytes.Clear();
if (numFields < 0)
{
// This is the header
numFields = fields.Count;
int numTypesBlank = 0; int numTypesBlank = 0;
for (int j = 0; j < fields.Count; j++) for (int j = 0; j < fields.Count; j++)
@ -166,7 +177,7 @@ public class SaneTsv
} }
catch (Exception e) catch (Exception e)
{ {
throw new Exception($"Header field {fields.Count} is not valid UTF-8", e); throw new Exception($"Header {fields.Count} is not valid UTF-8", e);
} }
string columnTypeString; string columnTypeString;
@ -175,7 +186,7 @@ public class SaneTsv
{ {
if (format == FormatType.SIMPLE_TSV) if (format == FormatType.SIMPLE_TSV)
{ {
throw new Exception($"Header field {j} contains ':', which is not allowed for column names"); throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
} }
columnTypeString = columnString.Split(":").Last(); columnTypeString = columnString.Split(":").Last();
columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1); columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
@ -184,7 +195,7 @@ public class SaneTsv
{ {
if (format > FormatType.SIMPLE_TSV) if (format > FormatType.SIMPLE_TSV)
{ {
throw new Exception($"Header field {fields.Count} has no type"); throw new Exception($"Header {fields.Count} has no type");
} }
columnTypeString = ""; columnTypeString = "";
columnName = columnString; columnName = columnString;
@ -235,16 +246,17 @@ public class SaneTsv
throw new Exception($"Invalid type '{columnTypeString}' for column {j}"); throw new Exception($"Invalid type '{columnTypeString}' for column {j}");
} }
// TODO: Check column name uniqueness
// TODO: Allow lax parsing (only worry about parsing columns that are given in the specifying type // TODO: Allow lax parsing (only worry about parsing columns that are given in the specifying type
if (columnNames[j] != columnName) if (headerNames[j] != columnName)
{ {
throw new Exception($"Column {j} has name {columnName}, but expected {columnNames[j]}"); throw new Exception($"Column {j} has name {columnName}, but expected {headerNames[j]}");
} }
if (columnTypes[j] != type) if (headerTypes[j] != type)
{ {
throw new Exception($"Column {j} has type {type}, but expected {columnTypes[j]}"); throw new Exception($"Column {j} has type {type}, but expected {headerTypes[j]}");
} }
} }
@ -261,13 +273,27 @@ public class SaneTsv
} }
} }
fields.Clear(); fields.Clear();
}
else if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
}
else
{
string comment = null;
if (currentComment.Length > 0)
{
comment = currentComment.ToString();
currentComment.Clear();
}
parsed.Records.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, line));
fields.Clear();
}
line++; line++;
currentLineStart = i + 1; currentLineStart = i + 1;
// Done parsing header
break;
} }
else if (inputBuffer[i] == '#') else if (inputBuffer[i] == '#')
{ {
@ -304,123 +330,15 @@ public class SaneTsv
} }
} }
// TODO: need to figure out where the crossover is
// Complication: it probably depends on processor count
if (inputBuffer.Length < 10000)
{
parsed.Records.AddRange(Parse<T>(inputBuffer, format, columnPropertyInfos.ToArray(), columnTypes.ToArray(), currentLineStart - 1, inputBuffer.Length));
return parsed;
}
else
{
int parseStart = currentLineStart;
int tasks = Environment.ProcessorCount - 1;
int splitCount = (inputBuffer.Length - parseStart) / tasks;
T[][] parsedValues = new T[tasks][];
Parallel.For(0, tasks, i =>
{
int startIndex = i * splitCount + parseStart - 1;
int endIndex;
if (i == tasks - 1)
{
endIndex = inputBuffer.Length;
}
else
{
endIndex = (i + 1) * splitCount + parseStart;
}
parsedValues[i] = Parse<T>(inputBuffer, format, columnPropertyInfos.ToArray(), columnTypes.ToArray(), startIndex, endIndex);
});
// TODO: Handle relative line numbers
for (int i = 0; i < tasks; i++)
{
parsed.Records.AddRange(parsedValues[i]);
}
}
return parsed;
}
// This approach is slightly different than others. We skip the record that startIndex is in and
// include the record that endIndex is in. We do this because in order to include the record
// startIndex is in we'd have to go back to the start of the record's comment, and to know
// exactly where that comment started we'd have to go back to the start of the record before that
// (not including that other record's comment).
protected static T[] Parse<T>(byte[] inputBuffer, FormatType format, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, int startIndex, int endIndex) where T : TsvRecord, new()
{
var fieldBytes = new List<byte>();
var fields = new List<byte[]>();
var currentComment = new StringBuilder();
List<T> parsed = new List<T>();
bool parsingLastRecord = false;
int relativeLine = 0;
int i = startIndex;
while (i < inputBuffer.Length - 1 && inputBuffer[i] != '\n' && inputBuffer[i + 1] != '#')
{
i++;
}
if (i >= inputBuffer.Length - 1)
{
return Array.Empty<T>();
}
// Start parsing after \n
i++;
int currentLineStart = i;
for (; i < inputBuffer.Length && (i < endIndex || parsingLastRecord); i++)
{
if (inputBuffer[i] == '\\')
{
if (i + 1 == inputBuffer.Count())
{
throw new Exception($"Found '\\' at end of input");
}
if (inputBuffer[i + 1] == 'n')
{
fieldBytes.Add((byte)'\n');
i++;
}
else if (inputBuffer[i + 1] == '\\')
{
fieldBytes.Add((byte)'\\');
i++;
}
else if (inputBuffer[i + 1] == 't')
{
fieldBytes.Add((byte)'\t');
i++;
}
else if (inputBuffer[i + 1] == '#')
{
fieldBytes.Add((byte)'#');
i++;
}
else
{
throw new Exception($"Expected 'n', 't', '#', or '\\' after '\\' at line {relativeLine} column {i - currentLineStart}");
}
}
else if (inputBuffer[i] == '\t')
{
// end of field
fields.Add(fieldBytes.ToArray()); fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
}
else if (inputBuffer[i] == '\n')
{
fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
if (columnTypes.Length != fields.Count) if (numFields == 0)
{ {
throw new Exception($"Expected {columnTypes.Length} fields on line {relativeLine}, but found {fields.Count}"); throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
}
if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
} }
else else
{ {
@ -430,83 +348,11 @@ public class SaneTsv
comment = currentComment.ToString(); comment = currentComment.ToString();
currentComment.Clear(); currentComment.Clear();
} }
parsed.Add(ParseCurrentRecord<T>(columnTypes.ToArray(), columnPropertyInfos.ToArray(), fields, comment, relativeLine)); parsed.Records.Add(ParseCurrentRecord<T>(headerTypes.ToArray(), headerPropertyInfos.ToArray(), fields, comment, line));
fields.Clear(); fields.Clear();
} }
parsingLastRecord = false; return parsed;
relativeLine++;
currentLineStart = i + 1;
}
else if (inputBuffer[i] == '#')
{
if (i == currentLineStart && format >= FormatType.COMMENTED_TSV)
{
int j = i;
for (; j < inputBuffer.Length && inputBuffer[j] != '\n'; j++) { }
if (j < inputBuffer.Length)
{
var commentBytes = new byte[j - i - 1];
Array.Copy(inputBuffer, i + 1, commentBytes, 0, j - i - 1);
if (currentComment.Length > 0)
{
currentComment.Append('\n');
}
currentComment.Append(Encoding.UTF8.GetString(commentBytes));
i = j;
currentLineStart = i + 1;
relativeLine++;
}
else
{
throw new Exception("Comments at end of file are not allowed");
}
}
else
{
throw new Exception($"Found unescaped '#' at line {relativeLine}, column {i - currentLineStart}");
}
}
else
{
fieldBytes.Add(inputBuffer[i]);
}
if (i == endIndex - 1)
{
parsingLastRecord = true;
}
}
if (endIndex < inputBuffer.Length)
{
return parsed.ToArray();
}
fields.Add(fieldBytes.ToArray());
if (fields.Count == 0)
{
// TODO
throw new Exception("Not sure when this will happen. THis might actuall be fine");
}
if (fields.Count != columnTypes.Length)
{
throw new Exception($"Expected {columnTypes} fields on line {relativeLine}, but found {fields.Count}");
}
else
{
string comment = null;
if (currentComment.Length > 0)
{
comment = currentComment.ToString();
currentComment.Clear();
}
parsed.Add(ParseCurrentRecord<T>(columnTypes.ToArray(), columnPropertyInfos.ToArray(), fields, comment, relativeLine));
fields.Clear();
}
return parsed.ToArray();
} }
protected static T ParseCurrentCommentedRecord<T>(Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : CommentedTsvRecord, new() protected static T ParseCurrentCommentedRecord<T>(Type[] columnTypes, PropertyInfo[] properties, List<byte[]> fields, string comment, int line) where T : CommentedTsvRecord, new()
@ -702,6 +548,103 @@ public class SaneTsv
} }
public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data) public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data)
{
var escapedString = new StringBuilder();
// Serialize header
for (int i = 0; i < header.Count; i++)
{
if (header[i].Contains(':'))
{
throw new Exception($"Column {i} contains the character ':'");
}
for (int j = i + 1; j < header.Count; j++)
{
if (header[i] == header[j])
{
throw new Exception("Column names in header must be unique");
}
}
for (int j = 0; j < header[i].Count(); j++)
{
if (header[i][j] == '\n')
{
escapedString.Append("\\n");
}
else if (header[i][j] == '\t')
{
escapedString.Append("\\t");
}
else if (header[i][j] == '\\')
{
escapedString.Append("\\\\");
}
else if (header[i][j] == '#')
{
escapedString.Append("\\#");
}
else
{
escapedString.Append(header[i][j]);
}
}
if (i == header.Count - 1)
{
escapedString.Append('\n');
}
else
{
escapedString.Append('\t');
}
}
// Serialize data
for (int i = 0; i < data.Count; i++)
{
for (int j = 0; j < data[i].Count; j++)
{
for (int k = 0; k < data[i][j].Length; k++)
{
if (data[i][j][k] == '\n')
{
escapedString.Append("\\n");
}
else if (data[i][j][k] == '\t')
{
escapedString.Append("\\t");
}
else if (data[i][j][k] == '\\')
{
escapedString.Append("\\\\");
}
else if (data[i][j][k] == '#')
{
escapedString.Append("\\#");
}
else
{
escapedString.Append(data[i][j][k]);
}
}
if (j < data[i].Count - 1)
{
escapedString.Append('\t');
}
else if (i < data.Count - 1)
{
escapedString.Append('\n');
}
}
}
return Encoding.UTF8.GetBytes(escapedString.ToString());
}
public static byte[] SerializeSimpleTsvParallel(IList<string> header, IList<IList<string>> data)
{ {
var serialized = new List<byte>(); var serialized = new List<byte>();
var escapedString = new StringBuilder(); var escapedString = new StringBuilder();
@ -762,7 +705,7 @@ public class SaneTsv
// Complication: it probably depends on processor count // Complication: it probably depends on processor count
if (data.Count < 100) if (data.Count < 100)
{ {
serialized.AddRange(Encoding.UTF8.GetBytes(SerializeSimpleTsv(data, 0, data.Count))); serialized.AddRange(Encoding.UTF8.GetBytes(SerializeSimpleTsvParallel(data, 0, data.Count)));
} }
else else
{ {
@ -780,7 +723,7 @@ public class SaneTsv
{ {
endIndex = (i + 1) * splitCount; endIndex = (i + 1) * splitCount;
} }
string escapedString = SerializeSimpleTsv(data, i * splitCount, endIndex); string escapedString = SerializeSimpleTsvParallel(data, i * splitCount, endIndex);
bytes[i] = Encoding.UTF8.GetBytes(escapedString); bytes[i] = Encoding.UTF8.GetBytes(escapedString);
}); });
@ -793,7 +736,7 @@ public class SaneTsv
return serialized.ToArray(); return serialized.ToArray();
} }
public static string SerializeSimpleTsv(IList<IList<string>> data, int startIndex, int endIndex) public static string SerializeSimpleTsvParallel(IList<IList<string>> data, int startIndex, int endIndex)
{ {
var escapedString = new StringBuilder(); var escapedString = new StringBuilder();
@ -840,10 +783,162 @@ public class SaneTsv
return escapedString.ToString(); return escapedString.ToString();
} }
public static (string[] columns, string[][] data) ParseSimpleTsv(byte[] inputBuffer) public static (string[] columns, string[][] data) ParseSimpleTsv(byte[] inputBuffer)
{ {
string[] columnNames = null; string[] columnNames = null;
var headerFields = new List<byte[]>();
var fieldBytes = new List<byte>();
var fields = new List<byte[]>();
var records = new List<string[]>();
int numFields = -1;
int line = 1;
int currentLineStart = 0;
for (int i = 0; i < inputBuffer.Count(); i++)
{
if (inputBuffer[i] == '\\')
{
if (i + 1 == inputBuffer.Count())
{
throw new Exception($"Found '\\' at end of input");
}
if (inputBuffer[i + 1] == 'n')
{
fieldBytes.Add((byte)'\n');
i++;
}
else if (inputBuffer[i + 1] == '\\')
{
fieldBytes.Add((byte)'\\');
i++;
}
else if (inputBuffer[i + 1] == 't')
{
fieldBytes.Add((byte)'\t');
i++;
}
else if (inputBuffer[i + 1] == '#')
{
fieldBytes.Add((byte)'#');
i++;
}
else
{
throw new Exception($"Expected 'n', 't', '#', or '\\' after '\\' at line {line} column {i - currentLineStart}");
}
}
else if (inputBuffer[i] == '\t')
{
// end of field
fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
}
else if (inputBuffer[i] == '\n')
{
fields.Add(fieldBytes.ToArray());
fieldBytes.Clear();
if (numFields < 0)
{
// This is the header
numFields = fields.Count;
columnNames = new string[numFields];
for (int j = 0; j < fields.Count; j++)
{
string columnString;
try
{
columnString = Encoding.UTF8.GetString(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Column name {fields.Count} is not valid UTF-8", e);
}
if (columnString.Contains(':'))
{
throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
}
columnNames[j] = columnString;
}
fields.Clear();
}
else if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
}
else
{
var fieldStrings = new string[fields.Count];
for (int j = 0; j < fields.Count; j++)
{
try
{
fieldStrings[j] = Encoding.UTF8.GetString(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Line {line}, column {j} is not valid UTF-8", e);
}
}
records.Add(fieldStrings);
fields.Clear();
}
line++;
currentLineStart = i + 1;
}
else if (inputBuffer[i] == '#')
{
throw new Exception($"Found unescaped '#' at line {line}, column {i - currentLineStart}");
}
else
{
fieldBytes.Add(inputBuffer[i]);
}
}
fields.Add(fieldBytes.ToArray());
if (numFields == 0)
{
throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
}
if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
}
else
{
var fieldStrings = new string[fields.Count];
for (int j = 0; j < fields.Count; j++)
{
try
{
fieldStrings[j] = Encoding.UTF8.GetString(fields[j]);
}
catch (Exception e)
{
throw new Exception($"Line {line}, column {j} is not valid UTF-8", e);
}
}
records.Add(fieldStrings);
fields.Clear();
}
return (columnNames, records.ToArray());
}
public static (string[] columns, string[][] data) ParseSimpleTsvParallel(byte[] inputBuffer)
{
string[] columnNames = null;
var headers = new List<byte[]>();
var fieldBytes = new List<byte>(); var fieldBytes = new List<byte>();
int startOfData = -1; int startOfData = -1;
for (int i = 0; i < inputBuffer.Count(); i++) for (int i = 0; i < inputBuffer.Count(); i++)
@ -882,33 +977,33 @@ public class SaneTsv
else if (inputBuffer[i] == '\t') else if (inputBuffer[i] == '\t')
{ {
// end of field // end of field
headerFields.Add(fieldBytes.ToArray()); headers.Add(fieldBytes.ToArray());
fieldBytes.Clear(); fieldBytes.Clear();
} }
else if (inputBuffer[i] == '\n') else if (inputBuffer[i] == '\n')
{ {
// This is the end of the header // This is the end of the header
headerFields.Add(fieldBytes.ToArray()); headers.Add(fieldBytes.ToArray());
startOfData = i + 1; startOfData = i + 1;
columnNames = new string[headerFields.Count]; columnNames = new string[headers.Count];
fieldBytes.Clear(); fieldBytes.Clear();
for (int j = 0; j < headerFields.Count; j++) for (int j = 0; j < headers.Count; j++)
{ {
string columnString; string columnString;
try try
{ {
columnString = Encoding.UTF8.GetString(headerFields[j]); columnString = Encoding.UTF8.GetString(headers[j]);
} }
catch (Exception e) catch (Exception e)
{ {
throw new Exception($"Column {headerFields.Count} name is not valid UTF-8", e); throw new Exception($"Column {headers.Count} name is not valid UTF-8", e);
} }
if (columnString.Contains(':')) if (columnString.Contains(':'))
{ {
throw new Exception($"Header field {headerFields.Count} contain ':', which is not allowed for column names"); throw new Exception($"Header {headers.Count} contain ':', which is not allowed for column names");
} }
columnNames[j] = columnString; columnNames[j] = columnString;
@ -927,16 +1022,16 @@ public class SaneTsv
} }
} }
return (columnNames, ParseSimpleTsv(inputBuffer, columnNames.Length, startOfData, inputBuffer.Length)); return (columnNames, ParseSimpleTsvParallel(inputBuffer, columnNames.Length, startOfData, inputBuffer.Length));
} }
public static string[][] ParseSimpleTsv(byte[] inputBuffer, int numFields, int startIndex, int endIndex) public static string[][] ParseSimpleTsvParallel(byte[] inputBuffer, int numFields, int startIndex, int endIndex)
{ {
var fieldBytes = new List<byte>(); var fieldBytes = new List<byte>();
var fields = new List<byte[]>(); var fields = new List<byte[]>();
var records = new List<string[]>(); var records = new List<string[]>();
int line = 2; int line = 1;
int currentLineStart = 0; int currentLineStart = 0;
// Go back to the start of the current line // Go back to the start of the current line
@ -1030,22 +1125,15 @@ public class SaneTsv
fields.Add(fieldBytes.ToArray()); fields.Add(fieldBytes.ToArray());
if (fields.Count == 0 && endIndex == inputBuffer.Length) if (fields.Count == 0)
{ {
throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record"); throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
} }
if (numFields != fields.Count) if (numFields != fields.Count)
{
if (endIndex == inputBuffer.Length)
{ {
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}"); throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
} }
else else
{
return records.ToArray();
}
}
else
{ {
var fieldStrings = new string[fields.Count]; var fieldStrings = new string[fields.Count];
for (int j = 0; j < fields.Count; j++) for (int j = 0; j < fields.Count; j++)
@ -1177,16 +1265,15 @@ public class SaneTsv
return SerializeTsv<T>(data, FormatType.COMMENTED_TSV); return SerializeTsv<T>(data, FormatType.COMMENTED_TSV);
} }
protected static byte[] SerializeTsv<T>(IList<T> data, FormatType tsvFormat) protected static byte[] SerializeTsv<T>(IList<T> data, FormatType tsvFormat) where T : TsvRecord
{ {
var bytes = new List<byte>(); var bytes = new List<byte>();
var columnTypes = new List<Type>(); var headerTypes = new List<Type>();
var columnNames = new List<string>(); var headerNames = new List<string>();
var columnPropertyInfos = new List<PropertyInfo>(); var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0; int columnCount = 0;
// Serialize header
foreach (PropertyInfo property in typeof(T).GetProperties()) foreach (PropertyInfo property in typeof(T).GetProperties())
{ {
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute)); TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
@ -1196,30 +1283,30 @@ public class SaneTsv
} }
string headerName = attribute.ColumnName ?? property.Name; string headerName = attribute.ColumnName ?? property.Name;
columnNames.Add(headerName); headerNames.Add(headerName);
Type headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType); Type headerType = attribute.ColumnType ?? GetColumnFromType(property.PropertyType);
if (tsvFormat == FormatType.SIMPLE_TSV && headerType != typeof(StringType)) if (tsvFormat == FormatType.SIMPLE_TSV && headerType != typeof(StringType))
{ {
throw new Exception($"Serializing Simple TSV requires all columns be of type string, but column '{headerName}' has type '{headerType}'"); throw new Exception($"Serializing Simple TSV requires all columns be of type string, but column '{headerName}' has type '{headerType}'");
} }
columnTypes.Add(headerType); headerTypes.Add(headerType);
columnPropertyInfos.Add(property); headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible // TODO: Check that the property type and given column type are compatible
columnCount++; columnCount++;
} }
// Serialize header // Serialize header
for (int i = 0; i < columnNames.Count; i++) for (int i = 0; i < headerNames.Count; i++)
{ {
for (int j = i + 1; j < columnNames.Count; j++) for (int j = i + 1; j < headerNames.Count; j++)
{ {
if (columnNames[i] == columnNames[j]) if (headerNames[i] == headerNames[j])
{ {
throw new Exception("Column names in header must be unique"); throw new Exception("Column names in header must be unique");
} }
} }
byte[] nameEncoded = Encoding.UTF8.GetBytes(columnNames[i]); byte[] nameEncoded = Encoding.UTF8.GetBytes(headerNames[i]);
for (int j = 0; j < nameEncoded.Length; j++) for (int j = 0; j < nameEncoded.Length; j++)
{ {
@ -1254,15 +1341,15 @@ public class SaneTsv
bytes.Add((byte)':'); bytes.Add((byte)':');
try try
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(columnTypes[i]))); bytes.AddRange(Encoding.UTF8.GetBytes(GetNameFromColumn(headerTypes[i])));
} }
catch (Exception e) catch (Exception e)
{ {
throw new Exception($"Invalid column type for column {i}", e); throw new Exception($"Invalid header type for column {i}", e);
} }
} }
if (i == columnNames.Count - 1) if (i == headerNames.Count - 1)
{ {
bytes.Add((byte)'\n'); bytes.Add((byte)'\n');
} }
@ -1272,20 +1359,12 @@ public class SaneTsv
} }
} }
// Serialize data
SerializeTsv<T>(data, bytes, columnPropertyInfos.ToArray(), columnTypes.ToArray(), tsvFormat, 0, data.Count);
return bytes.ToArray();
}
protected static void SerializeTsv<T>(IList<T> data, List<byte> bytes, PropertyInfo[] columnPropertyInfos, Type[] columnTypes, FormatType tsvFormat, int startIndex, int endIndex)
{
// Serialize data // Serialize data
for (int i = 0; i < data.Count; i++) for (int i = 0; i < data.Count; i++)
{ {
for (int j = 0; j < columnTypes.Length; j++) for (int j = 0; j < columnCount; j++)
{ {
object datum = columnPropertyInfos[j].GetValue(data[i]); object datum = headerPropertyInfos[j].GetValue(data[i]);
try try
{ {
@ -1293,16 +1372,16 @@ public class SaneTsv
// Some fields definitely don't need escaping, so we add them directly to bytes // Some fields definitely don't need escaping, so we add them directly to bytes
bool skipEscaping = false; bool skipEscaping = false;
if (columnTypes[j] == typeof(StringType)) if (headerTypes[j] == typeof(StringType))
{ {
fieldEncoded = Encoding.UTF8.GetBytes((string)datum); fieldEncoded = Encoding.UTF8.GetBytes((string)datum);
} }
else if (columnTypes[j] == typeof(BooleanType)) else if (headerTypes[j] == typeof(BooleanType))
{ {
bytes.AddRange((bool)datum ? TrueEncoded : FalseEncoded); bytes.AddRange((bool)datum ? TrueEncoded : FalseEncoded);
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(Float32Type)) else if (headerTypes[j] == typeof(Float32Type))
{ {
if (datum is float f) if (datum is float f)
{ {
@ -1326,7 +1405,7 @@ public class SaneTsv
} }
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(Float32LEType)) else if (headerTypes[j] == typeof(Float32LEType))
{ {
if (LittleEndian) if (LittleEndian)
{ {
@ -1342,7 +1421,7 @@ public class SaneTsv
} }
} }
} }
else if (columnTypes[j] == typeof(Float64Type)) else if (headerTypes[j] == typeof(Float64Type))
{ {
if (datum is double d) if (datum is double d)
{ {
@ -1366,7 +1445,7 @@ public class SaneTsv
} }
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(Float64LEType)) else if (headerTypes[j] == typeof(Float64LEType))
{ {
if (LittleEndian) if (LittleEndian)
{ {
@ -1382,33 +1461,33 @@ public class SaneTsv
} }
} }
} }
else if (columnTypes[j] == typeof(UInt32Type)) else if (headerTypes[j] == typeof(UInt32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)datum).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt32)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(UInt64Type)) else if (headerTypes[j] == typeof(UInt64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)datum).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((UInt64)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(Int32Type)) else if (headerTypes[j] == typeof(Int32Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)datum).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int32)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(Int64Type)) else if (headerTypes[j] == typeof(Int64Type))
{ {
bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)datum).ToString())); bytes.AddRange(Encoding.UTF8.GetBytes(((Int64)datum).ToString()));
skipEscaping = true; skipEscaping = true;
} }
else if (columnTypes[j] == typeof(BinaryType)) else if (headerTypes[j] == typeof(BinaryType))
{ {
fieldEncoded = (byte[])datum; fieldEncoded = (byte[])datum;
} }
else else
{ {
throw new Exception($"Unexpected column type {columnTypes[j]} for column {j}"); throw new Exception($"Unexpected column type {headerTypes[j]} for column {j}");
} }
if (!skipEscaping) if (!skipEscaping)
@ -1442,7 +1521,7 @@ public class SaneTsv
} }
} }
if (j < columnTypes.Length - 1) if (j < columnCount - 1)
{ {
bytes.Add((byte)'\t'); bytes.Add((byte)'\t');
} }
@ -1453,10 +1532,12 @@ public class SaneTsv
} }
catch (InvalidCastException e) catch (InvalidCastException e)
{ {
throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(columnTypes[j])}", e); throw new Exception($"Record {i}, field {j} expected type compatible with {GetNameFromColumn(headerTypes[j])}", e);
} }
} }
} }
return bytes.ToArray();
} }
public class SimpleTsvRecord public class SimpleTsvRecord

View File

@ -1,8 +1,7 @@
using NathanMcRae; using NathanMcRae;
using System.Reflection;
using System.Text; using System.Text;
internal class Program : SaneTsv internal class Program
{ {
public class TestRecord : SaneTsv.TsvRecord public class TestRecord : SaneTsv.TsvRecord
{ {
@ -81,18 +80,6 @@ internal class Program : SaneTsv
public string Column3 { get; set; } public string Column3 { get; set; }
} }
public class BoolTestRecord3 : SaneTsv.CommentedTsvRecord
{
[SaneTsv.TsvColumn("column1")]
public string Column1 { get; set; }
[SaneTsv.TsvColumn]
public string column2 { get; set; }
[SaneTsv.TsvColumn("columnthree\nyep")]
public string Column3 { get; set; }
}
public class SerdeTestRecord : SaneTsv.CommentedTsvRecord public class SerdeTestRecord : SaneTsv.CommentedTsvRecord
{ {
[SaneTsv.TypedTsvColumn("column1")] [SaneTsv.TypedTsvColumn("column1")]
@ -362,26 +349,45 @@ internal class Program : SaneTsv
} }
{ {
string testName = "With and without file comment"; string testName = "Check parallel serialization";
string testString1 = "#This is a file comment\n" + int N = 100000;
"#One more file comment line\n" + var records = new StringTestRecord[N];
"column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + var rand = new Random(1);
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
string testString2 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + for (int i = 0; i < N; i++)
"\n#This is a comment" + {
"\n#Another comment line" + records[i] = new StringTestRecord()
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" + {
"\nFALSE\tnother\tno\\ther"; Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
}
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1)); string[][] recordStrings = records.Select(record => new string[] { record.Column1, record.column2, record.Column3 }).ToArray();
CommentedTsv<BoolTestRecord2> parsed2 = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2));
if (parsed.FileComment == "This is a file comment\nOne more file comment line" && parsed2.FileComment == null) DateTime lastTime = DateTime.Now;
byte[] serialized1 = SaneTsv.SerializeSimpleTsv(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan unparallelTime = DateTime.Now - lastTime;
lastTime = DateTime.Now;
byte[] serialized2 = SaneTsv.SerializeSimpleTsvParallel(new string[] { "column1", "column2", "columnthree\nyep" }, recordStrings);
TimeSpan parallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
Console.WriteLine($"Parallel serialization time: {parallelTime}");
bool matching = true;
for (int i = 0; i < Math.Min(serialized1.Length, serialized2.Length); i++)
{
if (serialized1[i] != serialized2[i])
{
matching = false;
break;
}
}
if (matching)
{ {
Console.WriteLine($"Passed {testName}"); Console.WriteLine($"Passed {testName}");
} }
@ -392,370 +398,63 @@ internal class Program : SaneTsv
} }
{ {
string testName = "With and without types"; string testName = "Check parallel parsing";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" + int N = 100000;
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" + var records = new StringTestRecord[N];
"\nFALSE\tnother\tno\\ther"; var rand = new Random(1);
try for (int i = 0; i < N; i++)
{ {
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1)); records[i] = new StringTestRecord()
Console.WriteLine($"Passed {testName} 1A");
}
catch (Exception e)
{ {
Console.WriteLine($"Failed {testName} 1A"); Column1 = rand.Next().ToString(),
column2 = rand.Next().ToString(),
Column3 = rand.Next().ToString(),
};
} }
try byte[] serialized = SaneTsv.SerializeSimpleTsv<StringTestRecord>(records);
{
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
try DateTime lastTime = DateTime.Now;
{ (string[] headers2, string[][] data2) = SaneTsv.ParseSimpleTsv(serialized);
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1)); TimeSpan unparallelTime = DateTime.Now - lastTime;
Console.WriteLine($"Failed {testName} 1C"); lastTime = DateTime.Now;
} (string[] headers, string[][] data) = SaneTsv.ParseSimpleTsvParallel(serialized);
catch (Exception e) TimeSpan parallelTime = DateTime.Now - lastTime;
{
Console.WriteLine($"Passed {testName} 1C");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" + Console.WriteLine($"Unparallel serialization time: {unparallelTime}");
"\nTRUE\tvalue\\\\twoo\tvaluetrhee" + Console.WriteLine($"Parallel serialization time: {parallelTime}");
"\nFALSE\tnother\tno\\ther";
try bool matching = true;
for (int j = 0; j < Math.Min(headers2.Length, headers.Length); j++)
{ {
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString2)); if (headers[j] != headers2[j])
Console.WriteLine($"Failed {testName} 2A");
}
catch (Exception e)
{ {
Console.WriteLine($"Passed {testName} 2A"); matching = false;
} break;
try
{
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 2B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 2B");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 2C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 2C");
} }
} }
for (int i = 0; i < Math.Min(data.Length, data2.Length) && matching; i++)
{ {
string testName = "With and without line comment"; for (int j = 0; j < data[0].Length; j++)
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\n#This is a comment" +
"\n#Another comment line" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
try
{ {
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1)); if (data[i][j] != data2[i][j])
Console.WriteLine($"Passed {testName} 1A"); {
matching = false;
break;
} }
catch (Exception e)
{
Console.WriteLine($"Failed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
try
{
Tsv<BoolTestRecord2> parsed2 = SaneTsv.ParseSimpleTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
} }
} }
if (matching)
{ {
string testName = "End of file comment"; Console.WriteLine($"Passed {testName}");
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n# Hey, you're not supposed to have comments at the end of the tsv!";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
}
}
{
string testName = "Partial parsing";
string line1 = "column1\tcolumn2\tcolumnthree\\nyep";
string line2 = "\nTRUE\tvalue\\\\t\0woo\tvaluetrhee";
string line3 = "\nFALSE\tnother\tno\\ther";
byte[] inputBuffer = Encoding.UTF8.GetBytes(line1 + line2 + line3);
var headerTypes = new List<Type>();
var headerNames = new List<string>();
var headerPropertyInfos = new List<PropertyInfo>();
int columnCount = 0;
foreach (PropertyInfo property in typeof(BoolTestRecord3).GetProperties())
{
TsvColumnAttribute attribute = (TsvColumnAttribute)Attribute.GetCustomAttribute(property, typeof(TsvColumnAttribute));
if (attribute == null)
{
continue;
}
headerNames.Add(attribute.ColumnName ?? property.Name);
headerTypes.Add(attribute.ColumnType ?? GetColumnFromType(property.PropertyType));
headerPropertyInfos.Add(property);
// TODO: Check that the property type and given column type are compatible
columnCount++;
}
BoolTestRecord3[] records = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
FormatType.SIMPLE_TSV,
headerPropertyInfos.ToArray(),
headerTypes.ToArray(),
line1.Length + line2.Length + 1,
inputBuffer.Length);
if (records.Length == 0 )
{
Console.WriteLine($"Passed {testName} 1");
} }
else else
{ {
Console.WriteLine($"Failed {testName} 1"); Console.WriteLine($"Failed {testName}");
}
BoolTestRecord3[] records2 = SaneTsv.Parse<BoolTestRecord3>(inputBuffer,
FormatType.SIMPLE_TSV,
headerPropertyInfos.ToArray(),
headerTypes.ToArray(),
line1.Length,
line1.Length + 3);
if (records2[0].Column3 == "valuetrhee")
{
Console.WriteLine($"Passed {testName} 2");
}
else
{
Console.WriteLine($"Failed {testName} 2");
}
string[][] data = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length + line2.Length + 1, inputBuffer.Length);
if (data[0][1] == "nother")
{
Console.WriteLine($"Passed {testName} 3");
}
else
{
Console.WriteLine($"Failed {testName} 3");
}
string[][] data2 = SaneTsv.ParseSimpleTsv(inputBuffer, 3, line1.Length, line1.Length + 3);
if (data2.Length == 0)
{
Console.WriteLine($"Passed {testName} 4");
}
else
{
Console.WriteLine($"Failed {testName} 4");
}
}
{
string testName = "End of file \\n";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\n";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
}
}
{
string testName = "End of file partial record";
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\nTRUE\t";
try
{
CommentedTsv<BoolTestRecord2> parsed = SaneTsv.ParseCommentedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1A");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1A");
}
try
{
Tsv<BoolTestRecord2> parsed = SaneTsv.ParseTypedTsv<BoolTestRecord2>(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName} 1B");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1B");
}
string testString2 = "column1\tcolumn2\tcolumnthree\\nyep" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther" +
"\nTRUE\t";
try
{
Tsv<BoolTestRecord3> parsed3 = SaneTsv.ParseSimpleTsv<BoolTestRecord3>(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1C");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1C");
}
try
{
(string[] columns, string[][] data) = SaneTsv.ParseSimpleTsv(Encoding.UTF8.GetBytes(testString2));
Console.WriteLine($"Failed {testName} 1D");
}
catch (Exception e)
{
Console.WriteLine($"Passed {testName} 1D");
} }
} }