nathanmcrae/sane-tsv
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: nathanmcrae:6cea9b7e596e489740f703a029e02ded6bc7d023
Branches Tags
nathanmcrae:master
..
compare: nathanmcrae:ab72d875bf233e2e42e12dc428274c33e03cd67d
Branches Tags
nathanmcrae:master

No commits in common. "6cea9b7e596e489740f703a029e02ded6bc7d023" and "ab72d875bf233e2e42e12dc428274c33e03cd67d" have entirely different histories.
6cea9b7e59 ... ab72d875bf

3 changed files with 23 additions and 167 deletions
Show Stats Expand all files Collapse all files

124
SaneTsv/SaneTsv.cs
View File

@ -20,37 +20,13 @@ public class SaneTsv
BINARY,
}
protected enum FormatType
{
SANE_TSV = 0,
TYPED_TSV = 1,
COMMENTED_TSV = 2,
}
// TODO: We need to be able to update all these in tandem somehow
public string[] ColumnNames { get; protected set; }
public ColumnType[] ColumnTypes { get; protected set; }
public Dictionary<string, List<object>> Columns { get; protected set; }
public List<SaneTsvRecord> Records { get; protected set; }
public string FileComment { get; protected set; } = null;
public static SaneTsv ParseSaneTsv(byte[] inputBuffer)
{
return Parse(inputBuffer, FormatType.SANE_TSV);
}
public static SaneTsv ParseTypedTsv(byte[] inputBuffer)
{
return Parse(inputBuffer, FormatType.TYPED_TSV);
}
public static SaneTsv ParseCommentedTsv(byte[] inputBuffer)
{
return Parse(inputBuffer, FormatType.COMMENTED_TSV);
}
// TODO: Have parsing errors include line / column #
protected static SaneTsv Parse(byte[] inputBuffer, FormatType format)
public static SaneTsv Parse(byte[] inputBuffer)
{
var parsed = new SaneTsv();
parsed.Columns = new Dictionary<string, List<object>>();
@ -60,11 +36,7 @@ public class SaneTsv
var fieldBytes = new List<byte>();
var fields = new List<byte[]>();
var currentComment = new StringBuilder();
int numFields = -1;
int line = 1;
int currentLineStart = 0;
for (int i = 0; i < inputBuffer.Count(); i++)
{
if (inputBuffer[i] == '\\')
@ -88,11 +60,6 @@ public class SaneTsv
fieldBytes.Add((byte)'\t');
i++;
}
else if (inputBuffer[i + 1] == '#')
{
fieldBytes.Add((byte)'#');
i++;
}
else
{
throw new Exception($"Expected 'n', 't', or '\\' after '\\' at {i}");
@ -134,20 +101,12 @@ public class SaneTsv
string columnTypeString;
string columnName;
if (columnString.Contains(':')) {
if (format == FormatType.SANE_TSV)
{
throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
}
if (columnString.Contains(":")) {
columnTypeString = columnString.Split(":").Last();
columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
}
else
{
if (format > FormatType.SANE_TSV)
{
throw new Exception($"Header {fields.Count} has no type");
}
columnTypeString = "";
columnName = columnString;
}
@ -204,57 +163,20 @@ public class SaneTsv
parsed.ColumnTypes[j] = type;
}
if (currentComment.Length > 0)
if (numTypesBlank != 0 && numTypesBlank != fields.Count)
{
parsed.FileComment = currentComment.ToString();
currentComment.Clear();
throw new Exception("Types must be provided for all columns or none. Use 'string' for columns missing types.");
}
fields.Clear();
}
else if (numFields != fields.Count)
{
throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
}
else
{
string comment = null;
if (currentComment.Length > 0)
{
comment = currentComment.ToString();
currentComment.Clear();
}
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment));
fields.Clear();
}
line++;
currentLineStart = i + 1;
}
else if (inputBuffer[i] == '#')
{
if (i == currentLineStart && format >= FormatType.COMMENTED_TSV)
{
int j = i;
for (; j < inputBuffer.Length && inputBuffer[j] != '\n'; j++) { }
if (j < inputBuffer.Length)
{
var commentBytes = new byte[j - i - 1];
Array.Copy(inputBuffer, i + 1, commentBytes, 0, j - i - 1);
currentComment.Append(Encoding.UTF8.GetString(commentBytes));
currentComment.Append("\n");
i = j;
currentLineStart = i + 1;
line++;
}
else
{
throw new Exception("Comments at end of file are not allowed");
}
}
else
{
throw new Exception($"Found unescaped '#' at line {line}, column {i - currentLineStart}");
AddRecord(parsed, fields);
}
}
else
@ -275,20 +197,13 @@ public class SaneTsv
}
else
{
string comment = null;
if (currentComment.Length > 0)
{
comment = currentComment.ToString();
currentComment.Clear();
}
parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment));
fields.Clear();
AddRecord(parsed, fields);
}
return parsed;
}
protected static object[] ParseCurrentRecord(SaneTsv parsed, List<byte[]> fields, int line)
protected static void AddRecord(SaneTsv parsed, List<byte[]> fields)
{
var parsedFields = new object[fields.Count];
for (int j = 0; j < fields.Count; j++)
@ -308,7 +223,7 @@ public class SaneTsv
}
catch (Exception e)
{
throw new Exception($"Field {j} on line {line} is not valid UTF-8", e);
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UTF-8", e);
}
switch (parsed.ColumnTypes[j])
@ -329,7 +244,7 @@ public class SaneTsv
}
else
{
throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
}
parsedFields[j] = parsedBool;
@ -338,7 +253,7 @@ public class SaneTsv
case ColumnType.FLOAT32:
if (!float.TryParse(fieldString, out float parsedFloat))
{
throw new Exception($"Field {j} on line {line} is not valid single-precision float");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid single-precision float");
}
parsedFields[j] = parsedFloat;
@ -347,7 +262,7 @@ public class SaneTsv
case ColumnType.FLOAT64:
if (!double.TryParse(fieldString, out double parsedDouble))
{
throw new Exception($"Field {j} on line {line} is not valid double-precision float");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid double-precision float");
}
parsedFields[j] = parsedDouble;
@ -356,7 +271,7 @@ public class SaneTsv
case ColumnType.UINT32:
if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
{
throw new Exception($"Field {j} on line {line} is not valid UInt32");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UInt32");
}
parsedFields[j] = parsedUInt32;
@ -365,7 +280,7 @@ public class SaneTsv
case ColumnType.UINT64:
if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
{
throw new Exception($"Field {j} on line {line} is not valid UInt64");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid UInt64");
}
parsedFields[j] = parsedUInt64;
@ -374,7 +289,7 @@ public class SaneTsv
case ColumnType.INT32:
if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
{
throw new Exception($"Field {j} on line {line} is not valid Int32");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid Int32");
}
parsedFields[j] = parsedInt32;
@ -383,7 +298,7 @@ public class SaneTsv
case ColumnType.INT64:
if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
{
throw new Exception($"Field {j} on line {line} is not valid Int64");
throw new Exception($"Field {j} on line {parsed.Records.Count + 2} is not valid Int64");
}
parsedFields[j] = parsedInt64;
@ -396,7 +311,8 @@ public class SaneTsv
}
}
return parsedFields;
parsed.Records.Add(new SaneTsvRecord(parsed, parsedFields));
fields.Clear();
}
public SaneTsvRecord this[int i] => Records[i];
@ -404,16 +320,14 @@ public class SaneTsv
public class SaneTsvRecord
{
public SaneTsv Parent { get; }
public string Comment { get; }
public object[] Fields { get; }
public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)];
public SaneTsvRecord(SaneTsv parent, object[] fields, string comment)
public SaneTsvRecord(SaneTsv parent, object[] fields)
{
Parent = parent;
Fields = fields;
Comment = comment;
}
}
}

8
SaneTsv/SaneTsvTest/Program.cs
View File

@ -3,12 +3,12 @@ using System.Text;
{
string testName = "Bool test";
string testString1 = "column1:ty\\#pe:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
string testString1 = "column1:type:boolean\tcolumn2:binary\tcolumnthree\\nyep:string" +
"\nTRUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseTypedTsv(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0]["column1:ty#pe"] is bool result && result)
SaneTsv parsed = SaneTsv.Parse(Encoding.UTF8.GetBytes(testString1));
if (parsed.Records[0]["column1:type"] is bool result && result)
{
Console.WriteLine($"Passed {testName}");
}
@ -26,7 +26,7 @@ using System.Text;
"\nTUE\tvalue\\\\t\0woo\tvaluetrhee" +
"\nFALSE\tnother\tno\\ther";
SaneTsv parsed = SaneTsv.ParseTypedTsv(Encoding.UTF8.GetBytes(testString1));
SaneTsv parsed = SaneTsv.Parse(Encoding.UTF8.GetBytes(testString1));
Console.WriteLine($"Failed {testName}");
}
catch (Exception)

58
SaneTsv/readme.md
View File

@ -1,58 +0,0 @@
# Sane TSV
Sane TSV is a strict format for tabular data.
'\n' (0x0A) character delimit lines, and '\t' (0x09) characters delimit fields within a line.
'\n' and '\t' characters are allowed within fields by escaping them with a backslash character (0x5C) followed by 'n' (0x6E) and 't' (0x74) respectively. Additionaly, '\\' and '#' (0x23) must also be escaped. The '#' character is escaped for compatility with [Commented TSVs](#commented-tsv).
All fields must be UTF-8 encoded text. All escaping can be done before decoding (and after encoding).
Empty fields (i.e. two subsequent '\t' characters) are allowed.
The first line is always the header and the fields of the header are the column names for the file. Column names must be unique within the file and must not contain ':' characters (for compatibility with [Typed TSVs](#typed-tsv)).
All lines in the file must have the same number of fields.
The file must not end with '\n'. That will be treated as if there is an empty row at the end of a file and cause an error.
Implementations of the format do not need to handle file reading and writing directly, but if they do, they should enforce usage of the file extension '.stsv'. They should also provide a manual override option so that other extensions may be forced.
# Typed TSV
Typed TSV allows for typing of columns. All column names in a typed TSV must end with ':' (0x3A) and then one of the following types:
- 'string'
- 'boolean'
- 'float32'
- 'float64'
- 'uint32'
- 'uint64'
- 'int32'
- 'int64'
- 'binary'
Any other values are an error, however, the portion of the name prior to the last ':' may be anything and may include ':' characters.
All fields in the rest of the file must be of the type corresponding the their column.
Aside from the 'binary' column type, all fields must be UTF-8 encoded text. Each type has the following restrictions:
- 'boolean' fields must contain only and exactly the text "TRUE" or "FALSE".
- 'float32' and 'float64' correspond to single and double precision IEEE 754 floating-point numbers respectively. They should be formatted like this regex: `-?[0-9]\.([0-9]|[0-9]+[1-9])E-?[1-9][0-9]*`
Both float types may additionally have these values:
- 'sNaN'
- 'qNaN'
- '+inf'
- '-inf'
- 'uint32' and 'uint64' are unsigned 32 and 64 bit integers respectively. They should be formatted like this regex: `[1-9][0-9]*`
- 'int32' and 'int64' are signed 32 and 64 bit integers respectively. They should be formatted like this regex: `-?[1-9][0-9]*` (except that '-0' is not allowed)
# Commented TSV
Commented lines start with a '#' character at the beginning of the line. Unescaped '#' characters are not allowed on a line that does not start with a '#'. Any '#' characters in fields must be escaped.
Comments must be UTF-8 encoded text.
Comments after the last record are an error.