extra-tsv/SaneTsv.cs

using System.Text;

namespace NathanMcRae;

/// <summary>
/// Sane Tab-Separated Values
/// </summary>
public class SaneTsv
{
  // Like an enum, but more extensible
  public class ColumnType { }
  public class StringType : ColumnType { }
  public class BooleanType : ColumnType { }
  public class Float32Type : ColumnType { }
  public class Float64Type : ColumnType { }
  public class UInt32Type : ColumnType { }
  public class UInt64Type : ColumnType { }
  public class Int32Type : ColumnType { }
  public class Int64Type : ColumnType { }
  public class BinaryType : ColumnType { }

  protected enum FormatType
  {
    SIMPLE_TSV = 0,
    TYPED_TSV = 1,
    COMMENTED_TSV = 2,
  }

  // TODO: We need to be able to update all these in tandem somehow
  public string[] ColumnNames { get; protected set; }
  public Type[] ColumnTypes { get; protected set; }
  public List<SaneTsvRecord> Records { get; protected set; }
  public string FileComment { get; protected set; } = null;

  public static SaneTsv ParseSimpleTsv(byte[] inputBuffer)
  {
    return Parse(inputBuffer, FormatType.SIMPLE_TSV);
  }

  public static SaneTsv ParseTypedTsv(byte[] inputBuffer)
  {
    return Parse(inputBuffer, FormatType.TYPED_TSV);
  }

  public static SaneTsv ParseCommentedTsv(byte[] inputBuffer)
  {
    return Parse(inputBuffer, FormatType.COMMENTED_TSV);
  }

  // TODO: Have parsing errors include line / column #
  protected static SaneTsv Parse(byte[] inputBuffer, FormatType format)
  {
    var parsed = new SaneTsv();
    parsed.ColumnNames = new string[] { };
    parsed.ColumnTypes = new Type[] { };
    parsed.Records = new List<SaneTsvRecord>();

    var fieldBytes = new List<byte>();
    var fields = new List<byte[]>();
    var currentComment = new StringBuilder();

    int numFields = -1;
    int line = 1;
    int currentLineStart = 0;
    for (int i = 0; i < inputBuffer.Count(); i++)
    {
      if (inputBuffer[i] == '\\')
      {
        if (i + 1 == inputBuffer.Count())
        {
          throw new Exception($"Found '\\' at end of input");
        }
        if (inputBuffer[i + 1] == 'n')
        {
          fieldBytes.Add((byte)'\n');
          i++;
        }
        else if (inputBuffer[i + 1] == '\\')
        {
          fieldBytes.Add((byte)'\\');
          i++;
        }
        else if (inputBuffer[i + 1] == 't')
        {
          fieldBytes.Add((byte)'\t');
          i++;
        }
        else if (inputBuffer[i + 1] == '#')
        {
          fieldBytes.Add((byte)'#');
          i++;
        }
        else
        {
          throw new Exception($"Expected 'n', 't', '#', or '\\' after '\\' at line {line} column {i - currentLineStart}");
        }
      }
      else if (inputBuffer[i] == '\t')
      {
        // end of field
        fields.Add(fieldBytes.ToArray());
        fieldBytes.Clear();
      }
      else if (inputBuffer[i] == '\n')
      {
        fields.Add(fieldBytes.ToArray());
        fieldBytes.Clear();

        if (numFields < 0)
        {
          // This is the header

          numFields = fields.Count;

          parsed.ColumnNames = new string[numFields];
          parsed.ColumnTypes = new Type[numFields];

          int numTypesBlank = 0;

          for (int j = 0; j < fields.Count; j++)
          {
            string columnString;
            try
            {
              columnString = Encoding.UTF8.GetString(fields[j]);
            }
            catch (Exception e)
            {
              throw new Exception($"Header {fields.Count} is not valid UTF-8", e);
            }

            string columnTypeString;
            string columnName;
            if (columnString.Contains(':')) {
              if (format == FormatType.SIMPLE_TSV)
              {
                throw new Exception($"Header {fields.Count} contain ':', which is not allowed for column names");
              }
              columnTypeString = columnString.Split(":").Last();
              columnName = columnString.Substring(0, columnString.Length - columnTypeString.Length - 1);
            }
            else
            {
              if (format > FormatType.SIMPLE_TSV)
              {
                throw new Exception($"Header {fields.Count} has no type");
              }
              columnTypeString = "";
              columnName = columnString;
            }

            Type type;

            switch (columnTypeString)
            {
              case "":
                numTypesBlank++;
                type = typeof(StringType);
                break;
              case "string":
                type = typeof(StringType);
                break;
              case "boolean":
                type = typeof(BooleanType);
                break;
              case "float32":
                type = typeof(Float32Type);
                break;
              case "float64":
                type = typeof(Float64Type);
                break;
              case "uint32":
                type = typeof(UInt32Type);
                break;
              case "uint64":
                type = typeof(UInt64Type);
                break;
              case "int32":
                type = typeof(Int32Type);
                break;
              case "int64":
                type = typeof(Int64Type);
                break;
              case "binary":
                type = typeof(BinaryType);
                break;
              default:
                throw new Exception($"Invalid type '{columnTypeString}' for column {j}");
            }

            // TODO: Check column name uniqueness

            parsed.ColumnNames[j] = columnName;
            parsed.ColumnTypes[j] = type;
          }

          if (currentComment.Length > 0)
          {
            parsed.FileComment = currentComment.ToString();
            currentComment.Clear();
          }

          fields.Clear();
        }
        else if (numFields != fields.Count)
        {
          throw new Exception($"Expected {numFields} fields on line {line}, but found {fields.Count}");
        }
        else
        {
          string comment = null;
          if (currentComment.Length > 0)
          {
            comment = currentComment.ToString();
            currentComment.Clear();
          }
          parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line));
          fields.Clear();
        }

        line++;
        currentLineStart = i + 1;
      }
      else if (inputBuffer[i] == '#')
      {
        if (i == currentLineStart && format >= FormatType.COMMENTED_TSV)
        {
          int j = i;
          for (; j < inputBuffer.Length && inputBuffer[j] != '\n'; j++) { }
          if (j < inputBuffer.Length)
          {
            var commentBytes = new byte[j - i - 1];
            Array.Copy(inputBuffer, i + 1, commentBytes, 0, j - i - 1);
            if (currentComment.Length > 0)
            {
              currentComment.Append('\n');
            }
            currentComment.Append(Encoding.UTF8.GetString(commentBytes));
            i = j;
            currentLineStart = i + 1;
            line++;
          }
          else
          {
            throw new Exception("Comments at end of file are not allowed");
          }
        }
        else
        {
          throw new Exception($"Found unescaped '#' at line {line}, column {i - currentLineStart}");
        }
      }
      else
      {
        fieldBytes.Add(inputBuffer[i]);
      }
    }

    fields.Add(fieldBytes.ToArray());

    if (numFields == 0)
    {
      throw new Exception("Found 0 fields on last line. Possibly because of extra \\n after last record");
    }
    if (numFields != fields.Count)
    {
      throw new Exception($"Expected {numFields} fields on line {parsed.Records.Count + 2}, but found {fields.Count}");
    }
    else
    {
      string comment = null;
      if (currentComment.Length > 0)
      {
        comment = currentComment.ToString();
        currentComment.Clear();
      }
      parsed.Records.Add(new SaneTsvRecord(parsed, ParseCurrentRecord(parsed, fields, line), comment, line));
      fields.Clear();
    }

    return parsed;
  }

  /// <summary>
  /// Note: this modifies 'parsed'
  /// </summary>
  protected static object[] ParseCurrentRecord(SaneTsv parsed, List<byte[]> fields, int line)
  {
    var parsedFields = new object[fields.Count];
    for (int j = 0; j < fields.Count; j++)
    {
      // All other types require the content to be UTF-8. Binary fields can ignore that.
      if (parsed.ColumnTypes[j] == typeof(BinaryType))
      {
        parsedFields[j] = fields[j];
        continue;
      }

      string fieldString;
      try
      {
        fieldString = Encoding.UTF8.GetString(fields[j]);
      }
      catch (Exception e)
      {
        throw new Exception($"Field {j} on line {line} is not valid UTF-8", e);
      }

      // TODO: Add checking for numeric types format

      if (parsed.ColumnTypes[j] == typeof(StringType))
      {
        parsedFields[j] = fieldString;
      }
      else if (parsed.ColumnTypes[j] == typeof(BooleanType))
      {
        bool parsedBool;
        if (fieldString == "TRUE")
        {
          parsedBool = true;
        }
        else if (fieldString == "FALSE")
        {
          parsedBool = false;
        }
        else
        {
          throw new Exception($"Field {j} on line {line} is not valid boolean. Must be 'TRUE' or 'FALSE' exactly");
        }

        parsedFields[j] = parsedBool;
      }
      else if (parsed.ColumnTypes[j] == typeof(Float32Type))
      {
        if (!float.TryParse(fieldString, out float parsedFloat))
        {
          throw new Exception($"Field {j} on line {line} is not valid single-precision float");
        }

        parsedFields[j] = parsedFloat;
      }
      else if (parsed.ColumnTypes[j] == typeof(Float64Type))
      {
        if (!double.TryParse(fieldString, out double parsedDouble))
        {
          throw new Exception($"Field {j} on line {line} is not valid double-precision float");
        }

        parsedFields[j] = parsedDouble;
      }
      else if (parsed.ColumnTypes[j] == typeof(UInt32Type))
      {
        if (!UInt32.TryParse(fieldString, out UInt32 parsedUInt32))
        {
          throw new Exception($"Field {j} on line {line} is not valid UInt32");
        }

        parsedFields[j] = parsedUInt32;
      }
      else if (parsed.ColumnTypes[j] == typeof(UInt64Type))
      {
        if (!UInt64.TryParse(fieldString, out UInt64 parsedUInt64))
        {
          throw new Exception($"Field {j} on line {line} is not valid UInt64");
        }

        parsedFields[j] = parsedUInt64;
      }
      else if (parsed.ColumnTypes[j] == typeof(Int32Type))
      {
        if (!Int32.TryParse(fieldString, out Int32 parsedInt32))
        {
          throw new Exception($"Field {j} on line {line} is not valid Int32");
        }

        parsedFields[j] = parsedInt32;
      }
      else if (parsed.ColumnTypes[j] == typeof(Int64Type))
      {
        if (!Int64.TryParse(fieldString, out Int64 parsedInt64))
        {
          throw new Exception($"Field {j} on line {line} is not valid Int64");
        }

        parsedFields[j] = parsedInt64;
      }
      else
      {
        throw new Exception($"Unexpected type {parsed.ColumnTypes[j]}");
      }
    }

    return parsedFields;
  }

  public static byte[] SerializeSimpleTsv(IList<string> header, IList<IList<string>> data)
  {
    var escapedString = new StringBuilder();

    // Serialize header
    for (int i = 0; i < header.Count; i++)
    {
      if (header[i].Contains(':'))
      {
        throw new Exception($"Column {i} contains the character ':'");
      }

      for (int j = i + 1; j < header.Count; j++)
      {
        if (header[i] == header[j])
        {
          throw new Exception("Column names in header must be unique");
        }
      }

      for (int j = 0; j < header[i].Count(); j++)
      {
        if (header[i][j] == '\n')
        {
          escapedString.Append("\\n");
        }
        else if (header[i][j] == '\t')
        {
          escapedString.Append("\\t");
        }
        else if (header[i][j] == '\\')
        {
          escapedString.Append("\\\\");
        }
        else if (header[i][j] == '#')
        {
          escapedString.Append("\\#");
        }
        else
        {
          escapedString.Append(header[i][j]);
        }
      }

      if (i == header.Count - 1)
      {
        escapedString.Append('\n');
      }
      else
      {
        escapedString.Append('\t');
      }
    }

    // Serialize data
    for (int i = 0; i < data.Count; i++)
    { 
      for (int j = 0; j < data[i].Count; j++)
      {
        for (int k = 0; k < data[i][j].Length; k++)
        {
          if (data[i][j][k] == '\n')
          {
            escapedString.Append("\\n");
          }
          else if (data[i][j][k] == '\t')
          {
            escapedString.Append("\\t");
          }
          else if (data[i][j][k] == '\\')
          {
            escapedString.Append("\\\\");
          }
          else if (data[i][j][k] == '#')
          {
            escapedString.Append("\\#");
          }
          else
          {
            escapedString.Append(data[i][j][k]);
          }
        }

        if (j < data[i].Count - 1)
        {
          escapedString.Append('\t');
        }
        else if (i < data.Count - 1)
        {
          escapedString.Append('\n');
        }
      }
    }

    return Encoding.UTF8.GetBytes(escapedString.ToString());
  }

  public SaneTsvRecord this[int i] => Records[i];

  public class SaneTsvRecord
  {
    public SaneTsv Parent { get; }
    public string Comment { get; }
    public object[] Fields { get; }
    public int Line { get; }

    public object this[string columnName] => Fields[Array.IndexOf(Parent.ColumnNames, columnName)];

    public object this[int columnIndex] => Fields[columnIndex];

    public SaneTsvRecord(SaneTsv parent, object[] fields, string comment, int line)
    {
      Parent = parent;
      Fields = fields;
      Comment = comment;
      Line = line;
    }
  }
}