From 51c17697d16f6658e71743d1a0ca5d3d4bf1489e Mon Sep 17 00:00:00 2001 From: Phill Date: Mon, 19 May 2025 18:13:27 -0400 Subject: [PATCH 1/9] separate end of line characters --- source/SharedFunctions.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/SharedFunctions.cs b/source/SharedFunctions.cs index 078d2e1..00f43f6 100644 --- a/source/SharedFunctions.cs +++ b/source/SharedFunctions.cs @@ -6,7 +6,12 @@ internal static class SharedFunctions public static bool IsWhitespace(char character) { - return character == ' ' || character == '\t' || character == '\n' || character == '\r' || character == BOM; + return IsEndOfLine(character) || character == ' ' || character == '\t' || character == BOM; + } + + public static bool IsEndOfLine(char character) + { + return character == '\n' || character == '\r'; } } } \ No newline at end of file From bfc35d29d248eff431f860fdbd5995168e9dd6ca Mon Sep 17 00:00:00 2001 From: Phill Date: Mon, 19 May 2025 18:14:12 -0400 Subject: [PATCH 2/9] basic toml lexer --- source/TOML/TOMLObject.cs | 28 +++++++ source/TOML/TOMLReader.cs | 152 ++++++++++++++++++++++++++++++++++++++ source/TOML/Token.cs | 87 ++++++++++++++++++++++ tests/TOMLTests.cs | 54 ++++++++++++++ 4 files changed, 321 insertions(+) create mode 100644 source/TOML/TOMLObject.cs create mode 100644 source/TOML/TOMLReader.cs create mode 100644 source/TOML/Token.cs create mode 100644 tests/TOMLTests.cs diff --git a/source/TOML/TOMLObject.cs b/source/TOML/TOMLObject.cs new file mode 100644 index 0000000..8661378 --- /dev/null +++ b/source/TOML/TOMLObject.cs @@ -0,0 +1,28 @@ +using System; +using System.Runtime.CompilerServices; +using Unmanaged; + +namespace Serialization.TOML +{ + [SkipLocalsInit] + public readonly struct TOMLObject : IDisposable, ISerializable + { + public readonly void Dispose() + { + } + + void ISerializable.Read(ByteReader byteReader) + { + + } + + readonly void ISerializable.Write(ByteWriter byteWriter) + { + } + } + + public readonly struct TOMLKeyValue + { + + } +} \ No newline at end of file diff --git a/source/TOML/TOMLReader.cs b/source/TOML/TOMLReader.cs new file mode 100644 index 0000000..5dcc2e0 --- /dev/null +++ b/source/TOML/TOMLReader.cs @@ -0,0 +1,152 @@ +using System; +using System.Runtime.CompilerServices; +using Unmanaged; + +namespace Serialization.TOML +{ + [SkipLocalsInit] + public ref struct TOMLReader + { + private ByteReader reader; + + public TOMLReader(ByteReader reader) + { + this.reader = reader; + } + + public readonly bool PeekToken(out Token token) + { + return PeekToken(out token, out _); + } + + public readonly bool PeekToken(out Token token, out int readBytes) + { + token = default; + int position = reader.Position; + int length = reader.Length; + while (position < length) + { + byte bytesRead = reader.PeekUTF8(position, out char c, out _); + if (c == '#') + { + token = new Token(position, bytesRead, Token.Type.Hash); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == '=') + { + token = new Token(position, bytesRead, Token.Type.Equals); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == ',') + { + token = new Token(position, bytesRead, Token.Type.Comma); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == '.') + { + token = new Token(position, bytesRead, Token.Type.Period); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == '[') + { + token = new Token(position, bytesRead, Token.Type.StartSquareBracket); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == ']') + { + token = new Token(position, bytesRead, Token.Type.EndSquareBracket); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == '{') + { + token = new Token(position, bytesRead, Token.Type.StartCurlyBrace); + readBytes = position - reader.Position + 1; + return true; + } + else if (c == '}') + { + token = new Token(position, bytesRead, Token.Type.EndCurlyBrace); + readBytes = position - reader.Position + 1; + return true; + } + else if (SharedFunctions.IsWhitespace(c)) + { + position += bytesRead; + } + else + { + int start = position; + position += bytesRead; + while (position < length) + { + bytesRead = reader.PeekUTF8(position, out c, out _); + if (SharedFunctions.IsEndOfLine(c) || Token.Tokens.Contains(c)) + { + token = new Token(start, position - start, Token.Type.Text); + readBytes = position - reader.Position; + return true; + } + + position += bytesRead; + } + + token = new Token(start, position - start, Token.Type.Text); + readBytes = position - reader.Position; + return true; + } + } + + readBytes = default; + return false; + } + + public readonly Token ReadToken() + { + PeekToken(out Token token, out int readBytes); + reader.Advance(readBytes); + return token; + } + + public readonly bool ReadToken(out Token token) + { + bool read = PeekToken(out token, out int readBytes); + reader.Advance(readBytes); + return read; + } + + /// + /// Copies the underlying text of the given into + /// the . + /// + /// Amount of values copied. + public readonly int GetText(Token token, Span destination) + { + int length = reader.PeekUTF8(token.position, token.length, destination); + if (destination[0] == '"') + { + for (int i = 0; i < length - 1; i++) + { + destination[i] = destination[i + 1]; + } + + return length - 2; + } + else return length; + } + + [SkipLocalsInit] + public readonly int GetText(Token token, Text destination) + { + Span buffer = stackalloc char[token.length]; + int length = GetText(token, buffer); + destination.Append(buffer.Slice(0, length)); + return length; + } + } +} \ No newline at end of file diff --git a/source/TOML/Token.cs b/source/TOML/Token.cs new file mode 100644 index 0000000..0f346b8 --- /dev/null +++ b/source/TOML/Token.cs @@ -0,0 +1,87 @@ +using Unmanaged; + +namespace Serialization.TOML +{ + public readonly struct Token + { + public const string Tokens = "#=,.[]{}"; + + public readonly int position; + public readonly int length; + public readonly Type type; + + public readonly int End => position + length; + + public Token(int position, int length, Type type) + { + this.position = position; + this.length = length; + this.type = type; + } + + public readonly override string ToString() + { + return $"Token(type: {type} position:{position} length:{length})"; + } + + public readonly string ToString(TOMLReader reader) + { + using Text buffer = new(0); + ToString(reader, buffer); + return buffer.ToString(); + } + + /// + /// Adds the string representation of this token to the . + /// + /// Amount of values added. + public readonly int ToString(TOMLReader reader, Text destination) + { + switch (type) + { + case Type.Text: + return reader.GetText(this, destination); + case Type.Hash: + destination.Append('#'); + return 1; + case Type.Equals: + destination.Append('='); + return 1; + case Type.Comma: + destination.Append(','); + return 1; + case Type.Period: + destination.Append('.'); + return 1; + case Type.StartSquareBracket: + destination.Append('['); + return 1; + case Type.EndSquareBracket: + destination.Append(']'); + return 1; + case Type.StartCurlyBrace: + destination.Append('{'); + return 1; + case Type.EndCurlyBrace: + destination.Append('}'); + return 1; + default: + return 0; + } + } + + public enum Type : byte + { + Unknown, + Text, + Hash, + Equals, + Comma, + Period, + StartSquareBracket, + EndSquareBracket, + StartCurlyBrace, + EndCurlyBrace + } + } +} \ No newline at end of file diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs new file mode 100644 index 0000000..14d4398 --- /dev/null +++ b/tests/TOMLTests.cs @@ -0,0 +1,54 @@ +using Collections.Generic; +using Serialization.TOML; +using System; +using Unmanaged; +using Unmanaged.Tests; + +namespace Serialization.Tests +{ + public class TOMLTests : UnmanagedTests + { + [Test] + public void ReadTokens() + { + const string Source = @"# This is a TOML document + +title = ""TOML Example"" + +[owner] +name = ""Tom Preston-Werner"" +dob = 1979-05-27T07:32:00-08:00 + +[database] +enabled = true +ports = [ 8000, 8001, 8002 ] +data = [ [""delta"", ""phi""], [3.14] ] +temp_targets = { cpu = 79.5, case = 72.0 } + +[servers] + +[servers.alpha] +ip = ""10.0.0.1"" +role = ""frontend"" + +[servers.beta] +ip = ""10.0.0.2"" +role = ""backend"""; + + using ByteWriter writer = new(); + writer.WriteUTF8(Source); + using ByteReader reader = new(writer.AsSpan()); + TOMLReader tomlReader = new(reader); + using List tokens = new(); + while (tomlReader.ReadToken(out Token token)) + { + tokens.Add(token); + } + + foreach (Token token in tokens) + { + Console.WriteLine($"{token.type} = {token.ToString(tomlReader)}"); + } + } + } +} \ No newline at end of file From e9c4f8f59724da496e55fb1d133154c238c1a96f Mon Sep 17 00:00:00 2001 From: Phill Date: Mon, 19 May 2025 21:55:44 -0400 Subject: [PATCH 3/9] basic toml reading --- source/TOML/TOMLKeyValue.cs | 145 +++++++++++++++++++++++++++ source/TOML/TOMLObject.cs | 193 ++++++++++++++++++++++++++++++++++-- source/TOML/TOMLReader.cs | 64 +++++++----- source/TOML/Token.cs | 6 +- source/TOML/ValueType.cs | 13 +++ tests/TOMLTests.cs | 34 ++++++- 6 files changed, 420 insertions(+), 35 deletions(-) create mode 100644 source/TOML/TOMLKeyValue.cs create mode 100644 source/TOML/ValueType.cs diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs new file mode 100644 index 0000000..e86bc65 --- /dev/null +++ b/source/TOML/TOMLKeyValue.cs @@ -0,0 +1,145 @@ +using System; +using System.Diagnostics; +using Unmanaged; + +namespace Serialization.TOML +{ + public unsafe struct TOMLKeyValue : IDisposable, ISerializable + { + private Implementation* keyValue; + + public readonly ReadOnlySpan Key + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + + return keyValue->key.GetSpan(keyValue->keyLength); + } + } + + public readonly ValueType ValueType + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + + return keyValue->valueType; + } + } + + public readonly bool IsDisposed => keyValue == default; + +#if NET + [Obsolete("Default constructor not supported", true)] + public TOMLKeyValue() + { + } +#endif + + public TOMLKeyValue(ReadOnlySpan key, ReadOnlySpan text) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.Text; + keyValue->keyLength = key.Length; + keyValue->key = MemoryAddress.Allocate(key); + keyValue->valueLength = text.Length; + keyValue->value = MemoryAddress.Allocate(text); + } + + public TOMLKeyValue(ReadOnlySpan key, double number) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.Number; + keyValue->keyLength = key.Length; + keyValue->key = MemoryAddress.Allocate(key); + keyValue->valueLength = sizeof(double); + keyValue->value = MemoryAddress.AllocateValue(number); + } + + public TOMLKeyValue(ReadOnlySpan key, bool boolean) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.Boolean; + keyValue->keyLength = key.Length; + keyValue->key = MemoryAddress.Allocate(key); + keyValue->valueLength = sizeof(bool); + keyValue->value = MemoryAddress.AllocateValue(boolean); + } + + public readonly override string ToString() + { + using Text destination = new(32); + ToString(destination); + return destination.ToString(); + } + + public readonly void ToString(Text destination) + { + } + + public void Dispose() + { + MemoryAddress.ThrowIfDefault(keyValue); + + keyValue->key.Dispose(); + keyValue->value.Dispose(); + MemoryAddress.Free(ref keyValue); + } + + readonly void ISerializable.Write(ByteWriter byteWriter) + { + } + + void ISerializable.Read(ByteReader byteReader) + { + keyValue = MemoryAddress.AllocatePointer(); + TOMLReader tomlReader = new(byteReader); + Token token = tomlReader.ReadToken(); + Span buffer = stackalloc char[token.length * 4]; + keyValue->keyLength = tomlReader.GetText(token, buffer); + keyValue->key = MemoryAddress.Allocate(buffer.Slice(0, keyValue->keyLength)); + + token = tomlReader.ReadToken(); + ThrowIfNotEqualsAfterKey(token.type); + + token = tomlReader.ReadToken(); + buffer = stackalloc char[token.length * 4]; + keyValue->valueLength = tomlReader.GetText(token, buffer); + ReadOnlySpan valueText = buffer.Slice(0, keyValue->valueLength); + if (double.TryParse(valueText, out double number)) + { + keyValue->valueType = ValueType.Number; + keyValue->value = MemoryAddress.AllocateValue(number); + } + else if (bool.TryParse(valueText, out bool boolean)) + { + keyValue->valueType = ValueType.Boolean; + keyValue->value = MemoryAddress.AllocateValue(boolean); + } + else + { + keyValue->valueType = ValueType.Text; + keyValue->value = MemoryAddress.Allocate(valueText); + } + } + + [Conditional("DEBUG")] + private readonly void ThrowIfNotEqualsAfterKey(Token.Type type) + { + if (type != Token.Type.Equals) + { + throw new InvalidOperationException($"Expected '=' after key, but got '{type}'"); + } + } + + private struct Implementation + { + public ValueType valueType; + public int keyLength; + public MemoryAddress key; + public int valueLength; + public MemoryAddress value; + } + } +} \ No newline at end of file diff --git a/source/TOML/TOMLObject.cs b/source/TOML/TOMLObject.cs index 8661378..d300a6c 100644 --- a/source/TOML/TOMLObject.cs +++ b/source/TOML/TOMLObject.cs @@ -1,28 +1,207 @@ -using System; +using Collections.Generic; +using System; +using System.Diagnostics; using System.Runtime.CompilerServices; using Unmanaged; namespace Serialization.TOML { [SkipLocalsInit] - public readonly struct TOMLObject : IDisposable, ISerializable + public unsafe struct TOMLObject : IDisposable, ISerializable { - public readonly void Dispose() + private Implementation* tomlObject; + + public readonly ReadOnlySpan KeyValues { + get + { + MemoryAddress.ThrowIfDefault(tomlObject); + + return tomlObject->keyValues.AsSpan(); + } } - void ISerializable.Read(ByteReader byteReader) + public readonly TOMLKeyValue this[ReadOnlySpan key] { + get + { + MemoryAddress.ThrowIfDefault(tomlObject); + ThrowIfKeyIsMissing(key); + + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + return keyValue; + } + } + return default; + } + } + + public readonly bool IsDisposed => tomlObject == default; + +#if NET + /// + /// Creates an empty TOML object. + /// + public TOMLObject() + { + tomlObject = MemoryAddress.AllocatePointer(); + tomlObject->keyValues = new(4); + } +#endif + + public TOMLObject(void* pointer) + { + this.tomlObject = (Implementation*)pointer; + } + + public readonly override string ToString() + { + using Text destination = new(32); + ToString(destination); + return destination.ToString(); + } + + public readonly void ToString(Text destination) + { + MemoryAddress.ThrowIfDefault(tomlObject); + foreach (TOMLKeyValue keyValue in tomlObject->keyValues) + { + keyValue.ToString(destination); + } + } + + public void Dispose() + { + MemoryAddress.ThrowIfDefault(tomlObject); + + foreach (TOMLKeyValue keyValue in tomlObject->keyValues) + { + keyValue.Dispose(); + } + + tomlObject->keyValues.Dispose(); + MemoryAddress.Free(ref tomlObject); + } + + public readonly void Add(ReadOnlySpan key, ReadOnlySpan text) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + TOMLKeyValue keyValue = new(key, text); + tomlObject->keyValues.Add(keyValue); + } + + public readonly void Add(ReadOnlySpan key, double number) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + TOMLKeyValue keyValue = new(key, number); + tomlObject->keyValues.Add(keyValue); + } + + public readonly void Add(ReadOnlySpan key, bool boolean) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + TOMLKeyValue keyValue = new(key, boolean); + tomlObject->keyValues.Add(keyValue); + } + + public readonly bool ContainsKey(ReadOnlySpan key) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + return true; + } + } + + return false; + } + + public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + value = keyValue; + return true; + } + } + + value = default; + return false; + } + + void ISerializable.Read(ByteReader byteReader) + { + tomlObject = MemoryAddress.AllocatePointer(); + tomlObject->keyValues = new(4); + TOMLReader tomlReader = new(byteReader); + while (tomlReader.PeekToken(out Token token)) + { + if (token.type == Token.Type.Hash) + { + //skip comments + tomlReader.ReadToken(); + tomlReader.ReadToken(); + } + else if (token.type == Token.Type.Text) + { + TOMLKeyValue keyValue = byteReader.ReadObject(); + tomlObject->keyValues.Add(keyValue); + } + else + { + tomlReader.ReadToken(); + } + } } readonly void ISerializable.Write(ByteWriter byteWriter) { + Text list = new(0); + ToString(list); + byteWriter.WriteUTF8(list.AsSpan()); + list.Dispose(); } - } - public readonly struct TOMLKeyValue - { + [Conditional("DEBUG")] + private readonly void ThrowIfKeyIsMissing(ReadOnlySpan key) + { + if (!ContainsKey(key)) + { + throw new ArgumentException($"Key '{key.ToString()}' is missing in TOML object", nameof(key)); + } + } + /// + /// Creates a new empty TOML object. + /// + /// + public static TOMLObject Create() + { + Implementation* tomlObject = MemoryAddress.AllocatePointer(); + tomlObject->keyValues = new(4); + return new(tomlObject); + } + + private struct Implementation + { + public List keyValues; + } } } \ No newline at end of file diff --git a/source/TOML/TOMLReader.cs b/source/TOML/TOMLReader.cs index 5dcc2e0..6d7e3be 100644 --- a/source/TOML/TOMLReader.cs +++ b/source/TOML/TOMLReader.cs @@ -5,13 +5,13 @@ namespace Serialization.TOML { [SkipLocalsInit] - public ref struct TOMLReader + public readonly ref struct TOMLReader { - private ByteReader reader; + private readonly ByteReader byteReader; - public TOMLReader(ByteReader reader) + public TOMLReader(ByteReader byteReader) { - this.reader = reader; + this.byteReader = byteReader; } public readonly bool PeekToken(out Token token) @@ -22,57 +22,57 @@ public readonly bool PeekToken(out Token token) public readonly bool PeekToken(out Token token, out int readBytes) { token = default; - int position = reader.Position; - int length = reader.Length; + int position = byteReader.Position; + int length = byteReader.Length; while (position < length) { - byte bytesRead = reader.PeekUTF8(position, out char c, out _); + byte bytesRead = byteReader.PeekUTF8(position, out char c, out _); if (c == '#') { token = new Token(position, bytesRead, Token.Type.Hash); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == '=') { token = new Token(position, bytesRead, Token.Type.Equals); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == ',') { token = new Token(position, bytesRead, Token.Type.Comma); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == '.') { token = new Token(position, bytesRead, Token.Type.Period); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == '[') { token = new Token(position, bytesRead, Token.Type.StartSquareBracket); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == ']') { token = new Token(position, bytesRead, Token.Type.EndSquareBracket); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == '{') { token = new Token(position, bytesRead, Token.Type.StartCurlyBrace); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (c == '}') { token = new Token(position, bytesRead, Token.Type.EndCurlyBrace); - readBytes = position - reader.Position + 1; + readBytes = position - byteReader.Position + 1; return true; } else if (SharedFunctions.IsWhitespace(c)) @@ -85,19 +85,37 @@ public readonly bool PeekToken(out Token token, out int readBytes) position += bytesRead; while (position < length) { - bytesRead = reader.PeekUTF8(position, out c, out _); + bytesRead = byteReader.PeekUTF8(position, out c, out _); if (SharedFunctions.IsEndOfLine(c) || Token.Tokens.Contains(c)) { - token = new Token(start, position - start, Token.Type.Text); - readBytes = position - reader.Position; - return true; + if (c == '=') + { + //trim whitespace + int trim = 0; + byteReader.PeekUTF8(position - trim - 1, out c, out _); + while (SharedFunctions.IsWhitespace(c)) + { + trim++; + byteReader.PeekUTF8(position - trim - 1, out c, out _); + } + + token = new Token(start, position - start - trim, Token.Type.Text); + readBytes = position - byteReader.Position; + return true; + } + else + { + token = new Token(start, position - start, Token.Type.Text); + readBytes = position - byteReader.Position; + return true; + } } position += bytesRead; } token = new Token(start, position - start, Token.Type.Text); - readBytes = position - reader.Position; + readBytes = position - byteReader.Position; return true; } } @@ -109,14 +127,14 @@ public readonly bool PeekToken(out Token token, out int readBytes) public readonly Token ReadToken() { PeekToken(out Token token, out int readBytes); - reader.Advance(readBytes); + byteReader.Advance(readBytes); return token; } public readonly bool ReadToken(out Token token) { bool read = PeekToken(out token, out int readBytes); - reader.Advance(readBytes); + byteReader.Advance(readBytes); return read; } @@ -127,7 +145,7 @@ public readonly bool ReadToken(out Token token) /// Amount of values copied. public readonly int GetText(Token token, Span destination) { - int length = reader.PeekUTF8(token.position, token.length, destination); + int length = byteReader.PeekUTF8(token.position, token.length, destination); if (destination[0] == '"') { for (int i = 0; i < length - 1; i++) diff --git a/source/TOML/Token.cs b/source/TOML/Token.cs index 0f346b8..ca1482e 100644 --- a/source/TOML/Token.cs +++ b/source/TOML/Token.cs @@ -26,9 +26,9 @@ public readonly override string ToString() public readonly string ToString(TOMLReader reader) { - using Text buffer = new(0); - ToString(reader, buffer); - return buffer.ToString(); + using Text destination = new(4); + ToString(reader, destination); + return destination.ToString(); } /// diff --git a/source/TOML/ValueType.cs b/source/TOML/ValueType.cs new file mode 100644 index 0000000..421605f --- /dev/null +++ b/source/TOML/ValueType.cs @@ -0,0 +1,13 @@ +namespace Serialization.TOML +{ + public enum ValueType : byte + { + Unknown, + Text, + Number, + Boolean, + DateTime, + Array, + Table + } +} \ No newline at end of file diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index 14d4398..02cf5e3 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -1,6 +1,6 @@ using Collections.Generic; using Serialization.TOML; -using System; +using System.Diagnostics; using Unmanaged; using Unmanaged.Tests; @@ -8,6 +8,13 @@ namespace Serialization.Tests { public class TOMLTests : UnmanagedTests { + [Test] + public void CreateAndDisposeObject() + { + TOMLObject tomlObject = new(); + tomlObject.Dispose(); + } + [Test] public void ReadTokens() { @@ -47,8 +54,31 @@ public void ReadTokens() foreach (Token token in tokens) { - Console.WriteLine($"{token.type} = {token.ToString(tomlReader)}"); + System.Console.WriteLine($"{token.type} = {token.ToString(tomlReader)}"); } } + + [Test] + public void ReadSimpleSource() + { + const string Source = @"# This is a TOML document + +title = ""TOML Example"" +amount = -3213 +enabled = true"; + + using ByteWriter writer = new(); + writer.WriteUTF8(Source); + using ByteReader reader = new(writer.AsSpan()); + TOMLReader tomlReader = new(reader); + using TOMLObject tomlObject = reader.ReadObject(); + Assert.That(tomlObject.ContainsKey("title"), Is.True); + Assert.That(tomlObject.ContainsKey("amount"), Is.True); + Assert.That(tomlObject.ContainsKey("enabled"), Is.True); + Assert.That(tomlObject.ContainsKey("unknown"), Is.False); + Assert.That(tomlObject["title"].ValueType, Is.EqualTo(ValueType.Text)); + Assert.That(tomlObject["amount"].ValueType, Is.EqualTo(ValueType.Number)); + Assert.That(tomlObject["enabled"].ValueType, Is.EqualTo(ValueType.Boolean)); + } } } \ No newline at end of file From 8008cfbde155487fb3d645203f32e93da3f4e060 Mon Sep 17 00:00:00 2001 From: Phill Date: Mon, 19 May 2025 23:11:25 -0400 Subject: [PATCH 4/9] reading toml tables --- source/TOML/TOMLKeyValue.cs | 126 ++++++++++++++++++------ source/TOML/TOMLObject.cs | 141 ++++++++++++++++++++++----- source/TOML/TOMLReader.cs | 6 -- source/TOML/TOMLTable.cs | 187 ++++++++++++++++++++++++++++++++++++ source/TOML/Token.cs | 6 +- tests/TOMLTests.cs | 34 ++++--- 6 files changed, 422 insertions(+), 78 deletions(-) create mode 100644 source/TOML/TOMLTable.cs diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs index e86bc65..64f89c6 100644 --- a/source/TOML/TOMLKeyValue.cs +++ b/source/TOML/TOMLKeyValue.cs @@ -14,7 +14,7 @@ public readonly ReadOnlySpan Key { MemoryAddress.ThrowIfDefault(keyValue); - return keyValue->key.GetSpan(keyValue->keyLength); + return keyValue->data.GetSpan(keyValue->keyLength); } } @@ -28,6 +28,39 @@ public readonly ValueType ValueType } } + public readonly ReadOnlySpan Text + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.Text); + + return keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength); + } + } + + public readonly ref double Number + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.Number); + + return ref keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + + public readonly ref bool Boolean + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.Boolean); + + return ref keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + public readonly bool IsDisposed => keyValue == default; #if NET @@ -42,9 +75,13 @@ public TOMLKeyValue(ReadOnlySpan key, ReadOnlySpan text) keyValue = MemoryAddress.AllocatePointer(); keyValue->valueType = ValueType.Text; keyValue->keyLength = key.Length; - keyValue->key = MemoryAddress.Allocate(key); keyValue->valueLength = text.Length; - keyValue->value = MemoryAddress.Allocate(text); + + int keyByteLength = sizeof(char) * key.Length; + int textByteLength = sizeof(char) * text.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + textByteLength); + keyValue->data.CopyFrom(key, 0); + keyValue->data.CopyFrom(text, keyByteLength); } public TOMLKeyValue(ReadOnlySpan key, double number) @@ -52,9 +89,12 @@ public TOMLKeyValue(ReadOnlySpan key, double number) keyValue = MemoryAddress.AllocatePointer(); keyValue->valueType = ValueType.Number; keyValue->keyLength = key.Length; - keyValue->key = MemoryAddress.Allocate(key); - keyValue->valueLength = sizeof(double); - keyValue->value = MemoryAddress.AllocateValue(number); + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(double)); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, number); } public TOMLKeyValue(ReadOnlySpan key, bool boolean) @@ -62,14 +102,17 @@ public TOMLKeyValue(ReadOnlySpan key, bool boolean) keyValue = MemoryAddress.AllocatePointer(); keyValue->valueType = ValueType.Boolean; keyValue->keyLength = key.Length; - keyValue->key = MemoryAddress.Allocate(key); - keyValue->valueLength = sizeof(bool); - keyValue->value = MemoryAddress.AllocateValue(boolean); + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + 1); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, boolean); } public readonly override string ToString() { - using Text destination = new(32); + using Text destination = new(0); ToString(destination); return destination.ToString(); } @@ -82,8 +125,7 @@ public void Dispose() { MemoryAddress.ThrowIfDefault(keyValue); - keyValue->key.Dispose(); - keyValue->value.Dispose(); + keyValue->data.Dispose(); MemoryAddress.Free(ref keyValue); } @@ -95,37 +137,62 @@ void ISerializable.Read(ByteReader byteReader) { keyValue = MemoryAddress.AllocatePointer(); TOMLReader tomlReader = new(byteReader); - Token token = tomlReader.ReadToken(); - Span buffer = stackalloc char[token.length * 4]; - keyValue->keyLength = tomlReader.GetText(token, buffer); - keyValue->key = MemoryAddress.Allocate(buffer.Slice(0, keyValue->keyLength)); - - token = tomlReader.ReadToken(); - ThrowIfNotEqualsAfterKey(token.type); - - token = tomlReader.ReadToken(); - buffer = stackalloc char[token.length * 4]; - keyValue->valueLength = tomlReader.GetText(token, buffer); - ReadOnlySpan valueText = buffer.Slice(0, keyValue->valueLength); + + //read text + Token keyToken = tomlReader.ReadToken(); + Span keyBuffer = stackalloc char[keyToken.length * 4]; + keyValue->keyLength = tomlReader.GetText(keyToken, keyBuffer); + ReadOnlySpan keyText = keyBuffer.Slice(0, keyValue->keyLength); + + //read equals + Token equalsToken = tomlReader.ReadToken(); + ThrowIfNotEqualsAfterKey(equalsToken.type); + + //read text + Token valueToken = tomlReader.ReadToken(); + Span valueBuffer = stackalloc char[valueToken.length * 4]; + int valueLength = tomlReader.GetText(valueToken, valueBuffer); + ReadOnlySpan valueText = valueBuffer.Slice(0, valueLength); + + //build data + int keyByteLength = sizeof(char) * keyValue->keyLength; if (double.TryParse(valueText, out double number)) { keyValue->valueType = ValueType.Number; - keyValue->value = MemoryAddress.AllocateValue(number); + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(double)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, number); } else if (bool.TryParse(valueText, out bool boolean)) { keyValue->valueType = ValueType.Boolean; - keyValue->value = MemoryAddress.AllocateValue(boolean); + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + 1); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, boolean); } else { keyValue->valueType = ValueType.Text; - keyValue->value = MemoryAddress.Allocate(valueText); + keyValue->valueLength = valueLength; + keyValue->data = MemoryAddress.Allocate(keyByteLength + (sizeof(char) * valueLength)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.CopyFrom(valueText, keyByteLength); + } + } + + [Conditional("DEBUG")] + private readonly void ThrowIfNotTypeOf(ValueType type) + { + if (keyValue->valueType != type) + { + throw new InvalidOperationException($"Expected value type `{type}`, but got `{keyValue->valueType}`"); } } [Conditional("DEBUG")] - private readonly void ThrowIfNotEqualsAfterKey(Token.Type type) + private static void ThrowIfNotEqualsAfterKey(Token.Type type) { if (type != Token.Type.Equals) { @@ -137,9 +204,8 @@ private struct Implementation { public ValueType valueType; public int keyLength; - public MemoryAddress key; public int valueLength; - public MemoryAddress value; + public MemoryAddress data; } } } \ No newline at end of file diff --git a/source/TOML/TOMLObject.cs b/source/TOML/TOMLObject.cs index d300a6c..3f158e9 100644 --- a/source/TOML/TOMLObject.cs +++ b/source/TOML/TOMLObject.cs @@ -21,23 +21,13 @@ public readonly ReadOnlySpan KeyValues } } - public readonly TOMLKeyValue this[ReadOnlySpan key] + public readonly ReadOnlySpan Tables { get { MemoryAddress.ThrowIfDefault(tomlObject); - ThrowIfKeyIsMissing(key); - Span keyValues = tomlObject->keyValues.AsSpan(); - foreach (TOMLKeyValue keyValue in keyValues) - { - if (keyValue.Key.SequenceEqual(key)) - { - return keyValue; - } - } - - return default; + return tomlObject->tables.AsSpan(); } } @@ -51,6 +41,7 @@ public TOMLObject() { tomlObject = MemoryAddress.AllocatePointer(); tomlObject->keyValues = new(4); + tomlObject->tables = new(4); } #endif @@ -61,7 +52,7 @@ public TOMLObject(void* pointer) public readonly override string ToString() { - using Text destination = new(32); + using Text destination = new(0); ToString(destination); return destination.ToString(); } @@ -69,22 +60,38 @@ public readonly override string ToString() public readonly void ToString(Text destination) { MemoryAddress.ThrowIfDefault(tomlObject); - foreach (TOMLKeyValue keyValue in tomlObject->keyValues) + + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) { keyValue.ToString(destination); } + + Span tables = tomlObject->tables.AsSpan(); + foreach (TOMLTable table in tables) + { + table.ToString(destination); + } } public void Dispose() { MemoryAddress.ThrowIfDefault(tomlObject); - foreach (TOMLKeyValue keyValue in tomlObject->keyValues) + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) { keyValue.Dispose(); } tomlObject->keyValues.Dispose(); + Span tables = tomlObject->tables.AsSpan(); + foreach (TOMLTable table in tables) + { + table.Dispose(); + } + + tomlObject->tables.Dispose(); MemoryAddress.Free(ref tomlObject); } @@ -112,7 +119,7 @@ public readonly void Add(ReadOnlySpan key, bool boolean) tomlObject->keyValues.Add(keyValue); } - public readonly bool ContainsKey(ReadOnlySpan key) + public readonly bool ContainsValue(ReadOnlySpan key) { MemoryAddress.ThrowIfDefault(tomlObject); @@ -146,24 +153,97 @@ public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) return false; } + public readonly TOMLKeyValue GetValue(ReadOnlySpan key) + { + MemoryAddress.ThrowIfDefault(tomlObject); + ThrowIfValueIsMissing(key); + + Span keyValues = tomlObject->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + return keyValue; + } + } + + return default; + } + + public readonly bool ContainsTable(ReadOnlySpan name) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + Span tables = tomlObject->tables.AsSpan(); + foreach (TOMLTable table in tables) + { + if (table.Name.SequenceEqual(name)) + { + return true; + } + } + + return false; + } + + public readonly bool TryGetTable(ReadOnlySpan name, out TOMLTable table) + { + MemoryAddress.ThrowIfDefault(tomlObject); + + Span tables = tomlObject->tables.AsSpan(); + foreach (TOMLTable existingTable in tables) + { + if (existingTable.Name.SequenceEqual(name)) + { + table = existingTable; + return true; + } + } + + table = default; + return false; + } + + public readonly TOMLTable GetTable(ReadOnlySpan name) + { + MemoryAddress.ThrowIfDefault(tomlObject); + ThrowIfTableIsMissing(name); + + Span tables = tomlObject->tables.AsSpan(); + foreach (TOMLTable table in tables) + { + if (table.Name.SequenceEqual(name)) + { + return table; + } + } + + return default; + } + void ISerializable.Read(ByteReader byteReader) { tomlObject = MemoryAddress.AllocatePointer(); tomlObject->keyValues = new(4); + tomlObject->tables = new(4); TOMLReader tomlReader = new(byteReader); while (tomlReader.PeekToken(out Token token)) { if (token.type == Token.Type.Hash) { - //skip comments - tomlReader.ReadToken(); - tomlReader.ReadToken(); + tomlReader.ReadToken(); //# + tomlReader.ReadToken(); //text } else if (token.type == Token.Type.Text) { TOMLKeyValue keyValue = byteReader.ReadObject(); tomlObject->keyValues.Add(keyValue); } + else if (token.type == Token.Type.StartSquareBracket) + { + TOMLTable table = byteReader.ReadObject(); + tomlObject->tables.Add(table); + } else { tomlReader.ReadToken(); @@ -173,35 +253,44 @@ void ISerializable.Read(ByteReader byteReader) readonly void ISerializable.Write(ByteWriter byteWriter) { - Text list = new(0); - ToString(list); - byteWriter.WriteUTF8(list.AsSpan()); - list.Dispose(); + using Text destination = new(32); + ToString(destination); + byteWriter.WriteUTF8(destination.AsSpan()); } [Conditional("DEBUG")] - private readonly void ThrowIfKeyIsMissing(ReadOnlySpan key) + private readonly void ThrowIfValueIsMissing(ReadOnlySpan key) { - if (!ContainsKey(key)) + if (!ContainsValue(key)) { throw new ArgumentException($"Key '{key.ToString()}' is missing in TOML object", nameof(key)); } } + [Conditional("DEBUG")] + private readonly void ThrowIfTableIsMissing(ReadOnlySpan name) + { + if (!ContainsTable(name)) + { + throw new ArgumentException($"Table '{name.ToString()}' is missing in TOML object", nameof(name)); + } + } + /// /// Creates a new empty TOML object. /// - /// public static TOMLObject Create() { Implementation* tomlObject = MemoryAddress.AllocatePointer(); tomlObject->keyValues = new(4); + tomlObject->tables = new(4); return new(tomlObject); } private struct Implementation { public List keyValues; + public List tables; } } } \ No newline at end of file diff --git a/source/TOML/TOMLReader.cs b/source/TOML/TOMLReader.cs index 6d7e3be..3e314e9 100644 --- a/source/TOML/TOMLReader.cs +++ b/source/TOML/TOMLReader.cs @@ -45,12 +45,6 @@ public readonly bool PeekToken(out Token token, out int readBytes) readBytes = position - byteReader.Position + 1; return true; } - else if (c == '.') - { - token = new Token(position, bytesRead, Token.Type.Period); - readBytes = position - byteReader.Position + 1; - return true; - } else if (c == '[') { token = new Token(position, bytesRead, Token.Type.StartSquareBracket); diff --git a/source/TOML/TOMLTable.cs b/source/TOML/TOMLTable.cs new file mode 100644 index 0000000..80949c3 --- /dev/null +++ b/source/TOML/TOMLTable.cs @@ -0,0 +1,187 @@ +using Collections.Generic; +using System; +using System.Diagnostics; +using Unmanaged; + +namespace Serialization.TOML +{ + public unsafe struct TOMLTable : IDisposable, ISerializable + { + private Implementation* tomlTable; + + public readonly ReadOnlySpan Name + { + get + { + MemoryAddress.ThrowIfDefault(tomlTable); + + return tomlTable->name.GetSpan(tomlTable->nameLength); + } + } + + public readonly ReadOnlySpan KeyValues + { + get + { + MemoryAddress.ThrowIfDefault(tomlTable); + + return tomlTable->keyValues.AsSpan(); + } + } + + public readonly bool IsDisposed => tomlTable == default; + +#if NET + [Obsolete("Default constructor not supported", true)] + public TOMLTable() + { + } +#endif + + public TOMLTable(ReadOnlySpan name) + { + tomlTable = MemoryAddress.AllocatePointer(); + tomlTable->name = MemoryAddress.Allocate(name.Length * sizeof(char)); + tomlTable->keyValues = new(4); + tomlTable->nameLength = name.Length; + } + + public readonly override string ToString() + { + using Text destination = new(0); + ToString(destination); + return destination.ToString(); + } + + public readonly void ToString(Text destination) + { + } + + public void Dispose() + { + MemoryAddress.ThrowIfDefault(tomlTable); + + tomlTable->name.Dispose(); + + Span keyValues = tomlTable->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + keyValue.Dispose(); + } + + tomlTable->keyValues.Dispose(); + MemoryAddress.Free(ref tomlTable); + } + + void ISerializable.Read(ByteReader byteReader) + { + tomlTable = MemoryAddress.AllocatePointer(); + tomlTable->keyValues = new(4); + + TOMLReader tomlReader = new(byteReader); + tomlReader.ReadToken(); //[ + Token nameToken = tomlReader.ReadToken(); + tomlReader.ReadToken(); //] + + Span nameBuffer = stackalloc char[nameToken.length * 4]; + tomlTable->nameLength = tomlReader.GetText(nameToken, nameBuffer); + tomlTable->name = MemoryAddress.Allocate(nameBuffer.Slice(0, tomlTable->nameLength)); + + while (tomlReader.PeekToken(out Token nextToken)) + { + if (nextToken.type == Token.Type.Hash) + { + tomlReader.ReadToken(); //# + tomlReader.ReadToken(); //text + } + else if (nextToken.type == Token.Type.Text) + { + TOMLKeyValue keyValue = byteReader.ReadObject(); + tomlTable->keyValues.Add(keyValue); + } + else if (nextToken.type == Token.Type.StartSquareBracket) + { + break; + } + else + { + tomlReader.ReadToken(); + } + } + } + + readonly void ISerializable.Write(ByteWriter byteWriter) + { + using Text destination = new(32); + ToString(destination); + byteWriter.WriteUTF8(destination.AsSpan()); + } + + public readonly bool ContainsValue(ReadOnlySpan key) + { + MemoryAddress.ThrowIfDefault(tomlTable); + + Span keyValues = tomlTable->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + return true; + } + } + + return false; + } + + public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) + { + MemoryAddress.ThrowIfDefault(tomlTable); + + Span keyValues = tomlTable->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + value = keyValue; + return true; + } + } + + value = default; + return false; + } + + public readonly TOMLKeyValue GetValue(ReadOnlySpan key) + { + MemoryAddress.ThrowIfDefault(tomlTable); + ThrowIfValueIsMissing(key); + + Span keyValues = tomlTable->keyValues.AsSpan(); + foreach (TOMLKeyValue keyValue in keyValues) + { + if (keyValue.Key.SequenceEqual(key)) + { + return keyValue; + } + } + + return default; + } + + [Conditional("DEBUG")] + private readonly void ThrowIfValueIsMissing(ReadOnlySpan key) + { + if (!ContainsValue(key)) + { + throw new ArgumentException($"Key '{key.ToString()}' is missing in TOML object", nameof(key)); + } + } + + private struct Implementation + { + public int nameLength; + public MemoryAddress name; + public List keyValues; + } + } +} \ No newline at end of file diff --git a/source/TOML/Token.cs b/source/TOML/Token.cs index ca1482e..f39212b 100644 --- a/source/TOML/Token.cs +++ b/source/TOML/Token.cs @@ -4,7 +4,7 @@ namespace Serialization.TOML { public readonly struct Token { - public const string Tokens = "#=,.[]{}"; + public const string Tokens = "#=,[]{}"; public readonly int position; public readonly int length; @@ -50,9 +50,6 @@ public readonly int ToString(TOMLReader reader, Text destination) case Type.Comma: destination.Append(','); return 1; - case Type.Period: - destination.Append('.'); - return 1; case Type.StartSquareBracket: destination.Append('['); return 1; @@ -77,7 +74,6 @@ public enum Type : byte Hash, Equals, Comma, - Period, StartSquareBracket, EndSquareBracket, StartCurlyBrace, diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index 02cf5e3..1b2de0c 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -1,6 +1,5 @@ using Collections.Generic; using Serialization.TOML; -using System.Diagnostics; using Unmanaged; using Unmanaged.Tests; @@ -61,24 +60,37 @@ public void ReadTokens() [Test] public void ReadSimpleSource() { - const string Source = @"# This is a TOML document + const string Source = +@"# This is a TOML document title = ""TOML Example"" -amount = -3213 -enabled = true"; +amount = -3213.777 +enabled = true + +[table] +name = ""Yes"" + +[two] +name = ""No"""; using ByteWriter writer = new(); writer.WriteUTF8(Source); using ByteReader reader = new(writer.AsSpan()); TOMLReader tomlReader = new(reader); using TOMLObject tomlObject = reader.ReadObject(); - Assert.That(tomlObject.ContainsKey("title"), Is.True); - Assert.That(tomlObject.ContainsKey("amount"), Is.True); - Assert.That(tomlObject.ContainsKey("enabled"), Is.True); - Assert.That(tomlObject.ContainsKey("unknown"), Is.False); - Assert.That(tomlObject["title"].ValueType, Is.EqualTo(ValueType.Text)); - Assert.That(tomlObject["amount"].ValueType, Is.EqualTo(ValueType.Number)); - Assert.That(tomlObject["enabled"].ValueType, Is.EqualTo(ValueType.Boolean)); + Assert.That(tomlObject.ContainsValue("title"), Is.True); + Assert.That(tomlObject.ContainsValue("amount"), Is.True); + Assert.That(tomlObject.ContainsValue("enabled"), Is.True); + Assert.That(tomlObject.ContainsValue("unknown"), Is.False); + Assert.That(tomlObject.GetValue("title").Text.ToString(), Is.EqualTo("TOML Example")); + Assert.That(tomlObject.GetValue("amount").Number, Is.EqualTo(-3213.777).Within(0.01)); + Assert.That(tomlObject.GetValue("enabled").Boolean, Is.True); + Assert.That(tomlObject.ContainsTable("table"), Is.True); + Assert.That(tomlObject.ContainsTable("two"), Is.True); + Assert.That(tomlObject.GetTable("table").ContainsValue("name"), Is.True); + Assert.That(tomlObject.GetTable("table").GetValue("name").Text.ToString(), Is.EqualTo("Yes")); + Assert.That(tomlObject.GetTable("two").ContainsValue("name"), Is.True); + Assert.That(tomlObject.GetTable("two").GetValue("name").Text.ToString(), Is.EqualTo("No")); } } } \ No newline at end of file From 0315630272d3dca31d7aef6a3fa05fac378d3440 Mon Sep 17 00:00:00 2001 From: Phill Date: Tue, 20 May 2025 08:50:28 -0400 Subject: [PATCH 5/9] rename toml object to toml document --- README.md | 60 ++++++----- source/JSON/IJSONSerializable.cs | 4 +- .../TOML/{TOMLObject.cs => TOMLDocument.cs} | 100 +++++++++--------- tests/TOMLTests.cs | 4 +- 4 files changed, 89 insertions(+), 79 deletions(-) rename source/TOML/{TOMLObject.cs => TOMLDocument.cs} (68%) diff --git a/README.md b/README.md index 60f8f3d..49a528f 100644 --- a/README.md +++ b/README.md @@ -10,23 +10,24 @@ With high-level types available for representing objects. - JSON - JSON 5 (named after ECMAScript 5) - XML +- TOML ### JSON reader and writer The reader and writers are low-level concepts used to traverse and write data: ```cs -using JSONWriter writer = new(); -writer.WriteStartObject(); -writer.WriteName("name"); -writer.WriteText("John Doe"); -writer.WriteEndObject(); - -using JSONReader reader = new(writer.GetBytes()); -reader.ReadStartObject(); -reader.ReadToken(); +using JSONWriter jsonWriter = new(); +jsonWriter.WriteStartObject(); +jsonWriter.WriteName("name"); +jsonWriter.WriteText("John Doe"); +jsonWriter.WriteEndObject(); + +using JSONReader jsonReader = new(jsonWriter.GetBytes()); +jsonReader.ReadStartObject(); +jsonReader.ReadToken(); Span nameBuffer = stackalloc char[32]; -int nameLength = reader.ReadText(nameBuffer); -reader.ReadEndObject(); +int nameLength = jsonReader.ReadText(nameBuffer); +jsonReader.ReadEndObject(); Assert.That(nameBuffer[..nameLength].ToString(), Is.EqualTo("John Doe")); ``` @@ -116,34 +117,34 @@ public struct Player : IJSONObject, IDisposable name.Dispose(); } - void IJSONObject.Read(ref JSONReader reader) + void IJSONObject.Read(ref JSONReader jsonReader) { //read hp - reader.ReadToken(); - hp = (int)reader.ReadNumber(out _); + jsonReader.ReadToken(); + hp = (int)jsonReader.ReadNumber(out _); //read alive - reader.ReadToken(); - alive = reader.ReadBoolean(out _); + jsonReader.ReadToken(); + alive = jsonReader.ReadBoolean(out _); //read name - reader.ReadToken(); + jsonReader.ReadToken(); Span nameBuffer = stackalloc char[32]; - int nameLength = reader.ReadText(nameBuffer); + int nameLength = jsonReader.ReadText(nameBuffer); name = new(nameBuffer.Slice(0, nameLength)); } - void IJSONObject.Write(JSONWriter writer) + void IJSONObject.Write(JSONWriter jsonWriter) { - writer.WriteProperty(nameof(hp), hp); - writer.WriteProperty(nameof(alive), alive); - writer.WriteProperty(nameof(name), name.AsSpan()); + jsonWriter.WriteProperty(nameof(hp), hp); + jsonWriter.WriteProperty(nameof(alive), alive); + jsonWriter.WriteProperty(nameof(name), name.AsSpan()); } } byte[] jsonBytes = File.ReadAllBytes("player.json"); -using ByteReader reader = new(jsonBytes); -JSONReader jsonReader = new(reader); +using ByteReader byteReader = new(jsonBytes); +JSONReader jsonReader = new(byteReader); using Player player = jsonReader.ReadObject(); ReadOnlySpan name = player.Name; ``` @@ -154,7 +155,8 @@ XML is supported through the `XMLNode` type, which can be created from either a Each node has a name, content, and a list of children. Attributes can be read using the indexer. ```cs byte[] xmlData = File.ReadAllBytes("solution.csproj"); -using XMLNode project = new(xmlData); +using ByteReader byteReader = new(xmlData); +using XMLNode project = byteReader.ReadObject(); XMLAttribute sdk = project["Sdk"]; sdk.Value = "Simulation.NET.Sdk"; project.TryGetFirst("PropertyGroup", out XMLNode propertyGroup); @@ -163,6 +165,14 @@ tfm.Content = "net9.0"; File.WriteAllText("solution.csproj", project.ToString()); ``` +### TOML + +```cs +byte[] tomlData = File.ReadAllBytes("config.toml"); +using ByteReader byteReader = new(tomlData); +using TOMLDocument document = byteReader.ReadObject(); +``` + ### Contributing and design Although the name of the library is `serialization`, it's not to solve serialization itself. diff --git a/source/JSON/IJSONSerializable.cs b/source/JSON/IJSONSerializable.cs index d458349..0804c64 100644 --- a/source/JSON/IJSONSerializable.cs +++ b/source/JSON/IJSONSerializable.cs @@ -2,7 +2,7 @@ { public interface IJSONSerializable { - void Read(JSONReader reader); - void Write(ref JSONWriter writer); + void Read(JSONReader byteReader); + void Write(ref JSONWriter byteWriter); } } \ No newline at end of file diff --git a/source/TOML/TOMLObject.cs b/source/TOML/TOMLDocument.cs similarity index 68% rename from source/TOML/TOMLObject.cs rename to source/TOML/TOMLDocument.cs index 3f158e9..4391678 100644 --- a/source/TOML/TOMLObject.cs +++ b/source/TOML/TOMLDocument.cs @@ -7,17 +7,17 @@ namespace Serialization.TOML { [SkipLocalsInit] - public unsafe struct TOMLObject : IDisposable, ISerializable + public unsafe struct TOMLDocument : IDisposable, ISerializable { - private Implementation* tomlObject; + private Implementation* document; public readonly ReadOnlySpan KeyValues { get { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - return tomlObject->keyValues.AsSpan(); + return document->keyValues.AsSpan(); } } @@ -25,29 +25,29 @@ public readonly ReadOnlySpan Tables { get { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - return tomlObject->tables.AsSpan(); + return document->tables.AsSpan(); } } - public readonly bool IsDisposed => tomlObject == default; + public readonly bool IsDisposed => document == default; #if NET /// - /// Creates an empty TOML object. + /// Creates an empty TOML document. /// - public TOMLObject() + public TOMLDocument() { - tomlObject = MemoryAddress.AllocatePointer(); - tomlObject->keyValues = new(4); - tomlObject->tables = new(4); + document = MemoryAddress.AllocatePointer(); + document->keyValues = new(4); + document->tables = new(4); } #endif - public TOMLObject(void* pointer) + public TOMLDocument(void* pointer) { - this.tomlObject = (Implementation*)pointer; + this.document = (Implementation*)pointer; } public readonly override string ToString() @@ -59,15 +59,15 @@ public readonly override string ToString() public readonly void ToString(Text destination) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span keyValues = tomlObject->keyValues.AsSpan(); + Span keyValues = document->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { keyValue.ToString(destination); } - Span tables = tomlObject->tables.AsSpan(); + Span tables = document->tables.AsSpan(); foreach (TOMLTable table in tables) { table.ToString(destination); @@ -76,54 +76,54 @@ public readonly void ToString(Text destination) public void Dispose() { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span keyValues = tomlObject->keyValues.AsSpan(); + Span keyValues = document->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { keyValue.Dispose(); } - tomlObject->keyValues.Dispose(); - Span tables = tomlObject->tables.AsSpan(); + document->keyValues.Dispose(); + Span tables = document->tables.AsSpan(); foreach (TOMLTable table in tables) { table.Dispose(); } - tomlObject->tables.Dispose(); - MemoryAddress.Free(ref tomlObject); + document->tables.Dispose(); + MemoryAddress.Free(ref document); } public readonly void Add(ReadOnlySpan key, ReadOnlySpan text) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); TOMLKeyValue keyValue = new(key, text); - tomlObject->keyValues.Add(keyValue); + document->keyValues.Add(keyValue); } public readonly void Add(ReadOnlySpan key, double number) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); TOMLKeyValue keyValue = new(key, number); - tomlObject->keyValues.Add(keyValue); + document->keyValues.Add(keyValue); } public readonly void Add(ReadOnlySpan key, bool boolean) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); TOMLKeyValue keyValue = new(key, boolean); - tomlObject->keyValues.Add(keyValue); + document->keyValues.Add(keyValue); } public readonly bool ContainsValue(ReadOnlySpan key) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span keyValues = tomlObject->keyValues.AsSpan(); + Span keyValues = document->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -137,9 +137,9 @@ public readonly bool ContainsValue(ReadOnlySpan key) public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span keyValues = tomlObject->keyValues.AsSpan(); + Span keyValues = document->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -155,10 +155,10 @@ public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) public readonly TOMLKeyValue GetValue(ReadOnlySpan key) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); ThrowIfValueIsMissing(key); - Span keyValues = tomlObject->keyValues.AsSpan(); + Span keyValues = document->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -172,9 +172,9 @@ public readonly TOMLKeyValue GetValue(ReadOnlySpan key) public readonly bool ContainsTable(ReadOnlySpan name) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span tables = tomlObject->tables.AsSpan(); + Span tables = document->tables.AsSpan(); foreach (TOMLTable table in tables) { if (table.Name.SequenceEqual(name)) @@ -188,9 +188,9 @@ public readonly bool ContainsTable(ReadOnlySpan name) public readonly bool TryGetTable(ReadOnlySpan name, out TOMLTable table) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); - Span tables = tomlObject->tables.AsSpan(); + Span tables = document->tables.AsSpan(); foreach (TOMLTable existingTable in tables) { if (existingTable.Name.SequenceEqual(name)) @@ -206,10 +206,10 @@ public readonly bool TryGetTable(ReadOnlySpan name, out TOMLTable table) public readonly TOMLTable GetTable(ReadOnlySpan name) { - MemoryAddress.ThrowIfDefault(tomlObject); + MemoryAddress.ThrowIfDefault(document); ThrowIfTableIsMissing(name); - Span tables = tomlObject->tables.AsSpan(); + Span tables = document->tables.AsSpan(); foreach (TOMLTable table in tables) { if (table.Name.SequenceEqual(name)) @@ -223,9 +223,9 @@ public readonly TOMLTable GetTable(ReadOnlySpan name) void ISerializable.Read(ByteReader byteReader) { - tomlObject = MemoryAddress.AllocatePointer(); - tomlObject->keyValues = new(4); - tomlObject->tables = new(4); + document = MemoryAddress.AllocatePointer(); + document->keyValues = new(4); + document->tables = new(4); TOMLReader tomlReader = new(byteReader); while (tomlReader.PeekToken(out Token token)) { @@ -237,12 +237,12 @@ void ISerializable.Read(ByteReader byteReader) else if (token.type == Token.Type.Text) { TOMLKeyValue keyValue = byteReader.ReadObject(); - tomlObject->keyValues.Add(keyValue); + document->keyValues.Add(keyValue); } else if (token.type == Token.Type.StartSquareBracket) { TOMLTable table = byteReader.ReadObject(); - tomlObject->tables.Add(table); + document->tables.Add(table); } else { @@ -263,7 +263,7 @@ private readonly void ThrowIfValueIsMissing(ReadOnlySpan key) { if (!ContainsValue(key)) { - throw new ArgumentException($"Key '{key.ToString()}' is missing in TOML object", nameof(key)); + throw new ArgumentException($"Key '{key.ToString()}' is missing in TOML document", nameof(key)); } } @@ -272,14 +272,14 @@ private readonly void ThrowIfTableIsMissing(ReadOnlySpan name) { if (!ContainsTable(name)) { - throw new ArgumentException($"Table '{name.ToString()}' is missing in TOML object", nameof(name)); + throw new ArgumentException($"Table '{name.ToString()}' is missing in TOML document", nameof(name)); } } /// - /// Creates a new empty TOML object. + /// Creates an empty TOML document. /// - public static TOMLObject Create() + public static TOMLDocument Create() { Implementation* tomlObject = MemoryAddress.AllocatePointer(); tomlObject->keyValues = new(4); diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index 1b2de0c..4d2b391 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -10,7 +10,7 @@ public class TOMLTests : UnmanagedTests [Test] public void CreateAndDisposeObject() { - TOMLObject tomlObject = new(); + TOMLDocument tomlObject = new(); tomlObject.Dispose(); } @@ -77,7 +77,7 @@ public void ReadSimpleSource() writer.WriteUTF8(Source); using ByteReader reader = new(writer.AsSpan()); TOMLReader tomlReader = new(reader); - using TOMLObject tomlObject = reader.ReadObject(); + using TOMLDocument tomlObject = reader.ReadObject(); Assert.That(tomlObject.ContainsValue("title"), Is.True); Assert.That(tomlObject.ContainsValue("amount"), Is.True); Assert.That(tomlObject.ContainsValue("enabled"), Is.True); From 500333d9ef3222bf335be67d3dd16288ff9226d2 Mon Sep 17 00:00:00 2001 From: Phill Date: Tue, 20 May 2025 09:33:26 -0400 Subject: [PATCH 6/9] handle parsing dates and times --- source/TOML/TOMLKeyValue.cs | 49 ++++++++++++++++++- source/TOML/ValueType.cs | 1 + tests/TOMLTests.cs | 95 ++++++++++++++++++++++++++----------- 3 files changed, 116 insertions(+), 29 deletions(-) diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs index 64f89c6..49ad95e 100644 --- a/source/TOML/TOMLKeyValue.cs +++ b/source/TOML/TOMLKeyValue.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics; +using System.Globalization; using Unmanaged; namespace Serialization.TOML @@ -61,6 +62,28 @@ public readonly ref bool Boolean } } + public readonly ref DateTime DateTime + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.DateTime); + + return ref keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + + public readonly ref TimeSpan TimeSpan + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.TimeSpan); + + return ref keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + public readonly bool IsDisposed => keyValue == default; #if NET @@ -156,7 +179,31 @@ void ISerializable.Read(ByteReader byteReader) //build data int keyByteLength = sizeof(char) * keyValue->keyLength; - if (double.TryParse(valueText, out double number)) + if (TimeSpan.TryParse(valueText, out TimeSpan timeSpan)) + { + keyValue->valueType = ValueType.TimeSpan; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TimeSpan)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, timeSpan); + } + else if (DateTimeOffset.TryParse(valueText, default, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset dateTimeOffset)) + { + keyValue->valueType = ValueType.DateTime; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, dateTimeOffset.DateTime); + } + else if (DateTime.TryParse(valueText, out DateTime dateTime)) + { + keyValue->valueType = ValueType.DateTime; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, dateTime); + } + else if (double.TryParse(valueText, out double number)) { keyValue->valueType = ValueType.Number; keyValue->valueLength = 1; diff --git a/source/TOML/ValueType.cs b/source/TOML/ValueType.cs index 421605f..f67e2fe 100644 --- a/source/TOML/ValueType.cs +++ b/source/TOML/ValueType.cs @@ -7,6 +7,7 @@ public enum ValueType : byte Number, Boolean, DateTime, + TimeSpan, Array, Table } diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index 4d2b391..e9bdde6 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -1,5 +1,6 @@ using Collections.Generic; using Serialization.TOML; +using System; using Unmanaged; using Unmanaged.Tests; @@ -10,8 +11,15 @@ public class TOMLTests : UnmanagedTests [Test] public void CreateAndDisposeObject() { - TOMLDocument tomlObject = new(); - tomlObject.Dispose(); + TOMLDocument document = new(); + Assert.That(document.IsDisposed, Is.False); + document.Dispose(); + Assert.That(document.IsDisposed, Is.True); + + document = new(); + Assert.That(document.IsDisposed, Is.False); + document.Dispose(); + Assert.That(document.IsDisposed, Is.True); } [Test] @@ -41,10 +49,10 @@ public void ReadTokens() ip = ""10.0.0.2"" role = ""backend"""; - using ByteWriter writer = new(); - writer.WriteUTF8(Source); - using ByteReader reader = new(writer.AsSpan()); - TOMLReader tomlReader = new(reader); + using ByteWriter byteWriter = new(); + byteWriter.WriteUTF8(Source); + using ByteReader byteReader = new(byteWriter.AsSpan()); + TOMLReader tomlReader = new(byteReader); using List tokens = new(); while (tomlReader.ReadToken(out Token token)) { @@ -60,37 +68,68 @@ public void ReadTokens() [Test] public void ReadSimpleSource() { - const string Source = + const string Source = @"# This is a TOML document title = ""TOML Example"" amount = -3213.777 enabled = true +ld1 = 1979-05-27 +lt1 = 07:32:00 +lt2 = 00:32:00.999999 [table] name = ""Yes"" +odt1 = 1979-05-27T07:32:00Z +odt2 = 1979-05-27T00:32:00-07:00 +odt3 = 1979-05-27T00:32:00.999999-07:00 +odt4 = 1979-05-27 07:32:00Z + +[another] +name = ""No"" +ldt1 = 1979-05-27T07:32:00 +ldt2 = 1979-05-27T00:32:00.999999"; + + using ByteWriter byteWriter = new(); + byteWriter.WriteUTF8(Source); + using ByteReader byteReader = new(byteWriter.AsSpan()); + TOMLReader tomlReader = new(byteReader); + + using TOMLDocument document = byteReader.ReadObject(); + Assert.That(document.ContainsValue("title"), Is.True); + Assert.That(document.ContainsValue("amount"), Is.True); + Assert.That(document.ContainsValue("enabled"), Is.True); + Assert.That(document.ContainsValue("ld1"), Is.True); + Assert.That(document.ContainsValue("lt1"), Is.True); + Assert.That(document.ContainsValue("lt2"), Is.True); + Assert.That(document.GetValue("title").Text.ToString(), Is.EqualTo("TOML Example")); + Assert.That(document.GetValue("amount").Number, Is.EqualTo(-3213.777).Within(0.01)); + Assert.That(document.GetValue("enabled").Boolean, Is.True); + Assert.That(document.GetValue("ld1").DateTime, Is.EqualTo(new DateTime(1979, 5, 27))); + Assert.That(document.GetValue("lt1").TimeSpan, Is.EqualTo(new TimeSpan(7, 32, 0))); + Assert.That(document.GetValue("lt2").TimeSpan, Is.EqualTo(new TimeSpan(0, 0, 32, 0, 999, 999))); + + Assert.That(document.ContainsTable("table"), Is.True); + TOMLTable table = document.GetTable("table"); + Assert.That(table.ContainsValue("name"), Is.True); + Assert.That(table.ContainsValue("odt1"), Is.True); + Assert.That(table.ContainsValue("odt2"), Is.True); + Assert.That(table.ContainsValue("odt3"), Is.True); + Assert.That(table.ContainsValue("odt4"), Is.True); + Assert.That(table.GetValue("name").Text.ToString(), Is.EqualTo("Yes")); + Assert.That(table.GetValue("odt1").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 7, 32, 0, DateTimeKind.Utc))); + Assert.That(table.GetValue("odt2").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 0, 32, 0, DateTimeKind.Utc).AddHours(7))); + Assert.That(table.GetValue("odt3").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 0, 32, 0, 999, 999, DateTimeKind.Utc).AddHours(7))); + Assert.That(table.GetValue("odt4").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 7, 32, 0, DateTimeKind.Utc))); -[two] -name = ""No"""; - - using ByteWriter writer = new(); - writer.WriteUTF8(Source); - using ByteReader reader = new(writer.AsSpan()); - TOMLReader tomlReader = new(reader); - using TOMLDocument tomlObject = reader.ReadObject(); - Assert.That(tomlObject.ContainsValue("title"), Is.True); - Assert.That(tomlObject.ContainsValue("amount"), Is.True); - Assert.That(tomlObject.ContainsValue("enabled"), Is.True); - Assert.That(tomlObject.ContainsValue("unknown"), Is.False); - Assert.That(tomlObject.GetValue("title").Text.ToString(), Is.EqualTo("TOML Example")); - Assert.That(tomlObject.GetValue("amount").Number, Is.EqualTo(-3213.777).Within(0.01)); - Assert.That(tomlObject.GetValue("enabled").Boolean, Is.True); - Assert.That(tomlObject.ContainsTable("table"), Is.True); - Assert.That(tomlObject.ContainsTable("two"), Is.True); - Assert.That(tomlObject.GetTable("table").ContainsValue("name"), Is.True); - Assert.That(tomlObject.GetTable("table").GetValue("name").Text.ToString(), Is.EqualTo("Yes")); - Assert.That(tomlObject.GetTable("two").ContainsValue("name"), Is.True); - Assert.That(tomlObject.GetTable("two").GetValue("name").Text.ToString(), Is.EqualTo("No")); + Assert.That(document.ContainsTable("another"), Is.True); + TOMLTable another = document.GetTable("another"); + Assert.That(another.ContainsValue("name"), Is.True); + Assert.That(another.ContainsValue("ldt1"), Is.True); + Assert.That(another.ContainsValue("ldt2"), Is.True); + Assert.That(another.GetValue("name").Text.ToString(), Is.EqualTo("No")); + Assert.That(another.GetValue("ldt1").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 7, 32, 0))); + Assert.That(another.GetValue("ldt2").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 0, 32, 0, 999, 999))); } } } \ No newline at end of file From 260fd997728e800f07b697e15a979355edbb8ad4 Mon Sep 17 00:00:00 2001 From: Phill Date: Tue, 20 May 2025 11:18:11 -0400 Subject: [PATCH 7/9] implement toml arrays (tables remaining) --- source/TOML/TOMLArray.cs | 266 ++++++++++++++++++++++++++++++++++++ source/TOML/TOMLKeyValue.cs | 244 ++++++++++++++++++++++++++------- source/TOML/TOMLReader.cs | 43 +++++- source/TOML/TOMLTable.cs | 61 +++++---- source/TOML/TOMLValue.cs | 224 ++++++++++++++++++++++++++++++ source/TOML/Token.cs | 10 +- source/XML/Token.cs | 2 +- source/XML/XMLNode.cs | 2 +- source/XML/XMLReader.cs | 6 +- tests/TOMLTests.cs | 68 ++++++++- 10 files changed, 832 insertions(+), 94 deletions(-) create mode 100644 source/TOML/TOMLArray.cs create mode 100644 source/TOML/TOMLValue.cs diff --git a/source/TOML/TOMLArray.cs b/source/TOML/TOMLArray.cs new file mode 100644 index 0000000..c46ec17 --- /dev/null +++ b/source/TOML/TOMLArray.cs @@ -0,0 +1,266 @@ +using Collections.Generic; +using System; +using System.Diagnostics; +using System.Globalization; +using Unmanaged; + +namespace Serialization.TOML +{ + public unsafe struct TOMLArray : IDisposable, ISerializable + { + internal Implementation* array; + + public readonly ReadOnlySpan Elements + { + get + { + MemoryAddress.ThrowIfDefault(array); + + return array->elements.AsSpan(); + } + } + + public readonly int Length + { + get + { + MemoryAddress.ThrowIfDefault(array); + + return array->elements.Count; + } + } + + public readonly TOMLValue this[int index] + { + get + { + MemoryAddress.ThrowIfDefault(array); + + return array->elements[index]; + } + } + +#if NET + /// + /// Creates an empty array. + /// + public TOMLArray() + { + array = MemoryAddress.AllocatePointer(); + array->elements = new(4); + } +#endif + + public TOMLArray(ReadOnlySpan array) + { + this.array = MemoryAddress.AllocatePointer(); + this.array->elements = new(array); + } + + public TOMLArray(void* pointer) + { + this.array = (Implementation*)pointer; + } + + public readonly override string ToString() + { + using Text destination = new(0); + ToString(destination); + return destination.ToString(); + } + + public readonly void ToString(Text destination) + { + MemoryAddress.ThrowIfDefault(array); + + destination.Append('['); + Span elements = array->elements.AsSpan(); + for (int i = 0; i < elements.Length; i++) + { + elements[i].ToString(destination); + if (i != elements.Length - 1) + { + destination.Append(','); + } + } + + destination.Append(']'); + } + + public void Dispose() + { + MemoryAddress.ThrowIfDefault(array); + + Span elements = array->elements.AsSpan(); + foreach (TOMLValue element in elements) + { + element.Dispose(); + } + + array->elements.Dispose(); + MemoryAddress.Free(ref array); + } + + public readonly void Add(ReadOnlySpan text) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(text); + array->elements.Add(value); + } + + public readonly void Add(double number) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(number); + array->elements.Add(value); + } + + public readonly void Add(bool boolean) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(boolean); + array->elements.Add(value); + } + + public readonly void Add(DateTime dateTime) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(dateTime); + array->elements.Add(value); + } + + public readonly void Add(TimeSpan timeSpan) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(timeSpan); + array->elements.Add(value); + } + + public readonly void Add(TOMLArray array) + { + MemoryAddress.ThrowIfDefault(this.array); + + TOMLValue value = new(array); + this.array->elements.Add(value); + } + + public readonly void Add(TOMLTable table) + { + MemoryAddress.ThrowIfDefault(array); + + TOMLValue value = new(table); + array->elements.Add(value); + } + + void ISerializable.Read(ByteReader byteReader) + { + array = MemoryAddress.AllocatePointer(); + array->elements = new(4); + + using Text buffer = new(256); + TOMLReader tomlReader = new(byteReader); + Token startToken = tomlReader.ReadToken(); //[ + ThrowIfNotArrayStart(startToken.type); + while (tomlReader.PeekToken(out Token token) && token.type != Token.Type.EndSquareBracket) + { + if (token.type == Token.Type.Text) + { + if (buffer.Length < token.length * 4) + { + buffer.SetLength(token.length * 4); + } + + tomlReader.ReadToken(); + int length = tomlReader.GetText(token, buffer.AsSpan()); + ReadOnlySpan valueText = buffer.Slice(0, length); + if (double.TryParse(valueText, out double number)) + { + TOMLValue value = new(number); + array->elements.Add(value); + } + else if (bool.TryParse(valueText, out bool boolean)) + { + TOMLValue value = new(boolean); + array->elements.Add(value); + } + else if (TimeSpan.TryParse(valueText, out TimeSpan timeSpan)) + { + TOMLValue value = new(timeSpan); + array->elements.Add(value); + } + else if (DateTimeOffset.TryParse(valueText, default, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset dateTimeOffset)) + { + TOMLValue value = new(dateTimeOffset.DateTime); + array->elements.Add(value); + } + else if (DateTime.TryParse(valueText, out DateTime dateTime)) + { + TOMLValue value = new(dateTime); + array->elements.Add(value); + } + else + { + TOMLValue value = new(valueText); + array->elements.Add(value); + } + } + else if (token.type == Token.Type.StartSquareBracket) + { + //nested array + TOMLArray nestedArray = byteReader.ReadObject(); + TOMLValue value = new(nestedArray); + array->elements.Add(value); + } + else + { + ThrowIfNotComma(token.type); + tomlReader.ReadToken(); + } + } + + Token endToken = tomlReader.ReadToken(); //] + ThrowIfNotArrayEnd(endToken.type); + } + + readonly void ISerializable.Write(ByteWriter byteWriter) + { + } + + [Conditional("DEBUG")] + private static void ThrowIfNotArrayStart(Token.Type type) + { + if (type != Token.Type.StartSquareBracket) + { + throw new InvalidOperationException($"Expected [ to start the array, but got {type}"); + } + } + + [Conditional("DEBUG")] + private static void ThrowIfNotComma(Token.Type type) + { + if (type != Token.Type.Comma) + { + throw new InvalidOperationException($"Expected a comma, but got {type}"); + } + } + + [Conditional("DEBUG")] + private static void ThrowIfNotArrayEnd(Token.Type type) + { + if (type != Token.Type.EndSquareBracket) + { + throw new InvalidOperationException($"Expected ] to end the array, but got {type}"); + } + } + + internal struct Implementation + { + public List elements; + } + } +} \ No newline at end of file diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs index 49ad95e..3ad1001 100644 --- a/source/TOML/TOMLKeyValue.cs +++ b/source/TOML/TOMLKeyValue.cs @@ -84,6 +84,28 @@ public readonly ref TimeSpan TimeSpan } } + public readonly TOMLArray Array + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.Array); + + return keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + + public readonly TOMLTable Table + { + get + { + MemoryAddress.ThrowIfDefault(keyValue); + ThrowIfNotTypeOf(ValueType.Table); + + return keyValue->data.Read(keyValue->keyLength * sizeof(char)); + } + } + public readonly bool IsDisposed => keyValue == default; #if NET @@ -133,6 +155,58 @@ public TOMLKeyValue(ReadOnlySpan key, bool boolean) keyValue->data.Write(keyByteLength, boolean); } + public TOMLKeyValue(ReadOnlySpan key, DateTime dateTime) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.DateTime; + keyValue->keyLength = key.Length; + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, dateTime); + } + + public TOMLKeyValue(ReadOnlySpan key, TimeSpan timeSpan) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.TimeSpan; + keyValue->keyLength = key.Length; + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TimeSpan)); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, timeSpan); + } + + public TOMLKeyValue(ReadOnlySpan key, TOMLArray array) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.Array; + keyValue->keyLength = key.Length; + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TOMLArray)); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, array); + } + + public TOMLKeyValue(ReadOnlySpan key, TOMLTable table) + { + keyValue = MemoryAddress.AllocatePointer(); + keyValue->valueType = ValueType.Table; + keyValue->keyLength = key.Length; + keyValue->valueLength = 1; + + int keyByteLength = sizeof(char) * key.Length; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TOMLTable)); + keyValue->data.CopyFrom(key, 0); + keyValue->data.Write(keyByteLength, table); + } + public readonly override string ToString() { using Text destination = new(0); @@ -142,12 +216,71 @@ public readonly override string ToString() public readonly void ToString(Text destination) { + MemoryAddress.ThrowIfDefault(keyValue); + + destination.Append(keyValue->data.GetSpan(keyValue->keyLength)); + destination.Append('='); + if (keyValue->valueType == ValueType.Text) + { + destination.Append(keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength)); + } + else if (keyValue->valueType == ValueType.Number) + { + double number = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + Span buffer = stackalloc char[32]; + int length = number.ToString(buffer); + destination.Append(buffer.Slice(0, length)); + } + else if (keyValue->valueType == ValueType.Boolean) + { + bool boolean = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + destination.Append(boolean ? "true" : "false"); + } + else if (keyValue->valueType == ValueType.DateTime) + { + DateTime dateTime = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + Span buffer = stackalloc char[32]; + int length = dateTime.ToString(buffer); + destination.Append(buffer.Slice(0, length)); + } + else if (keyValue->valueType == ValueType.TimeSpan) + { + TimeSpan timeSpan = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + Span buffer = stackalloc char[32]; + int length = timeSpan.ToString(buffer); + destination.Append(buffer.Slice(0, length)); + } + else if (keyValue->valueType == ValueType.Array) + { + TOMLArray array = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + array.ToString(destination); + } + else if (keyValue->valueType == ValueType.Table) + { + TOMLTable table = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + table.ToString(destination); + } + else + { + throw new InvalidOperationException($"Unsupported value type `{keyValue->valueType}` for ToString()"); + } } public void Dispose() { MemoryAddress.ThrowIfDefault(keyValue); + if (keyValue->valueType == ValueType.Array) + { + TOMLArray array = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + array.Dispose(); + } + else if (keyValue->valueType == ValueType.Table) + { + TOMLTable table = keyValue->data.Read(keyValue->keyLength * sizeof(char)); + table.Dispose(); + } + keyValue->data.Dispose(); MemoryAddress.Free(ref keyValue); } @@ -171,61 +304,78 @@ void ISerializable.Read(ByteReader byteReader) Token equalsToken = tomlReader.ReadToken(); ThrowIfNotEqualsAfterKey(equalsToken.type); - //read text - Token valueToken = tomlReader.ReadToken(); - Span valueBuffer = stackalloc char[valueToken.length * 4]; - int valueLength = tomlReader.GetText(valueToken, valueBuffer); - ReadOnlySpan valueText = valueBuffer.Slice(0, valueLength); - - //build data + //read text or array or table int keyByteLength = sizeof(char) * keyValue->keyLength; - if (TimeSpan.TryParse(valueText, out TimeSpan timeSpan)) + tomlReader.PeekToken(out Token valueToken); + if (valueToken.type == Token.Type.Text) { - keyValue->valueType = ValueType.TimeSpan; - keyValue->valueLength = 1; - keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TimeSpan)); - keyValue->data.CopyFrom(keyText, 0); - keyValue->data.Write(keyByteLength, timeSpan); - } - else if (DateTimeOffset.TryParse(valueText, default, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset dateTimeOffset)) - { - keyValue->valueType = ValueType.DateTime; - keyValue->valueLength = 1; - keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); - keyValue->data.CopyFrom(keyText, 0); - keyValue->data.Write(keyByteLength, dateTimeOffset.DateTime); + tomlReader.ReadToken(); + Span valueBuffer = stackalloc char[valueToken.length * 4]; + int valueLength = tomlReader.GetText(valueToken, valueBuffer); + ReadOnlySpan valueText = valueBuffer.Slice(0, valueLength); + + //build data + if (TimeSpan.TryParse(valueText, out TimeSpan timeSpan)) + { + keyValue->valueType = ValueType.TimeSpan; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TimeSpan)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, timeSpan); + } + else if (DateTimeOffset.TryParse(valueText, default, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset dateTimeOffset)) + { + keyValue->valueType = ValueType.DateTime; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, dateTimeOffset.DateTime); + } + else if (DateTime.TryParse(valueText, out DateTime dateTime)) + { + keyValue->valueType = ValueType.DateTime; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, dateTime); + } + else if (double.TryParse(valueText, out double number)) + { + keyValue->valueType = ValueType.Number; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(double)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, number); + } + else if (bool.TryParse(valueText, out bool boolean)) + { + keyValue->valueType = ValueType.Boolean; + keyValue->valueLength = 1; + keyValue->data = MemoryAddress.Allocate(keyByteLength + 1); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.Write(keyByteLength, boolean); + } + else + { + keyValue->valueType = ValueType.Text; + keyValue->valueLength = valueLength; + keyValue->data = MemoryAddress.Allocate(keyByteLength + (sizeof(char) * valueLength)); + keyValue->data.CopyFrom(keyText, 0); + keyValue->data.CopyFrom(valueText, keyByteLength); + } } - else if (DateTime.TryParse(valueText, out DateTime dateTime)) + else if (valueToken.type == Token.Type.StartSquareBracket) { - keyValue->valueType = ValueType.DateTime; + TOMLArray newArray = byteReader.ReadObject(); + keyValue->valueType = ValueType.Array; keyValue->valueLength = 1; - keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(DateTime)); + keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(TOMLArray)); keyValue->data.CopyFrom(keyText, 0); - keyValue->data.Write(keyByteLength, dateTime); + keyValue->data.Write(keyByteLength, newArray); } - else if (double.TryParse(valueText, out double number)) + else if (valueToken.type == Token.Type.StartCurlyBrace) { - keyValue->valueType = ValueType.Number; - keyValue->valueLength = 1; - keyValue->data = MemoryAddress.Allocate(keyByteLength + sizeof(double)); - keyValue->data.CopyFrom(keyText, 0); - keyValue->data.Write(keyByteLength, number); - } - else if (bool.TryParse(valueText, out bool boolean)) - { - keyValue->valueType = ValueType.Boolean; - keyValue->valueLength = 1; - keyValue->data = MemoryAddress.Allocate(keyByteLength + 1); - keyValue->data.CopyFrom(keyText, 0); - keyValue->data.Write(keyByteLength, boolean); - } - else - { - keyValue->valueType = ValueType.Text; - keyValue->valueLength = valueLength; - keyValue->data = MemoryAddress.Allocate(keyByteLength + (sizeof(char) * valueLength)); - keyValue->data.CopyFrom(keyText, 0); - keyValue->data.CopyFrom(valueText, keyByteLength); + } } diff --git a/source/TOML/TOMLReader.cs b/source/TOML/TOMLReader.cs index 3e314e9..3e66b0d 100644 --- a/source/TOML/TOMLReader.cs +++ b/source/TOML/TOMLReader.cs @@ -69,6 +69,44 @@ public readonly bool PeekToken(out Token token, out int readBytes) readBytes = position - byteReader.Position + 1; return true; } + else if (c == '"') + { + position += bytesRead; + int start = position; + while (position < length) + { + bytesRead = byteReader.PeekUTF8(position, out c, out _); + if (c == '"') + { + token = new Token(start, position - start, Token.Type.Text); + readBytes = position - byteReader.Position + 1; + return true; + } + + position += bytesRead; + } + + throw new InvalidOperationException("Unterminated string literal"); + } + else if (c == '\'') + { + position += bytesRead; + int start = position; + while (position < length) + { + bytesRead = byteReader.PeekUTF8(position, out c, out _); + if (c == '\'') + { + token = new Token(start, position - start, Token.Type.Text); + readBytes = position - byteReader.Position + 1; + return true; + } + + position += bytesRead; + } + + throw new InvalidOperationException("Unterminated string literal"); + } else if (SharedFunctions.IsWhitespace(c)) { position += bytesRead; @@ -152,10 +190,9 @@ public readonly int GetText(Token token, Span destination) else return length; } - [SkipLocalsInit] - public readonly int GetText(Token token, Text destination) + public readonly int AppendText(Token token, Text destination) { - Span buffer = stackalloc char[token.length]; + Span buffer = stackalloc char[token.length * 4]; int length = GetText(token, buffer); destination.Append(buffer.Slice(0, length)); return length; diff --git a/source/TOML/TOMLTable.cs b/source/TOML/TOMLTable.cs index 80949c3..86a8b3d 100644 --- a/source/TOML/TOMLTable.cs +++ b/source/TOML/TOMLTable.cs @@ -7,15 +7,15 @@ namespace Serialization.TOML { public unsafe struct TOMLTable : IDisposable, ISerializable { - private Implementation* tomlTable; + internal Implementation* table; public readonly ReadOnlySpan Name { get { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); - return tomlTable->name.GetSpan(tomlTable->nameLength); + return table->name.GetSpan(table->nameLength); } } @@ -23,13 +23,13 @@ public readonly ReadOnlySpan KeyValues { get { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); - return tomlTable->keyValues.AsSpan(); + return table->keyValues.AsSpan(); } } - public readonly bool IsDisposed => tomlTable == default; + public readonly bool IsDisposed => table == default; #if NET [Obsolete("Default constructor not supported", true)] @@ -40,10 +40,15 @@ public TOMLTable() public TOMLTable(ReadOnlySpan name) { - tomlTable = MemoryAddress.AllocatePointer(); - tomlTable->name = MemoryAddress.Allocate(name.Length * sizeof(char)); - tomlTable->keyValues = new(4); - tomlTable->nameLength = name.Length; + table = MemoryAddress.AllocatePointer(); + table->name = MemoryAddress.Allocate(name.Length * sizeof(char)); + table->keyValues = new(4); + table->nameLength = name.Length; + } + + public TOMLTable(void* pointer) + { + this.table = (Implementation*)pointer; } public readonly override string ToString() @@ -59,24 +64,24 @@ public readonly void ToString(Text destination) public void Dispose() { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); - tomlTable->name.Dispose(); + table->name.Dispose(); - Span keyValues = tomlTable->keyValues.AsSpan(); + Span keyValues = table->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { keyValue.Dispose(); } - tomlTable->keyValues.Dispose(); - MemoryAddress.Free(ref tomlTable); + table->keyValues.Dispose(); + MemoryAddress.Free(ref table); } void ISerializable.Read(ByteReader byteReader) { - tomlTable = MemoryAddress.AllocatePointer(); - tomlTable->keyValues = new(4); + table = MemoryAddress.AllocatePointer(); + table->keyValues = new(4); TOMLReader tomlReader = new(byteReader); tomlReader.ReadToken(); //[ @@ -84,8 +89,8 @@ void ISerializable.Read(ByteReader byteReader) tomlReader.ReadToken(); //] Span nameBuffer = stackalloc char[nameToken.length * 4]; - tomlTable->nameLength = tomlReader.GetText(nameToken, nameBuffer); - tomlTable->name = MemoryAddress.Allocate(nameBuffer.Slice(0, tomlTable->nameLength)); + table->nameLength = tomlReader.GetText(nameToken, nameBuffer); + table->name = MemoryAddress.Allocate(nameBuffer.Slice(0, table->nameLength)); while (tomlReader.PeekToken(out Token nextToken)) { @@ -97,7 +102,7 @@ void ISerializable.Read(ByteReader byteReader) else if (nextToken.type == Token.Type.Text) { TOMLKeyValue keyValue = byteReader.ReadObject(); - tomlTable->keyValues.Add(keyValue); + table->keyValues.Add(keyValue); } else if (nextToken.type == Token.Type.StartSquareBracket) { @@ -112,16 +117,16 @@ void ISerializable.Read(ByteReader byteReader) readonly void ISerializable.Write(ByteWriter byteWriter) { - using Text destination = new(32); + using Text destination = new(0); ToString(destination); byteWriter.WriteUTF8(destination.AsSpan()); } public readonly bool ContainsValue(ReadOnlySpan key) { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); - Span keyValues = tomlTable->keyValues.AsSpan(); + Span keyValues = table->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -135,9 +140,9 @@ public readonly bool ContainsValue(ReadOnlySpan key) public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); - Span keyValues = tomlTable->keyValues.AsSpan(); + Span keyValues = table->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -153,10 +158,10 @@ public readonly bool TryGetValue(ReadOnlySpan key, out TOMLKeyValue value) public readonly TOMLKeyValue GetValue(ReadOnlySpan key) { - MemoryAddress.ThrowIfDefault(tomlTable); + MemoryAddress.ThrowIfDefault(table); ThrowIfValueIsMissing(key); - Span keyValues = tomlTable->keyValues.AsSpan(); + Span keyValues = table->keyValues.AsSpan(); foreach (TOMLKeyValue keyValue in keyValues) { if (keyValue.Key.SequenceEqual(key)) @@ -177,7 +182,7 @@ private readonly void ThrowIfValueIsMissing(ReadOnlySpan key) } } - private struct Implementation + internal struct Implementation { public int nameLength; public MemoryAddress name; diff --git a/source/TOML/TOMLValue.cs b/source/TOML/TOMLValue.cs new file mode 100644 index 0000000..aeb2495 --- /dev/null +++ b/source/TOML/TOMLValue.cs @@ -0,0 +1,224 @@ +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Unmanaged; + +namespace Serialization.TOML +{ + [SkipLocalsInit] + public unsafe struct TOMLValue : IDisposable + { + public readonly ValueType valueType; + public readonly int length; + private MemoryAddress data; + + public readonly ReadOnlySpan Text + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.Text); + + return data.GetSpan(length); + } + } + + public readonly ref double Number + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.Number); + + return ref data.Read(); + } + } + + public readonly ref bool Boolean + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.Boolean); + + return ref data.Read(); + } + } + + public readonly ref DateTime DateTime + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.DateTime); + + return ref data.Read(); + } + } + + public readonly ref TimeSpan TimeSpan + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.TimeSpan); + + return ref data.Read(); + } + } + + public readonly TOMLArray Array + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.Array); + + return new(data.Pointer); + } + } + + public readonly TOMLTable Table + { + get + { + MemoryAddress.ThrowIfDefault(data); + ThrowIfNotTypeOf(ValueType.Table); + + return new(data.Pointer); + } + } + + public readonly bool IsDisposed => data == default; + +#if NET + [Obsolete("Default constructor not supported", true)] + public TOMLValue() + { + } +#endif + + public TOMLValue(ReadOnlySpan text) + { + valueType = ValueType.Text; + length = text.Length; + data = MemoryAddress.Allocate(text); + } + + public TOMLValue(double number) + { + valueType = ValueType.Number; + length = 1; + data = MemoryAddress.AllocateValue(number); + } + + public TOMLValue(bool boolean) + { + valueType = ValueType.Boolean; + length = 1; + data = MemoryAddress.AllocateValue(boolean); + } + + public TOMLValue(DateTime dateTime) + { + valueType = ValueType.DateTime; + length = 1; + data = MemoryAddress.AllocateValue(dateTime); + } + + public TOMLValue(TimeSpan timeSpan) + { + valueType = ValueType.TimeSpan; + length = 1; + data = MemoryAddress.AllocateValue(timeSpan); + } + + public TOMLValue(TOMLArray array) + { + valueType = ValueType.Array; + length = 1; + data = new(array.array); + } + + public TOMLValue(TOMLTable table) + { + valueType = ValueType.Table; + length = 1; + data = new(table.table); + } + + public readonly override string ToString() + { + using Text destination = new(0); + ToString(destination); + return destination.ToString(); + } + + public readonly void ToString(Text destination) + { + MemoryAddress.ThrowIfDefault(data); + + if (valueType == ValueType.Text) + { + destination.Append(data.GetSpan(length)); + } + else if (valueType == ValueType.Boolean) + { + destination.Append(data.Read() ? "true" : "false"); + } + else if (valueType == ValueType.Number) + { + destination.Append(data.Read()); + } + else if (valueType == ValueType.DateTime) + { + destination.Append(data.Read()); + } + else if (valueType == ValueType.TimeSpan) + { + destination.Append(data.Read()); + } + else if (valueType == ValueType.Array) + { + new TOMLArray(data).ToString(destination); + } + else if (valueType == ValueType.Table) + { + new TOMLTable(data).ToString(destination); + } + else + { + throw new NotSupportedException($"Unsupported TOML value type `{valueType}`"); + } + } + + public void Dispose() + { + MemoryAddress.ThrowIfDefault(data); + + if (valueType == ValueType.Array) + { + TOMLArray array = new(data.Pointer); + array.Dispose(); + } + else if (valueType == ValueType.Table) + { + TOMLTable table = new(data.Pointer); + table.Dispose(); + } + else + { + data.Dispose(); + } + } + + [Conditional("DEBUG")] + private readonly void ThrowIfNotTypeOf(ValueType valueType) + { + if (this.valueType != valueType) + { + throw new InvalidOperationException($"Array element is not of type `{valueType}`"); + } + } + } +} \ No newline at end of file diff --git a/source/TOML/Token.cs b/source/TOML/Token.cs index f39212b..b602acb 100644 --- a/source/TOML/Token.cs +++ b/source/TOML/Token.cs @@ -24,10 +24,10 @@ public readonly override string ToString() return $"Token(type: {type} position:{position} length:{length})"; } - public readonly string ToString(TOMLReader reader) + public readonly string ToString(TOMLReader tomlReader) { - using Text destination = new(4); - ToString(reader, destination); + using Text destination = new(0); + ToString(tomlReader, destination); return destination.ToString(); } @@ -35,12 +35,12 @@ public readonly string ToString(TOMLReader reader) /// Adds the string representation of this token to the . /// /// Amount of values added. - public readonly int ToString(TOMLReader reader, Text destination) + public readonly int ToString(TOMLReader tomlReader, Text destination) { switch (type) { case Type.Text: - return reader.GetText(this, destination); + return tomlReader.AppendText(this, destination); case Type.Hash: destination.Append('#'); return 1; diff --git a/source/XML/Token.cs b/source/XML/Token.cs index ecbca86..e129653 100644 --- a/source/XML/Token.cs +++ b/source/XML/Token.cs @@ -47,7 +47,7 @@ public readonly int ToString(XMLReader reader, Text destination) destination.Append('/'); return 1; case Type.Text: - return reader.GetText(this, destination); + return reader.AppendText(this, destination); case Type.Prologue: destination.Append('?'); return 1; diff --git a/source/XML/XMLNode.cs b/source/XML/XMLNode.cs index 3de1eca..3e44deb 100644 --- a/source/XML/XMLNode.cs +++ b/source/XML/XMLNode.cs @@ -658,7 +658,7 @@ public static XMLNode Create(string name) public static XMLNode Create() { - Text name = new(4); + Text name = new(0); List attributes = new(4); Text content = new(0); List children = new(4); diff --git a/source/XML/XMLReader.cs b/source/XML/XMLReader.cs index 634d706..b1747fd 100644 --- a/source/XML/XMLReader.cs +++ b/source/XML/XMLReader.cs @@ -4,6 +4,7 @@ namespace Serialization.XML { + [SkipLocalsInit] public ref struct XMLReader { private ByteReader reader; @@ -178,10 +179,9 @@ public readonly int GetText(Token token, Span destination) else return length; } - [SkipLocalsInit] - public readonly int GetText(Token token, Text destination) + public readonly int AppendText(Token token, Text destination) { - Span buffer = stackalloc char[token.length]; + Span buffer = stackalloc char[token.length * 4]; int length = GetText(token, buffer); destination.Append(buffer.Slice(0, length)); return length; diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index e9bdde6..5ca7764 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -53,16 +53,72 @@ public void ReadTokens() byteWriter.WriteUTF8(Source); using ByteReader byteReader = new(byteWriter.AsSpan()); TOMLReader tomlReader = new(byteReader); - using List tokens = new(); while (tomlReader.ReadToken(out Token token)) { - tokens.Add(token); + Console.WriteLine($"{token.type}:{token.ToString(tomlReader)}"); } + } - foreach (Token token in tokens) - { - System.Console.WriteLine($"{token.type} = {token.ToString(tomlReader)}"); - } + [Test] + public void ReadArraysWithInlineTables() + { + const string Source = +@"integers = [ 1, 2, 3 ] +colors = [ ""red"", ""yellow"", ""green"" ] +nested_arrays_of_ints = [ [ 1, 2 ], [3, 4, 5] ] +nested_mixed_array = [ [ 1, 2 ], [""a"", ""b"", ""c""] ] +]"; + + using ByteWriter byteWriter = new(); + byteWriter.WriteUTF8(Source); + using ByteReader byteReader = new(byteWriter.AsSpan()); + TOMLReader tomlReader = new(byteReader); + + using TOMLDocument document = byteReader.ReadObject(); + Assert.That(document.ContainsValue("integers"), Is.True); + Assert.That(document.ContainsValue("colors"), Is.True); + Assert.That(document.ContainsValue("nested_arrays_of_ints"), Is.True); + Assert.That(document.ContainsValue("nested_mixed_array"), Is.True); + + TOMLArray integers = document.GetValue("integers").Array; + Assert.That(integers.Length, Is.EqualTo(3)); + Assert.That(integers[0].Number, Is.EqualTo(1)); + Assert.That(integers[1].Number, Is.EqualTo(2)); + Assert.That(integers[2].Number, Is.EqualTo(3)); + + TOMLArray colors = document.GetValue("colors").Array; + Assert.That(colors.Length, Is.EqualTo(3)); + Assert.That(colors[0].Text.ToString(), Is.EqualTo("red")); + Assert.That(colors[1].Text.ToString(), Is.EqualTo("yellow")); + Assert.That(colors[2].Text.ToString(), Is.EqualTo("green")); + + TOMLArray nestedArraysOfInts = document.GetValue("nested_arrays_of_ints").Array; + Assert.That(nestedArraysOfInts.Length, Is.EqualTo(2)); + + TOMLArray first = nestedArraysOfInts[0].Array; + Assert.That(first.Length, Is.EqualTo(2)); + Assert.That(first[0].Number, Is.EqualTo(1)); + Assert.That(first[1].Number, Is.EqualTo(2)); + + TOMLArray second = nestedArraysOfInts[1].Array; + Assert.That(second.Length, Is.EqualTo(3)); + Assert.That(second[0].Number, Is.EqualTo(3)); + Assert.That(second[1].Number, Is.EqualTo(4)); + Assert.That(second[2].Number, Is.EqualTo(5)); + + TOMLArray nestedMixedArray = document.GetValue("nested_mixed_array").Array; + Assert.That(nestedMixedArray.Length, Is.EqualTo(2)); + + first = nestedMixedArray[0].Array; + Assert.That(first.Length, Is.EqualTo(2)); + Assert.That(first[0].Number, Is.EqualTo(1)); + Assert.That(first[1].Number, Is.EqualTo(2)); + + second = nestedMixedArray[1].Array; + Assert.That(second.Length, Is.EqualTo(3)); + Assert.That(second[0].Text.ToString(), Is.EqualTo("a")); + Assert.That(second[1].Text.ToString(), Is.EqualTo("b")); + Assert.That(second[2].Text.ToString(), Is.EqualTo("c")); } [Test] From a6a712f819311f0f6f6f13b80d99451ac0a6ccc0 Mon Sep 17 00:00:00 2001 From: Phill Date: Tue, 20 May 2025 14:12:33 -0400 Subject: [PATCH 8/9] writing toml to bytes --- source/TOML/TOMLArray.cs | 13 ++++- source/TOML/TOMLDocument.cs | 17 ++++++ source/TOML/TOMLKeyValue.cs | 12 +++- source/TOML/TOMLValue.cs | 12 +++- tests/TOMLTests.cs | 108 +++++++++++++++++++++++++++++++++++- 5 files changed, 157 insertions(+), 5 deletions(-) diff --git a/source/TOML/TOMLArray.cs b/source/TOML/TOMLArray.cs index c46ec17..055e2df 100644 --- a/source/TOML/TOMLArray.cs +++ b/source/TOML/TOMLArray.cs @@ -57,9 +57,20 @@ public TOMLArray(ReadOnlySpan array) this.array->elements = new(array); } + public TOMLArray(ReadOnlySpan numbers) + { + array = MemoryAddress.AllocatePointer(); + array->elements = new(numbers.Length); + for (int i = 0; i < numbers.Length; i++) + { + TOMLValue value = new(numbers[i]); + array->elements.Add(value); + } + } + public TOMLArray(void* pointer) { - this.array = (Implementation*)pointer; + array = (Implementation*)pointer; } public readonly override string ToString() diff --git a/source/TOML/TOMLDocument.cs b/source/TOML/TOMLDocument.cs index 4391678..2cfda59 100644 --- a/source/TOML/TOMLDocument.cs +++ b/source/TOML/TOMLDocument.cs @@ -65,12 +65,14 @@ public readonly void ToString(Text destination) foreach (TOMLKeyValue keyValue in keyValues) { keyValue.ToString(destination); + destination.AppendLine(); } Span tables = document->tables.AsSpan(); foreach (TOMLTable table in tables) { table.ToString(destination); + destination.AppendLine(); } } @@ -119,6 +121,21 @@ public readonly void Add(ReadOnlySpan key, bool boolean) document->keyValues.Add(keyValue); } + public readonly void Add(ReadOnlySpan key, TOMLArray array) + { + MemoryAddress.ThrowIfDefault(document); + + TOMLKeyValue keyValue = new(key, array); + document->keyValues.Add(keyValue); + } + + public readonly void Add(TOMLTable table) + { + MemoryAddress.ThrowIfDefault(document); + + document->tables.Add(table); + } + public readonly bool ContainsValue(ReadOnlySpan key) { MemoryAddress.ThrowIfDefault(document); diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs index 3ad1001..1545d22 100644 --- a/source/TOML/TOMLKeyValue.cs +++ b/source/TOML/TOMLKeyValue.cs @@ -222,7 +222,17 @@ public readonly void ToString(Text destination) destination.Append('='); if (keyValue->valueType == ValueType.Text) { - destination.Append(keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength)); + Span text = keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength); + if (text.Contains(' ')) + { + destination.Append('"'); + destination.Append(text); + destination.Append('"'); + } + else + { + destination.Append(text); + } } else if (keyValue->valueType == ValueType.Number) { diff --git a/source/TOML/TOMLValue.cs b/source/TOML/TOMLValue.cs index aeb2495..e65f349 100644 --- a/source/TOML/TOMLValue.cs +++ b/source/TOML/TOMLValue.cs @@ -160,7 +160,17 @@ public readonly void ToString(Text destination) if (valueType == ValueType.Text) { - destination.Append(data.GetSpan(length)); + Span text = data.GetSpan(length); + if (text.Contains(' ')) + { + destination.Append('"'); + destination.Append(text); + destination.Append('"'); + } + else + { + destination.Append(text); + } } else if (valueType == ValueType.Boolean) { diff --git a/tests/TOMLTests.cs b/tests/TOMLTests.cs index 5ca7764..2df00ea 100644 --- a/tests/TOMLTests.cs +++ b/tests/TOMLTests.cs @@ -1,4 +1,3 @@ -using Collections.Generic; using Serialization.TOML; using System; using Unmanaged; @@ -94,7 +93,7 @@ public void ReadArraysWithInlineTables() TOMLArray nestedArraysOfInts = document.GetValue("nested_arrays_of_ints").Array; Assert.That(nestedArraysOfInts.Length, Is.EqualTo(2)); - + TOMLArray first = nestedArraysOfInts[0].Array; Assert.That(first.Length, Is.EqualTo(2)); Assert.That(first[0].Number, Is.EqualTo(1)); @@ -187,5 +186,110 @@ public void ReadSimpleSource() Assert.That(another.GetValue("ldt1").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 7, 32, 0))); Assert.That(another.GetValue("ldt2").DateTime, Is.EqualTo(new DateTime(1979, 5, 27, 0, 32, 0, 999, 999))); } + + [Test] + public void WriteSimpleSource() + { + using TOMLDocument document = new(); + document.Add("title", "TOML Example"); + document.Add("amount", -3213.777); + document.Add("enabled", true); + + using ByteWriter byteWriter = new(); + byteWriter.WriteObject(document); + + using ByteReader byteReader = new(byteWriter.AsSpan()); + TOMLReader tomlReader = new(byteReader); + using TOMLDocument readDocument = byteReader.ReadObject(); + Assert.That(readDocument.ContainsValue("title"), Is.True); + Assert.That(readDocument.ContainsValue("amount"), Is.True); + Assert.That(readDocument.ContainsValue("enabled"), Is.True); + Assert.That(readDocument.GetValue("title").Text.ToString(), Is.EqualTo("TOML Example")); + Assert.That(readDocument.GetValue("amount").Number, Is.EqualTo(-3213.777).Within(0.01)); + Assert.That(readDocument.GetValue("enabled").Boolean, Is.True); + } + + [Test] + public void WriteWithArrays() + { + using TOMLDocument document = new(); + TOMLArray integers = new([1, 2, 3]); + document.Add("integers", integers); + + TOMLArray colors = new(); + colors.Add("red"); + colors.Add("yellow"); + colors.Add("green"); + document.Add("colors", colors); + + TOMLArray nestedArraysOfInts = new(); + TOMLArray first = new([1, 2]); + TOMLArray second = new([3, 4, 5]); + nestedArraysOfInts.Add(first); + nestedArraysOfInts.Add(second); + document.Add("nested_arrays_of_ints", nestedArraysOfInts); + + TOMLArray nestedMixedArray = new(); + first = new([1, 2]); + second = new(); + second.Add("a"); + second.Add("b"); + second.Add("c"); + nestedMixedArray.Add(first); + nestedMixedArray.Add(second); + document.Add("nested_mixed_array", nestedMixedArray); + + using ByteWriter byteWriter = new(); + byteWriter.WriteObject(document); + + using ByteReader byteReader = new(byteWriter.AsSpan()); + TOMLReader tomlReader = new(byteReader); + using TOMLDocument readDocument = byteReader.ReadObject(); + + Assert.That(readDocument.ContainsValue("integers"), Is.True); + Assert.That(readDocument.ContainsValue("colors"), Is.True); + Assert.That(readDocument.ContainsValue("nested_arrays_of_ints"), Is.True); + Assert.That(readDocument.ContainsValue("nested_mixed_array"), Is.True); + + TOMLArray readIntegers = readDocument.GetValue("integers").Array; + Assert.That(readIntegers.Length, Is.EqualTo(3)); + Assert.That(readIntegers[0].Number, Is.EqualTo(1)); + Assert.That(readIntegers[1].Number, Is.EqualTo(2)); + Assert.That(readIntegers[2].Number, Is.EqualTo(3)); + + TOMLArray readColors = readDocument.GetValue("colors").Array; + Assert.That(readColors.Length, Is.EqualTo(3)); + Assert.That(readColors[0].Text.ToString(), Is.EqualTo("red")); + Assert.That(readColors[1].Text.ToString(), Is.EqualTo("yellow")); + Assert.That(readColors[2].Text.ToString(), Is.EqualTo("green")); + + TOMLArray readNestedArraysOfInts = readDocument.GetValue("nested_arrays_of_ints").Array; + Assert.That(readNestedArraysOfInts.Length, Is.EqualTo(2)); + + TOMLArray readFirst = readNestedArraysOfInts[0].Array; + Assert.That(readFirst.Length, Is.EqualTo(2)); + Assert.That(readFirst[0].Number, Is.EqualTo(1)); + Assert.That(readFirst[1].Number, Is.EqualTo(2)); + + TOMLArray readSecond = readNestedArraysOfInts[1].Array; + Assert.That(readSecond.Length, Is.EqualTo(3)); + Assert.That(readSecond[0].Number, Is.EqualTo(3)); + Assert.That(readSecond[1].Number, Is.EqualTo(4)); + Assert.That(readSecond[2].Number, Is.EqualTo(5)); + + TOMLArray readNestedMixedArray = readDocument.GetValue("nested_mixed_array").Array; + Assert.That(readNestedMixedArray.Length, Is.EqualTo(2)); + + readFirst = readNestedMixedArray[0].Array; + Assert.That(readFirst.Length, Is.EqualTo(2)); + Assert.That(readFirst[0].Number, Is.EqualTo(1)); + Assert.That(readFirst[1].Number, Is.EqualTo(2)); + + readSecond = readNestedMixedArray[1].Array; + Assert.That(readSecond.Length, Is.EqualTo(3)); + Assert.That(readSecond[0].Text.ToString(), Is.EqualTo("a")); + Assert.That(readSecond[1].Text.ToString(), Is.EqualTo("b")); + Assert.That(readSecond[2].Text.ToString(), Is.EqualTo("c")); + } } } \ No newline at end of file From 69dae7b38f16dc1c59b985a53f8eaf87ae5b77e5 Mon Sep 17 00:00:00 2001 From: Phill Date: Sun, 1 Jun 2025 12:32:27 -0400 Subject: [PATCH 9/9] use byte position as start --- source/TOML/TOMLKeyValue.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/TOML/TOMLKeyValue.cs b/source/TOML/TOMLKeyValue.cs index 1545d22..e94d8e1 100644 --- a/source/TOML/TOMLKeyValue.cs +++ b/source/TOML/TOMLKeyValue.cs @@ -36,7 +36,7 @@ public readonly ReadOnlySpan Text MemoryAddress.ThrowIfDefault(keyValue); ThrowIfNotTypeOf(ValueType.Text); - return keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength); + return keyValue->data.AsSpan(keyValue->keyLength * sizeof(char), keyValue->valueLength); } } @@ -222,7 +222,7 @@ public readonly void ToString(Text destination) destination.Append('='); if (keyValue->valueType == ValueType.Text) { - Span text = keyValue->data.AsSpan(keyValue->keyLength, keyValue->valueLength); + Span text = keyValue->data.AsSpan(keyValue->keyLength * sizeof(char), keyValue->valueLength); if (text.Contains(' ')) { destination.Append('"');