diff --git a/README.md b/README.md index 5909f922..02955c31 100644 --- a/README.md +++ b/README.md @@ -673,7 +673,7 @@ Lua-CSharp is designed with integration into .NET in mind, so there are several ### Character Encoding -The character encoding used in Lua-CSharp is UTF-16. Since standard Lua assumes a single-byte character encoding, string behavior differs significantly. +By default, Lua-CSharp uses UTF-16. Since standard Lua assumes a single-byte character encoding, string behavior differs significantly. See [String modes](docs/string-modes.md) for UTF-16 and byte-string behavior and C# conversions. For example, in regular Lua, the following code outputs `15`, but in Lua-CSharp, it outputs `5`. @@ -682,7 +682,53 @@ local l = string.len("あいうえお") print(l) ``` -All string library functions handle strings as UTF-16. +In the default mode, string library functions handle strings as UTF-16. + +#### Byte strings (opt-in) + +If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), use `LuaStringMode.Bytes`. `LuaPlatform.StringMode` defaults to `LuaStringMode.Utf16`. `LuaPlatform.DefaultByteStrings` provides byte strings and byte-oriented file and standard I/O: + +```cs +using Lua; +using Lua.Platforms; +using Lua.Standard; + +var state = LuaState.Create(LuaPlatform.DefaultByteStrings); +state.OpenStandardLibraries(); + +// Every char of a Lua string holds one byte, so convert .NET strings at the boundary. +state.Environment["name"] = LuaByteString.FromString("Tiến Lên"); // UTF-8 bytes, #name == 11 +var results = await state.DoStringAsync("return #name, name:upper()"); +var upper = LuaByteString.ToString(results[1].Read()); // back to a .NET string +``` + +In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16. + +Choose one representation for a state and configure it before executing scripts. Setting `StringMode = LuaStringMode.Bytes` on an otherwise unchanged platform does not switch its streams to raw bytes. Use matching modes for the platform and its streams. When configuring a file root, keep that root as a normal .NET path: + +```cs +var platform = LuaPlatform.DefaultByteStrings with +{ + FileSystem = new Lua.IO.FileSystem(scriptsDirectory, LuaStringMode.Bytes) +}; +using var byteState = LuaState.Create(platform); +byteState.OpenStandardLibraries(); + +// C# source and filename APIs accept normal .NET text. +var values = await byteState.DoStringAsync("return 'こんにちは'"); +var greeting = LuaByteString.ToString(values[0].Read()); + +// Raw packets can contain NUL, invalid UTF-8 and every value from 0 to 255. +byteState.Environment["packet"] = LuaByteString.FromBytes(new byte[] { 0xFF, 0, 0x80 }); +var packet = await byteState.DoStringAsync("return packet:reverse()"); +byte[] reversed = LuaByteString.ToBytes(packet[0].Read()); +``` + +Use `FromString`/`ToString` for string values and `FromBytes`/`ToBytes` for binary data. This also applies to non-ASCII table keys and the arguments/results of C# functions and generated bindings; value interop does not convert strings automatically. C# source APIs (`DoStringAsync`, `Load` with character source or a `LuaModule`) accept normal .NET text and encode it as UTF-8 in byte mode. Chunk names and paths passed to `LoadFileAsync`/`DoFileAsync` are also .NET text. Byte source passed to `Load`/`ExecuteAsync` and source read from files are preserved directly; their explicit chunk names are .NET text. Lua's `load`, `loadfile`, `dofile` and `require` continue to use Lua byte strings without re-encoding their source. + +Paths and names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader` are normal .NET strings in both modes. Text modules are treated as .NET text and encoded to UTF-8 in byte mode; byte modules remain bytes. Unity text assets use the same conversion. Custom file systems and standard-I/O providers must implement the selected stream representation themselves; use `ILuaStream.CreateFromStream(stream, mode, LuaStringMode.Bytes)` for raw streams. + +Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `ToString`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters. ### Garbage Collection diff --git a/docs/string-modes.md b/docs/string-modes.md new file mode 100644 index 00000000..a3714678 --- /dev/null +++ b/docs/string-modes.md @@ -0,0 +1,124 @@ +# String modes + +## Choose a mode + +`LuaStringMode.Utf16` is the default and uses ordinary .NET UTF-16 strings. Use `LuaStringMode.Bytes` for Lua byte semantics or binary data; each C# `char` then holds one byte (0–255). + +```csharp +using Lua; +using Lua.Platforms; +using Lua.Standard; + +using var textState = LuaState.Create(); // Utf16 +using var byteState = LuaState.Create(LuaPlatform.DefaultByteStrings); +textState.OpenStandardLibraries(); +byteState.OpenStandardLibraries(); +``` + +Keep the mode fixed for a state's lifetime. Changing its platform does not convert existing strings, table keys or compiled chunks. Values shared between states with different modes need explicit conversion. + +## Lua string operations + +Byte mode follows Lua 5.2's byte-string model: strings can contain arbitrary bytes, and lengths, substring positions and pattern matching operate on bytes. Character classes and case conversion follow ASCII C-locale rules. + +For UTF-8 source text: + +| Operation | `Utf16` | `Bytes` | +| --- | --- | --- | +| `#"ế"` | 1 | 3 | +| `#"😀"` | 2 | 4 | +| `string.len`, `string.sub`, `string.byte` | UTF-16 code units | Bytes | +| Case conversion and pattern classes | .NET character rules | ASCII C-locale rules | + +For example, this script returns different results in each mode: + +```lua +local s = "é" +return #s, string.byte(s, 1), s:match("^%a+$") ~= nil +-- Utf16: 1, 233, true +-- Bytes: 2, 195, false +``` + +Lengths and slices do not count user-perceived characters. UTF-16 slicing can split a surrogate pair; byte slicing can split a UTF-8 sequence. Byte positions from Lua cannot be reused as indices into decoded .NET text. Use .NET text-element APIs on decoded text for user-perceived-character editing. + +In byte mode, `string.char` accepts only 0–255. Non-ASCII letters remain unchanged by `string.upper`/`string.lower` and do not match `%a`. Decode text before applying Unicode case conversion or classification in C#. + +Neither mode normalizes Unicode. For example, C# `"é"` and `"e\u0301"` remain different strings and table keys. + +This describes string semantics. Existing Lua-CSharp limitations still apply; for example, `string.dump` is unsupported. C# values still use `string` and require the conversions below, with matching stream configuration. + +## Passing values between C# and Lua + +UTF-16 values can use .NET strings directly. In byte mode, convert values according to their contents: + +| Conversion | API | +| --- | --- | +| .NET text → Lua byte string | `LuaByteString.FromString(text)` | +| UTF-8 or binary bytes → Lua byte string | `LuaByteString.FromBytes(bytes)` | +| Lua byte string → .NET text | `LuaByteString.ToString(value)` | +| Lua byte string → raw bytes | `LuaByteString.ToBytes(value)` | + +C# UTF-8 literals (`u8`, C# 11+) can supply bytes directly. `FromString("é")` and `FromBytes("é"u8)` produce the same bytes, `C3 A9`. + +```csharp +byteState.Environment["greeting"] = LuaByteString.FromBytes("こんにちは"u8); +var values = await byteState.DoStringAsync("return greeting"); +string greeting = LuaByteString.ToString(values[0].Read()); + +byteState.Environment["packet"] = LuaByteString.FromBytes(new byte[] { 0xFF, 0, 0x80 }); +var packets = await byteState.DoStringAsync("return packet"); +byte[] packet = LuaByteString.ToBytes(packets[0].Read()); +``` + +### Convert text once + +Both representations use C# `string`; the type does not identify the representation. A string read from a byte-mode state is already a byte string. Calling `FromString` on it again changes non-ASCII data. + +Value conversion is explicit for table keys, C# function arguments/results and generated bindings too. ASCII-only testing can hide missing conversions: directly assigning .NET `"é"` in byte mode gives one byte, `E9`, rather than the UTF-8 bytes `C3 A9`. + +### Preserve binary data + +Use `FromBytes`/`ToBytes` for NUL bytes, invalid UTF-8 and other binary data. `FromString` replaces unpaired UTF-16 surrogates; `ToString` replaces invalid UTF-8 sequences with U+FFFD. These replacements lose the original data. + +Valid byte strings contain only chars up to U+00FF. `ToBytes` and raw output replace higher chars with `?`. + +## Loading source and naming files + +Source-loading APIs handle conversion for you: + +```csharp +// .NET text: encoded as UTF-8 in byte mode. +var textChunk = byteState.Load("return 'é'", "example"); + +// Already encoded UTF-8: pass bytes directly. +var utf8Chunk = byteState.Load("return 'é'"u8, "example"); +``` + +- `DoStringAsync`, character-source `Load` and text `LuaModule` sources accept normal .NET text. Applying `FromString` first would encode the source twice in byte mode. +- In byte mode, byte-source `Load`, byte modules and source files preserve source bytes, except for an initial UTF-8 BOM. Lua's `load` uses strings in the state's representation without encoding them again. +- C# chunk names and paths are normal .NET strings in both modes, including names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader`. + +## Configuring file and standard I/O + +`LuaPlatform.DefaultByteStrings` configures matching byte-mode file and standard I/O. Changing `StringMode` alone does not reconfigure streams. Custom providers must use the same representation as the state; wrap raw streams with `ILuaStream.CreateFromStream(stream, openMode, LuaStringMode.Bytes)`. + +For Unity, use `new Lua.Unity.UnityStandardIO(LuaStringMode.Bytes)` with a byte-mode state. It decodes UTF-8 for display through `Debug.Log`/`Debug.LogError`; use raw streams to preserve binary output. A supplied input stream must also use byte strings. + +A file root remains a normal .NET path: + +```csharp +var platform = LuaPlatform.DefaultByteStrings with +{ + FileSystem = new Lua.IO.FileSystem("scripts", LuaStringMode.Bytes) +}; +``` + +| I/O behavior | `Utf16` | `Bytes` | +| --- | --- | --- | +| File and standard stream contents | UTF-8 text decoded to/from UTF-16 | Raw bytes | +| `file:read(n)` | Up to `n` UTF-16 code units | Up to `n` bytes | +| `*l` reading `hello\r\n` | `hello` | `hello\r` | +| `*L` reading `hello\r\n` | `hello\r\n` | `hello\r\n` | +| Initial UTF-8 BOM read as data | Skipped | Preserved | + +The `b` flag in `io.open(path, "rb")` does not select byte-string mode. Built-in streams use their configured `LuaStringMode` for both `"r"` and `"rb"`. Byte-mode line reads stop at LF; remove a trailing CR explicitly when the application expects a line without either newline character. diff --git a/sandbox/Benchmark/ByteStringIoBenchmark.cs b/sandbox/Benchmark/ByteStringIoBenchmark.cs new file mode 100644 index 00000000..d961231e --- /dev/null +++ b/sandbox/Benchmark/ByteStringIoBenchmark.cs @@ -0,0 +1,50 @@ +using BenchmarkDotNet.Attributes; +using Lua; +using Lua.IO; + +[Config(typeof(BenchmarkConfig))] +public class ByteStringIoBenchmark +{ + [Params(LuaStringMode.Utf16, LuaStringMode.Bytes)] + public LuaStringMode StringMode { get; set; } + + [Params(1, 4096)] + public int Size { get; set; } + + MemoryStream input = default!; + MemoryStream output = default!; + LuaStream reader = default!; + LuaStream writer = default!; + string text = default!; + + [GlobalSetup] + public void Setup() + { + text = new string('a', Size); + input = new MemoryStream(Enumerable.Repeat((byte)'a', Size).ToArray()); + output = new MemoryStream(Size); + reader = new LuaStream(LuaFileOpenMode.Read, input, StringMode); + writer = new LuaStream(LuaFileOpenMode.Write, output, StringMode); + } + + [GlobalCleanup] + public void Cleanup() + { + reader.Dispose(); + writer.Dispose(); + } + + [Benchmark] + public string? Read() + { + reader.Seek(SeekOrigin.Begin, 0); + return reader.ReadAsync(Size, default).GetAwaiter().GetResult(); + } + + [Benchmark] + public void Write() + { + output.Position = 0; + writer.WriteAsync(text.AsMemory(), default).GetAwaiter().GetResult(); + } +} diff --git a/sandbox/Benchmark/Utf8CharConversionBenchmark.cs b/sandbox/Benchmark/Utf8CharConversionBenchmark.cs new file mode 100644 index 00000000..41f3ecbb --- /dev/null +++ b/sandbox/Benchmark/Utf8CharConversionBenchmark.cs @@ -0,0 +1,37 @@ +using System.Text; +using BenchmarkDotNet.Attributes; +using Lua; + +[MemoryDiagnoser] +[ShortRunJob] +public class Utf8CharConversionBenchmark +{ + [Params("Ascii", "SparseUnicode", "Japanese", "Mixed", "InvalidUtf16")] + public string Corpus { get; set; } = "Ascii"; + + [Params(4096)] + public int Length { get; set; } + + string source = default!; + + [GlobalSetup] + public void Setup() + { + var unit = Corpus switch + { + "Ascii" => "hello world 1234 ", + "SparseUnicode" => new string('a', Length - 1) + "ế", + "Japanese" => "こんにちは世界", + "Mixed" => "aế😀漢字 ", + "InvalidUtf16" => "a\uD800b\uDC00c", + _ => throw new ArgumentOutOfRangeException(nameof(Corpus)), + }; + source = string.Concat(Enumerable.Repeat(unit, Length / unit.Length + 1))[..Length]; + } + + [Benchmark(Baseline = true)] + public string ViaBytes() => LuaByteString.FromBytes(Encoding.UTF8.GetBytes(source)); + + [Benchmark] + public string DirectChars() => LuaByteString.FromString(source); +} diff --git a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs index 5003d6fb..81856e66 100644 --- a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs +++ b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs @@ -21,10 +21,7 @@ public static ValueTask ExecuteAsync( } var module = luaAssetBase.GetModule(name); - var closure = - module.Type == LuaModuleType.Bytes - ? state.Load(module.ReadBytes(), module.Name) - : state.Load(module.ReadText(), module.Name); + var closure = state.Load(module); return state.ExecuteAsync(closure, buffer, cancellationToken); } @@ -35,7 +32,13 @@ public static ValueTask ExecuteAsync( CancellationToken cancellationToken = default ) { - return state.ExecuteAsync(luaAssetBase, name, cancellationToken); + if (luaAssetBase == null) + { + throw new ArgumentNullException(nameof(luaAssetBase)); + } + + var module = luaAssetBase.GetModule(name); + return state.ExecuteAsync(state.Load(module), cancellationToken); } } } diff --git a/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs b/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs index 0c3a4b98..2c921dce 100644 --- a/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs +++ b/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs @@ -9,20 +9,18 @@ namespace Lua.Unity public class UnityStandardIO : ILuaStandardIO { public UnityStandardIO(ILuaStream input = null) + : this(LuaStringMode.Utf16, input) { } + + public UnityStandardIO(LuaStringMode stringMode, ILuaStream input = null) { - if (input != null) - { - Input = input; - } - else - { - Input = new DummyInputStream(); - } + Input = input ?? new DummyInputStream(); + Output = new DebugLogStream(false, stringMode); + Error = new DebugLogStream(true, stringMode); } public ILuaStream Input { get; } - public ILuaStream Output { get; } = new DebugLogStream(false); - public ILuaStream Error { get; } = new DebugLogStream(true); + public ILuaStream Output { get; } + public ILuaStream Error { get; } } internal class DummyInputStream : ILuaStream @@ -36,8 +34,12 @@ public void Dispose() { } public class DebugLogStream : ILuaStream { public DebugLogStream(bool isError = false) + : this(isError, LuaStringMode.Utf16) { } + + public DebugLogStream(bool isError, LuaStringMode stringMode) { IsError = isError; + this.stringMode = stringMode; } public bool IsError { get; } = false; @@ -45,6 +47,7 @@ public DebugLogStream(bool isError = false) public LuaFileOpenMode Mode => LuaFileOpenMode.Write; private readonly StringBuilder stringBuilder = new(); + private readonly LuaStringMode stringMode; ValueTask ILuaStream.WriteAsync( ReadOnlyMemory content, @@ -60,6 +63,8 @@ ValueTask ILuaStream.FlushAsync(CancellationToken cancellationToken) if (stringBuilder.Length > 0) { var message = stringBuilder.ToString(); + if (stringMode == LuaStringMode.Bytes) + message = LuaByteString.ToString(message); if (IsError) UnityEngine.Debug.LogError(message); else diff --git a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs index 074c1caa..6cd18cfd 100644 --- a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs +++ b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs @@ -12,6 +12,11 @@ static class BomUtility static ReadOnlySpan BomUtf32Little => [0xFF, 0xFE, 0x00, 0x00]; + public static ReadOnlySpan SkipUtf8Bom(ReadOnlySpan text) + { + return text.StartsWith(BomUtf8) ? text.Slice(BomUtf8.Length) : text; + } + /// /// Removes the BOM from the beginning of the text and returns the encoding. /// Supported encodings are UTF-8, UTF-16 (little and big endian), and UTF-32 (little endian). diff --git a/src/Lua/Exceptions.cs b/src/Lua/Exceptions.cs index a95a2b79..82555a12 100644 --- a/src/Lua/Exceptions.cs +++ b/src/Lua/Exceptions.cs @@ -2,6 +2,7 @@ using Lua.CodeAnalysis; using Lua.CodeAnalysis.Syntax; using Lua.Internal; +using Lua.Platforms; using Lua.Runtime; namespace Lua; @@ -490,9 +491,13 @@ internal void Forget() State = null; } - internal string MinimalMessage() + internal string MinimalMessage(LuaPlatform platform) { - var message = InnerException?.ToString() ?? ErrorObject.ToString(); + var message = InnerException is { } inner + ? platform.StringMode == LuaStringMode.Bytes + ? platform.GetErrorMessage(inner) + : inner.ToString() + : ErrorObject.ToString(); if (level <= 0) { return message; diff --git a/src/Lua/ILuaModuleLoader.cs b/src/Lua/ILuaModuleLoader.cs index db998eff..20cf4182 100644 --- a/src/Lua/ILuaModuleLoader.cs +++ b/src/Lua/ILuaModuleLoader.cs @@ -2,7 +2,12 @@ namespace Lua; public interface ILuaModuleLoader { + /// Checks a module name expressed as normal .NET text, including in byte-string mode. bool Exists(string moduleName); + + /// + /// Loads a host module. Names and text source use .NET text; byte source is kept raw. + /// ValueTask LoadAsync( string moduleName, CancellationToken cancellationToken = default diff --git a/src/Lua/IO/ConsoleStandardIO.cs b/src/Lua/IO/ConsoleStandardIO.cs index 8d532ecb..0834dc6b 100644 --- a/src/Lua/IO/ConsoleStandardIO.cs +++ b/src/Lua/IO/ConsoleStandardIO.cs @@ -5,24 +5,43 @@ namespace Lua.IO; /// /// Default implementation of ILuaStandardIO using Console /// -public sealed class ConsoleStandardIO : ILuaStandardIO +/// +/// Selects UTF-8 text I/O for , or raw byte I/O for . +/// Use the same mode as . +/// +public sealed class ConsoleStandardIO(LuaStringMode stringMode) : ILuaStandardIO { + public ConsoleStandardIO() + : this(LuaStringMode.Utf16) { } + ILuaStream? standardInput; ILuaStream? standardOutput; ILuaStream? standardError; public ILuaStream Input => standardInput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardInput(), LuaFileOpenMode.Read) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardInput(), + LuaFileOpenMode.Read, + stringMode + ) ); public ILuaStream Output => standardOutput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardOutput(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardOutput(), + LuaFileOpenMode.Write, + stringMode + ) ); public ILuaStream Error => standardError ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardError(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardError(), + LuaFileOpenMode.Write, + stringMode + ) ); } diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 4fb6b09a..8052e6b2 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -1,7 +1,15 @@ namespace Lua.IO; -public sealed class FileSystem(string? baseDirectory = null) : ILuaFileSystem +/// Directory that relative paths are resolved against. +/// +/// Selects UTF-8 text I/O for , or raw byte I/O for . +/// Use the same mode as . +/// +public sealed class FileSystem(string? baseDirectory, LuaStringMode stringMode) : ILuaFileSystem { + public FileSystem(string? baseDirectory = null) + : this(baseDirectory, LuaStringMode.Utf16) { } + public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory(); public static (FileMode, FileAccess access) GetFileMode(LuaFileOpenMode luaFileOpenMode) @@ -60,7 +68,7 @@ CancellationToken cancellationToken stream = File.Open(path, mode, access, FileShare.ReadWrite | FileShare.Delete); } - ILuaStream wrapper = new LuaStream(openMode, stream); + ILuaStream wrapper = new LuaStream(openMode, stream, stringMode); if (openMode == LuaFileOpenMode.AppendUpdate) { @@ -136,7 +144,8 @@ public ValueTask OpenTempFileStream(CancellationToken cancellationTo return new( new LuaStream( LuaFileOpenMode.WriteUpdate, - File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite) + File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite), + stringMode ) ); } diff --git a/src/Lua/IO/ILuaFileLoader.cs b/src/Lua/IO/ILuaFileLoader.cs index f5abb58d..14c7ba6e 100644 --- a/src/Lua/IO/ILuaFileLoader.cs +++ b/src/Lua/IO/ILuaFileLoader.cs @@ -1,5 +1,6 @@ namespace Lua.IO; +/// Loads files by .NET path; stream contents use the state's Lua string representation. public interface ILuaFileLoader { bool Exists(string path); diff --git a/src/Lua/IO/ILuaFileSystem.cs b/src/Lua/IO/ILuaFileSystem.cs index 16f5e9ff..dfd7eec9 100644 --- a/src/Lua/IO/ILuaFileSystem.cs +++ b/src/Lua/IO/ILuaFileSystem.cs @@ -1,5 +1,9 @@ namespace Lua.IO; +/// +/// Host file operations. Paths and temporary filenames are .NET text in every string mode. +/// Stream contents use the Lua representation configured for the state. +/// public interface ILuaFileSystem { bool IsReadable(string path); diff --git a/src/Lua/IO/ILuaStream.cs b/src/Lua/IO/ILuaStream.cs index ed6ab400..7a02b989 100644 --- a/src/Lua/IO/ILuaStream.cs +++ b/src/Lua/IO/ILuaStream.cs @@ -74,6 +74,15 @@ static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode) return new LuaStream(openMode, stream); } + static ILuaStream CreateFromStream( + Stream stream, + LuaFileOpenMode openMode, + LuaStringMode stringMode + ) + { + return new LuaStream(openMode, stream, stringMode); + } + static ILuaStream CreateFromString(string content) { return new StringStream(content); diff --git a/src/Lua/IO/LuaByteEncoding.cs b/src/Lua/IO/LuaByteEncoding.cs new file mode 100644 index 00000000..bd27ca6c --- /dev/null +++ b/src/Lua/IO/LuaByteEncoding.cs @@ -0,0 +1,122 @@ +using System.Text; + +namespace Lua.IO; + +/// +/// Maps one byte to one (0x00-0xFF) and back, so that a Lua string can carry +/// arbitrary bytes (for example UTF-8 text) with the same #s, string.sub and +/// string.byte results as the reference C implementation. +/// Characters above 0xFF cannot be represented and are written as '?'. +/// +sealed class LuaByteEncoding : Encoding +{ + public static LuaByteEncoding Instance { get; } = new(); + + public override int GetByteCount(char[] chars, int index, int count) => count; + + public override int GetByteCount(string s) => s.Length; + + public override int GetByteCount(ReadOnlySpan chars) => chars.Length; + + // Encoding's pointer defaults copy through temporary arrays. Its string, + // encoder and decoder APIs use these overloads even when span overrides exist. + public override unsafe int GetByteCount(char* chars, int count) + { + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + return new ReadOnlySpan(chars, count).Length; + } + + public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) + { + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + return GetBytes(new ReadOnlySpan(chars, charCount), new Span(bytes, byteCount)); + } + + public override int GetBytes( + char[] chars, + int charIndex, + int charCount, + byte[] bytes, + int byteIndex + ) => GetBytes(chars.AsSpan(charIndex, charCount), bytes.AsSpan(byteIndex)); + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + if (bytes.Length < chars.Length) + { + throw new ArgumentException("Destination is too small.", nameof(bytes)); + } + + for (var i = 0; i < chars.Length; i++) + { + var c = chars[i]; + bytes[i] = c <= 0xFF ? (byte)c : (byte)'?'; + } + + return chars.Length; + } + + public override int GetCharCount(byte[] bytes, int index, int count) => count; + + public override int GetCharCount(ReadOnlySpan bytes) => bytes.Length; + + public override unsafe int GetCharCount(byte* bytes, int count) + { + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + return new ReadOnlySpan(bytes, count).Length; + } + + public override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount) + { + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + return GetChars(new ReadOnlySpan(bytes, byteCount), new Span(chars, charCount)); + } + + public override int GetChars( + byte[] bytes, + int byteIndex, + int byteCount, + char[] chars, + int charIndex + ) => GetChars(bytes.AsSpan(byteIndex, byteCount), chars.AsSpan(charIndex)); + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + if (chars.Length < bytes.Length) + { + throw new ArgumentException("Destination is too small.", nameof(chars)); + } + + for (var i = 0; i < bytes.Length; i++) + { + chars[i] = (char)bytes[i]; + } + + return bytes.Length; + } + + public override string GetString(byte[] bytes, int index, int count) => + string.Create( + count, + (bytes, index), + static (span, state) => + { + for (var i = 0; i < span.Length; i++) + { + span[i] = (char)state.bytes[state.index + i]; + } + } + ); + + public override int GetMaxByteCount(int charCount) => charCount; + + public override int GetMaxCharCount(int byteCount) => byteCount; +} diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index a7c77369..7bdbfabc 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -4,8 +4,13 @@ namespace Lua.IO; -public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaStream, ILuaByteStream +public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, LuaStringMode stringMode) + : ILuaStream, + ILuaByteStream { + public LuaStream(LuaFileOpenMode mode, Stream innerStream) + : this(mode, innerStream, LuaStringMode.Utf16) { } + Utf8Reader? reader; ulong flushSize = ulong.MaxValue; ulong nextFlushSize = ulong.MaxValue; @@ -19,14 +24,14 @@ public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaSt public ValueTask ReadLineAsync(bool keepEol, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(stringMode); return new(reader.ReadLine(innerStream, keepEol)); } public ValueTask ReadAllAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(stringMode); var text = reader.ReadToEnd(innerStream); return new(text); } @@ -57,7 +62,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadAsync(int count, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(stringMode); if (count == 0) { return new(reader.IsEndOfStream(innerStream) ? null : string.Empty); @@ -69,7 +74,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadNumberAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(stringMode); // Use the Utf8Reader's ReadNumber method which handles positioning correctly var numberStr = reader.ReadNumber(innerStream); @@ -93,23 +98,34 @@ public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cance } using PooledArray byteBuffer = new(4096); - var encoder = Encoding.UTF8.GetEncoder(); + var bytes = byteBuffer.AsSpan(); + var encoder = stringMode == LuaStringMode.Bytes ? null : Encoding.UTF8.GetEncoder(); var remainingChars = buffer.Span; var totalBytes = 0; while (!remainingChars.IsEmpty) { - encoder.Convert( - remainingChars, - byteBuffer.AsSpan(), - flush: true, - out var charsUsed, - out var bytesUsed, - out _ - ); + int charsUsed, + bytesUsed; + if (stringMode == LuaStringMode.Bytes) + { + charsUsed = Math.Min(remainingChars.Length, bytes.Length); + bytesUsed = LuaByteEncoding.Instance.GetBytes(remainingChars[..charsUsed], bytes); + } + else + { + encoder!.Convert( + remainingChars, + bytes, + flush: true, + out charsUsed, + out bytesUsed, + out _ + ); + } if (bytesUsed > 0) { - innerStream.Write(byteBuffer.AsSpan()[..bytesUsed]); + innerStream.Write(bytes[..bytesUsed]); totalBytes += bytesUsed; } diff --git a/src/Lua/Internal/StringHelper.cs b/src/Lua/Internal/StringHelper.cs index 25fe5deb..bf543bf9 100644 --- a/src/Lua/Internal/StringHelper.cs +++ b/src/Lua/Internal/StringHelper.cs @@ -379,6 +379,66 @@ public static Regex ToRegex(ReadOnlySpan pattern) return new(builder.ToString()); } + /// + /// toupper in the C locale: only 'a'-'z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToUpperAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'a' && s[i] <= 'z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'a' && c <= 'z' ? (char)(c - 32) : c; + } + } + ); + } + + /// + /// tolower in the C locale: only 'A'-'Z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToLowerAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'A' && s[i] <= 'Z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'A' && c <= 'Z' ? (char)(c + 32) : c; + } + } + ); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNumber(char c) { diff --git a/src/Lua/Internal/Utf8CharEncoder.cs b/src/Lua/Internal/Utf8CharEncoder.cs new file mode 100644 index 00000000..0a464a1a --- /dev/null +++ b/src/Lua/Internal/Utf8CharEncoder.cs @@ -0,0 +1,76 @@ +using System.Runtime.InteropServices; + +namespace Lua.Internal; + +// Encodes UTF-16 text as UTF-8, storing each output byte in one char. +// The destination must have at least Encoding.UTF8.GetByteCount(source) chars. +static class Utf8CharEncoder +{ + public static int Encode(ReadOnlySpan source, Span destination) + { + var written = 0; + for (var i = 0; i < source.Length; i++) + { + uint value = source[i]; + if (value <= 0x7F) + { + // Copy four ASCII code units together, including within mixed text. + if (source.Length - i >= 4) + { + var block = MemoryMarshal.Read( + MemoryMarshal.AsBytes(source.Slice(i, 4)) + ); + if ((block & 0xFF80FF80FF80FF80UL) == 0) + { + BitConverter.TryWriteBytes( + MemoryMarshal.AsBytes(destination.Slice(written, 4)), + block + ); + written += 4; + i += 3; + continue; + } + } + destination[written++] = (char)value; + } + else if (value <= 0x7FF) + { + destination[written++] = (char)(0xC0 | (value >> 6)); + destination[written++] = (char)(0x80 | (value & 0x3F)); + } + else + { + if (char.IsSurrogate((char)value)) + { + if ( + char.IsHighSurrogate((char)value) + && i + 1 < source.Length + && char.IsLowSurrogate(source[i + 1]) + ) + { + value = (uint)char.ConvertToUtf32(source[i], source[i + 1]); + i++; + } + else + { + // Match Encoding.UTF8's replacement of each unpaired surrogate. + value = 0xFFFD; + } + } + + if (value > 0xFFFF) + { + destination[written++] = (char)(0xF0 | (value >> 18)); + destination[written++] = (char)(0x80 | ((value >> 12) & 0x3F)); + } + else + { + destination[written++] = (char)(0xE0 | (value >> 12)); + } + destination[written++] = (char)(0x80 | ((value >> 6) & 0x3F)); + destination[written++] = (char)(0x80 | (value & 0x3F)); + } + } + return written; + } +} diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index 6c23e0e2..ca3a78da 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; using Lua.CodeAnalysis.Compilation; +using Lua.IO; namespace Lua.Internal; @@ -13,14 +14,23 @@ sealed class Utf8Reader internal static bool scratchBufferUsed; readonly byte[] buffer; + readonly Encoding encoding; + readonly bool byteStrings; int bufPos, bufLen; Decoder? decoder; const int ThreadStaticBufferSize = 1024; - public Utf8Reader() + /// + /// In byte mode, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, + /// like the reference C implementation. Otherwise the stream is decoded as UTF-8. + /// + public Utf8Reader(LuaStringMode stringMode = LuaStringMode.Utf16) { + byteStrings = stringMode == LuaStringMode.Bytes; + encoding = byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8; + if (scratchBufferUsed) { buffer = new byte[ThreadStaticBufferSize]; @@ -46,7 +56,7 @@ public Utf8Reader() if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -54,7 +64,10 @@ public Utf8Reader() } Span span = new(buffer, bufPos, bufLen - bufPos); - var idx = span.IndexOfAny((byte)'\r', (byte)'\n'); + // Raw Lua byte streams end a line at LF; CR remains part of the data. + var idx = byteStrings + ? span.IndexOf((byte)'\n') + : span.IndexOfAny((byte)'\r', (byte)'\n'); if (idx >= 0) { @@ -92,7 +105,7 @@ ref lineLen } } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } else { @@ -107,7 +120,7 @@ ref lineLen return null; } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } finally { @@ -126,7 +139,7 @@ public string ReadToEnd(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -143,7 +156,7 @@ public string ReadToEnd(Stream stream) return ""; } - return Encoding.UTF8.GetString(resultBuffer, 0, len); + return encoding.GetString(resultBuffer, 0, len); } finally { @@ -209,7 +222,7 @@ public byte ReadByte(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -218,7 +231,17 @@ public byte ReadByte(Stream stream) ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); Span charSpan = new(resultBuffer, len, charCount - len); - decoder ??= Encoding.UTF8.GetDecoder(); + if (byteStrings) + { + var count = Math.Min(byteSpan.Length, charSpan.Length); + LuaByteEncoding.Instance.GetChars(byteSpan[..count], charSpan); + len += count; + bufPos += count; + dataRead = true; + continue; + } + + decoder ??= encoding.GetDecoder(); decoder.Convert( byteSpan, charSpan, @@ -269,6 +292,8 @@ static void AppendToBuffer(ref byte[] buffer, ReadOnlySpan segment, ref in length += segment.Length; } + static bool IsAsciiSpace(char c) => c == ' ' || (c >= '\t' && c <= '\r'); + public void Clear() { bufPos = 0; @@ -296,7 +321,7 @@ public void Clear() } var c = (char)b; - if (!char.IsWhiteSpace(c)) + if (byteStrings ? !IsAsciiSpace(c) : !char.IsWhiteSpace(c)) { break; } diff --git a/src/Lua/LuaByteString.cs b/src/Lua/LuaByteString.cs new file mode 100644 index 00000000..332a0d33 --- /dev/null +++ b/src/Lua/LuaByteString.cs @@ -0,0 +1,54 @@ +using System.Text; +using Lua.Internal; +using Lua.IO; + +namespace Lua; + +/// +/// Conversions between .NET strings and Lua byte strings (see +/// ), where every holds one byte. +/// +public static class LuaByteString +{ + /// + /// Converts raw bytes (for example UTF-8 text) to a byte string. + /// + public static string FromBytes(ReadOnlySpan bytes) + { + return LuaByteEncoding.Instance.GetString(bytes); + } + + /// + /// Converts a byte string back to raw bytes. Chars above 0xFF are written as '?'. + /// + public static byte[] ToBytes(string byteString) + { + return LuaByteEncoding.Instance.GetBytes(byteString); + } + + /// + /// Converts a .NET string to a byte string holding its UTF-8 bytes. + /// + public static string FromString(string value) + { + var count = Encoding.UTF8.GetByteCount(value); + if (count == value.Length) + { + return value; // ASCII already has the byte-string representation. + } + return string.Create( + count, + value, + static (destination, source) => Utf8CharEncoder.Encode(source, destination) + ); + } + + /// + /// Decodes the UTF-8 bytes held by a byte string into a .NET string. + /// Invalid UTF-8 is replaced with U+FFFD. + /// + public static string ToString(string byteString) + { + return Encoding.UTF8.GetString(ToBytes(byteString)); + } +} diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 290ba387..3b841d48 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -1,8 +1,10 @@ using System.Buffers; using System.Runtime.CompilerServices; +using System.Text; using Lua.CodeAnalysis.Compilation; using Lua.Internal; using Lua.Internal.CompilerServices; +using Lua.IO; using Lua.Platforms; using Lua.Runtime; @@ -592,7 +594,40 @@ public async ValueTask RunAsync( } } - public unsafe LuaClosure Load( + /// + /// Loads .NET source text and a .NET chunk name. In byte-string mode, text is + /// encoded as UTF-8. Use the byte overload for source that is already encoded. + /// + public LuaClosure Load(ReadOnlySpan chunk, string chunkName, LuaTable? environment = null) + { + var name = Platform.ToLuaString(chunkName); + if (Platform.StringMode != LuaStringMode.Bytes) + { + return LoadCore(chunk, name, environment); + } + + var byteCount = Encoding.UTF8.GetByteCount(chunk); + if (byteCount > 4 && chunk[0] == '\e') + { + throw new Exception("attempt to load a binary chunk (mode is 't')"); + } + // Match the byte-source loader's handling of an initial UTF-8 BOM. + if (!chunk.IsEmpty && chunk[0] == '\uFEFF') + { + chunk = chunk[1..]; + byteCount -= 3; + } + if (byteCount == chunk.Length) + { + return LoadCore(chunk, name, environment); + } + using PooledArray chars = new(byteCount); + var count = Utf8CharEncoder.Encode(chunk, chars.AsSpan()); + return LoadCore(chars.AsSpan()[..count], name, environment); + } + + // Source and chunk name already use this state's Lua string representation. + internal unsafe LuaClosure LoadCore( ReadOnlySpan chunk, string chunkName, LuaTable? environment = null @@ -607,12 +642,32 @@ public unsafe LuaClosure Load( return new(this, prototype, environment); } + /// + /// Loads encoded source bytes with a .NET chunk name. Byte-string mode preserves + /// the source bytes, except for an initial UTF-8 BOM. + /// public LuaClosure Load( ReadOnlySpan chunk, string? chunkName = null, string mode = "bt", LuaTable? environment = null ) + { + return LoadCore( + chunk, + chunkName is null ? null : Platform.ToLuaString(chunkName), + mode, + environment + ); + } + + // Source is encoded bytes; an explicit chunk name already uses the Lua string representation. + internal LuaClosure LoadCore( + ReadOnlySpan chunk, + string? chunkName = null, + string mode = "bt", + LuaTable? environment = null + ) { static bool AllowsMode(string mode, char chunkMode) { @@ -637,7 +692,17 @@ static bool AllowsMode(string mode, char chunkMode) throw new Exception("attempt to load a text chunk (mode is 'b')"); } - chunk = BomUtility.GetEncodingFromBytes(chunk, out var encoding); + Encoding encoding; + if (Platform.StringMode == LuaStringMode.Bytes) + { + // Byte strings: like luaL_loadfile, skip a UTF-8 BOM and keep every other byte as one char. + chunk = BomUtility.SkipUtf8Bom(chunk); + encoding = LuaByteEncoding.Instance; + } + else + { + chunk = BomUtility.GetEncodingFromBytes(chunk, out encoding); + } var charCount = encoding.GetCharCount(chunk); var pooled = ArrayPool.Shared.Rent(charCount); @@ -647,7 +712,7 @@ static bool AllowsMode(string mode, char chunkMode) encoding.GetChars(chunk, chars); chunkName ??= chars.ToString(); - return Load(chars, chunkName, environment); + return LoadCore(chars, chunkName, environment); } finally { diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index 1b751994..93e827b5 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -11,6 +11,18 @@ namespace Lua; public static class LuaStateExtensions { + /// + /// Loads a host module. Its name and text source are .NET text; byte source is + /// preserved. In byte-string mode, text source is encoded as UTF-8 before parsing. + /// + public static LuaClosure Load(this LuaState state, LuaModule module) + { + return module.Type == LuaModuleType.Bytes + ? state.Load(module.ReadBytes(), module.Name) + : state.Load(module.ReadText(), module.Name); + } + + /// Loads a file whose name is .NET text, including in byte-string mode. public static async ValueTask LoadFileAsync( this LuaState state, string fileName, @@ -19,7 +31,7 @@ public static async ValueTask LoadFileAsync( CancellationToken cancellationToken ) { - var name = "@" + fileName; + var name = state.Platform.ToLuaString("@" + fileName); using var stream = await state.GlobalState.Platform.FileSystem.Open( fileName, LuaFileOpenMode.Read, @@ -37,7 +49,7 @@ CancellationToken cancellationToken if (firstByte < 0) { - closure = state.Load(ReadOnlySpan.Empty, name, mode, environment); + closure = state.LoadCore(ReadOnlySpan.Empty, name, mode, environment); } else { @@ -45,7 +57,7 @@ CancellationToken cancellationToken source.GetSpan(1)[0] = (byte)firstByte; source.Advance(1); await byteStream.ReadBytesAsync(source, cancellationToken); - closure = state.Load(source.WrittenSpan, name, mode, environment); + closure = state.LoadCore(source.WrittenSpan, name, mode, environment); } } else if (!mode.Contains('t')) @@ -55,12 +67,13 @@ CancellationToken cancellationToken else { var source = await stream.ReadAllAsync(cancellationToken); - closure = state.Load(source, name, environment); + closure = state.LoadCore(source, name, environment); } return closure; } + /// Executes .NET source text with an optional .NET chunk name. public static ValueTask DoStringAsync( this LuaState state, string source, @@ -73,6 +86,7 @@ public static ValueTask DoStringAsync( return ExecuteAsync(state, closure, results, cancellationToken); } + /// Executes .NET source text with an optional .NET chunk name. public static ValueTask DoStringAsync( this LuaState state, string source, @@ -107,6 +121,7 @@ public static ValueTask ExecuteAsync( return ExecuteAsync(state, closure, cancellationToken); } + /// Executes a file whose path is .NET text, including in byte-string mode. [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] public static async ValueTask DoFileAsync( this LuaState state, @@ -122,6 +137,7 @@ public static async ValueTask DoFileAsync( return results.Count; } + /// Executes a file whose path is .NET text, including in byte-string mode. [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] public static async ValueTask DoFileAsync( this LuaState state, diff --git a/src/Lua/LuaStringMode.cs b/src/Lua/LuaStringMode.cs new file mode 100644 index 00000000..35a8da0a --- /dev/null +++ b/src/Lua/LuaStringMode.cs @@ -0,0 +1,11 @@ +namespace Lua; + +/// The string representation used by a Lua state and its streams. +public enum LuaStringMode +{ + /// Strings contain UTF-16 code units. This is the default. + Utf16, + + /// Each char holds one byte, including arbitrary binary data. + Bytes, +} diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index 009082cb..491b2f01 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -16,6 +16,38 @@ public record LuaPlatform( TimeProvider TimeProvider ) { + /// + /// In mode, every holds one byte (0x00-0xFF), + /// so #s, string.len, string.sub and string.byte count bytes exactly + /// like the reference C implementation (UTF-8 text such as "ế" has length 3 instead of 1). + /// Character classes (%a, %w, ...), string.upper and string.lower follow the + /// C locale (ASCII only) and string.char rejects values above 255. + /// The and of the platform must read and write + /// bytes as well, which does. The default is . + /// Use to convert between .NET strings and byte strings. + /// + public LuaStringMode StringMode { get; init; } + + // Converts a .NET string (file name, environment value, error text) into a Lua string for this platform. + internal string ToLuaString(string value) + { + return StringMode == LuaStringMode.Bytes ? LuaByteString.FromString(value) : value; + } + + internal string ToHostString(string value) + { + return StringMode == LuaStringMode.Bytes ? LuaByteString.ToString(value) : value; + } + + internal string GetErrorMessage(Exception exception) + { + // Lua diagnostics already contain Lua strings (including chunk names). + // Only host exceptions carry .NET text that needs UTF-8 conversion. + return exception is LuaRuntimeException or LuaParseException or LuaCompileException + ? exception.Message + : ToLuaString(exception.Message); + } + /// /// Standard console platform implementation. /// Uses real file system, console I/O, and system operations. @@ -27,4 +59,19 @@ TimeProvider TimeProvider StandardIO: new ConsoleStandardIO(), TimeProvider: TimeProvider.System ); + + /// + /// Same as , but with set to : files and standard I/O + /// are read and written as raw bytes. + /// + public static LuaPlatform DefaultByteStrings { get; } = + new( + FileSystem: new FileSystem(baseDirectory: null, stringMode: LuaStringMode.Bytes), + OsEnvironment: new SystemOsEnvironment(), + StandardIO: new ConsoleStandardIO(LuaStringMode.Bytes), + TimeProvider: TimeProvider.System + ) + { + StringMode = LuaStringMode.Bytes, + }; } diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 5d42540d..4c2a8514 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -131,7 +131,12 @@ CancellationToken cancellationToken { var arg0 = context.GetArgument(0); context.State.Stack.PopUntil(context.ReturnFrameBase); - var closure = await context.State.LoadFileAsync(arg0, "bt", null, cancellationToken); + var closure = await context.State.LoadFileAsync( + context.State.Platform.ToHostString(arg0), + "bt", + null, + cancellationToken + ); return await context.State.RunAsync(closure, cancellationToken); } @@ -222,7 +227,12 @@ CancellationToken cancellationToken // do not use LuaState.DoFileAsync as it uses the newExecutionContext try { - var closure = await context.State.LoadFileAsync(arg0, mode, null, cancellationToken); + var closure = await context.State.LoadFileAsync( + context.State.Platform.ToHostString(arg0), + mode, + null, + cancellationToken + ); if (hasEnvironment) { closure.SetEnvironment(environment); @@ -232,7 +242,7 @@ CancellationToken cancellationToken } catch (Exception ex) { - return context.Return(LuaValue.Nil, ex.Message); + return context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex)); } } @@ -260,7 +270,7 @@ CancellationToken cancellationToken throw new Exception("attempt to load a text chunk (mode is 'b')"); } - var closure = context.State.Load(str, name ?? str, null); + var closure = context.State.LoadCore(str, name ?? str, null); if (hasEnvironment) { closure.SetEnvironment(environment); @@ -286,7 +296,7 @@ CancellationToken cancellationToken } catch (Exception ex) { - return new(context.Return(LuaValue.Nil, ex.Message)); + return new(context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex))); } } @@ -384,12 +394,12 @@ CancellationToken cancellationToken } using PooledList builder = new(); - var message = luaEx.MinimalMessage(); + var message = luaEx.MinimalMessage(context.State.Platform); luaEx.Forget(); return context.Return(false, message); } default: - return context.Return(false, ex.Message); + return context.Return(false, context.State.Platform.GetErrorMessage(ex)); } } } @@ -768,12 +778,17 @@ CancellationToken cancellationToken if (ex is LuaRuntimeException luaEx) { + var error = + state.Platform.StringMode == LuaStringMode.Bytes + && luaEx.InnerException is { } inner + ? state.Platform.GetErrorMessage(inner) + : luaEx.ErrorObject; luaEx.Forget(); - state.Push(luaEx.ErrorObject); + state.Push(error); } else { - state.Push(ex.Message); + state.Push(state.Platform.GetErrorMessage(ex)); } // invoke error handler diff --git a/src/Lua/Standard/FileHandle.cs b/src/Lua/Standard/FileHandle.cs index 3fc11ce2..835c2485 100644 --- a/src/Lua/Standard/FileHandle.cs +++ b/src/Lua/Standard/FileHandle.cs @@ -146,7 +146,11 @@ public async ValueTask Close(CancellationToken cancellationToken) } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } ); @@ -164,7 +168,11 @@ public async ValueTask Close(CancellationToken cancellationToken) } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } ); @@ -245,7 +253,13 @@ static async (context, cancellationToken) => } catch (IOException ex) { - return new(context.Return(LuaValue.Nil, ex.Message, ex.HResult)); + return new( + context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ) + ); } } ); diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index c8ed4f3e..a67d8832 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -44,7 +44,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -62,7 +66,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -87,7 +95,7 @@ CancellationToken cancellationToken else { var stream = await context.GlobalState.Platform.FileSystem.Open( - arg.ToString(), + context.State.Platform.ToHostString(arg.ToString()), LuaFileOpenMode.Read, cancellationToken ); @@ -213,7 +221,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -251,7 +263,7 @@ CancellationToken cancellationToken } var stream = await context.GlobalState.Platform.FileSystem.Open( - arg.ToString(), + context.State.Platform.ToHostString(arg.ToString()), LuaFileOpenMode.Write, cancellationToken ); diff --git a/src/Lua/Standard/Internal/IOHelper.cs b/src/Lua/Standard/Internal/IOHelper.cs index 45738bbd..39a44b01 100644 --- a/src/Lua/Standard/Internal/IOHelper.cs +++ b/src/Lua/Standard/Internal/IOHelper.cs @@ -23,7 +23,7 @@ CancellationToken cancellationToken try { var stream = await state.GlobalState.Platform.FileSystem.Open( - fileName, + state.Platform.ToHostString(fileName), fileMode, cancellationToken ); @@ -39,7 +39,7 @@ CancellationToken cancellationToken } state.Stack.Push(LuaValue.Nil); - state.Stack.Push(ex.Message); + state.Stack.Push(state.Platform.ToLuaString(ex.Message)); state.Stack.Push(ex.HResult); return 3; } @@ -88,7 +88,7 @@ await file.WriteAsync( context.State.Stack.PopUntil(context.ReturnFrameBase); var stack = context.State.Stack; stack.Push(LuaValue.Nil); - stack.Push(ex.Message); + stack.Push(context.State.Platform.ToLuaString(ex.Message)); stack.Push(ex.HResult); return 3; } @@ -200,7 +200,7 @@ CancellationToken cancellationToken stack.PopUntil(top); stack.Push(LuaValue.Nil); - stack.Push(ex.Message); + stack.Push(state.Platform.ToLuaString(ex.Message)); stack.Push(ex.HResult); return 3; } diff --git a/src/Lua/Standard/Internal/MatchState.cs b/src/Lua/Standard/Internal/MatchState.cs index 586f87f9..3aed4afb 100644 --- a/src/Lua/Standard/Internal/MatchState.cs +++ b/src/Lua/Standard/Internal/MatchState.cs @@ -18,6 +18,9 @@ internal struct Capture public bool IsUnfinished => Len == CapUnfinished; } + // Byte strings use the C locale: classes only match ASCII, bytes 0x80-0xFF never belong to a class. + readonly bool byteStrings = state.Platform.StringMode == LuaStringMode.Bytes; + public readonly LuaState State = state; public readonly string Source = source; public readonly string Pattern = pattern; @@ -452,8 +455,13 @@ int ClassEnd(ReadOnlySpan pattern, int pIdx) } } - static bool MatchClass(char c, char cl) + bool MatchClass(char c, char cl) { + if (byteStrings) + { + return MatchClassAscii(c, cl); + } + bool res; switch (char.ToLower(cl)) { @@ -503,12 +511,62 @@ static bool MatchClass(char c, char cl) return char.IsLower(cl) ? res : !res; } + static bool MatchClassAscii(char c, char cl) + { + bool res; + switch (char.ToLowerInvariant(cl)) + { + case 'a': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + break; + case 'c': + res = c < ' ' || c == 127; + break; + case 'd': + res = c >= '0' && c <= '9'; + break; + case 'g': + res = c > ' ' && c < 127; + break; + case 'l': + res = c >= 'a' && c <= 'z'; + break; + case 'p': + res = + c + is (>= '!' and <= '/') + or (>= ':' and <= '@') + or (>= '[' and <= '`') + or (>= '{' and <= '~'); + break; + case 's': + res = c == ' ' || (c >= '\t' && c <= '\r'); + break; + case 'u': + res = c >= 'A' && c <= 'Z'; + break; + case 'w': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + break; + case 'x': + res = IsHexDigit(c); + break; + case 'z': + res = c == '\0'; + break; + default: + return cl == c; + } + + return cl is >= 'a' and <= 'z' ? res : !res; + } + static bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } - static bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) + bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) { var sig = true; if (pIdx + 1 < pattern.Length && pattern[pIdx + 1] == '^') diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index 994be706..788b1dbe 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -31,13 +31,11 @@ CancellationToken cancellationToken { LuaFunction loader; var moduleLoader = context.GlobalState.ModuleLoader; - if (moduleLoader != null && moduleLoader.Exists(arg0)) + var hostName = moduleLoader is null ? arg0 : context.State.Platform.ToHostString(arg0); + if (moduleLoader != null && moduleLoader.Exists(hostName)) { - var module = await moduleLoader.LoadAsync(arg0, cancellationToken); - loader = - module.Type == LuaModuleType.Bytes - ? context.State.Load(module.ReadBytes(), module.Name) - : context.State.Load(module.ReadText(), module.Name); + var module = await moduleLoader.LoadAsync(hostName, cancellationToken); + loader = context.State.Load(module); } else { @@ -108,7 +106,7 @@ string dirSeparator { path = pathSpan[..nextIndex].ToString(); var fileName = path.Replace("?", name); - if (state.GlobalState.Platform.FileSystem.IsReadable(fileName)) + if (state.Platform.FileSystem.IsReadable(state.Platform.ToHostString(fileName))) { return fileName; } @@ -200,7 +198,12 @@ CancellationToken cancellationToken } return context.Return( - await context.State.LoadFileAsync(fileName, "bt", null, cancellationToken) + await context.State.LoadFileAsync( + context.State.Platform.ToHostString(fileName), + "bt", + null, + cancellationToken + ) ); } } diff --git a/src/Lua/Standard/OperatingSystemLibrary.cs b/src/Lua/Standard/OperatingSystemLibrary.cs index 318c4542..de4deafa 100644 --- a/src/Lua/Standard/OperatingSystemLibrary.cs +++ b/src/Lua/Standard/OperatingSystemLibrary.cs @@ -156,12 +156,9 @@ CancellationToken cancellationToken ) { var variable = context.GetArgument(0); - return new( - context.Return( - context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable(variable) - ?? LuaValue.Nil - ) - ); + var platform = context.State.Platform; + var value = platform.OsEnvironment.GetEnvironmentVariable(platform.ToHostString(variable)); + return new(context.Return(value is null ? LuaValue.Nil : platform.ToLuaString(value))); } public async ValueTask Remove( @@ -172,12 +169,19 @@ CancellationToken cancellationToken var fileName = context.GetArgument(0); try { - await context.GlobalState.Platform.FileSystem.Remove(fileName, cancellationToken); + await context.GlobalState.Platform.FileSystem.Remove( + context.State.Platform.ToHostString(fileName), + cancellationToken + ); return context.Return(true); } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -191,15 +195,19 @@ CancellationToken cancellationToken try { await context.GlobalState.Platform.FileSystem.Rename( - oldName, - newName, + context.State.Platform.ToHostString(oldName), + context.State.Platform.ToHostString(newName), cancellationToken ); return context.Return(true); } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -246,6 +254,7 @@ public ValueTask TmpName( CancellationToken cancellationToken ) { - return new(context.Return(context.GlobalState.Platform.FileSystem.GetTempFileName())); + var platform = context.State.Platform; + return new(context.Return(platform.ToLuaString(platform.FileSystem.GetTempFileName()))); } } diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index 8e357294..f034972b 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -69,11 +69,17 @@ CancellationToken cancellationToken return new(context.Return("")); } + var byteStrings = context.State.Platform.StringMode == LuaStringMode.Bytes; ValueStringBuilder builder = new(context.ArgumentCount); for (var i = 0; i < context.ArgumentCount; i++) { var arg = context.GetArgument(i); LuaRuntimeException.ThrowBadArgumentIfNumberIsNotInteger(context.State, i + 1, arg); + if (byteStrings && (arg < 0 || arg > 255)) + { + LuaRuntimeException.BadArgument(context.State, i + 1, "value out of range"); + } + builder.Append((char)arg); } @@ -412,7 +418,11 @@ context with } break; case 'c': - formattedValue = ((char)(int)x).ToString(); + formattedValue = ( + context.State.Platform.StringMode == LuaStringMode.Bytes + ? (char)((int)x & 0xFF) + : (char)(int)x + ).ToString(); break; case 'x': { @@ -860,7 +870,13 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToLower())); + return new( + context.Return( + context.State.Platform.StringMode == LuaStringMode.Bytes + ? StringHelper.ToLowerAscii(s) + : s.ToLower() + ) + ); } public ValueTask Match( @@ -1054,6 +1070,12 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToUpper())); + return new( + context.Return( + context.State.Platform.StringMode == LuaStringMode.Bytes + ? StringHelper.ToUpperAscii(s) + : s.ToUpper() + ) + ); } } diff --git a/tests/Lua.Tests/ByteStringIntegrationTests.cs b/tests/Lua.Tests/ByteStringIntegrationTests.cs new file mode 100644 index 00000000..3d456cd9 --- /dev/null +++ b/tests/Lua.Tests/ByteStringIntegrationTests.cs @@ -0,0 +1,348 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +public sealed class ByteStringIntegrationTests : IDisposable +{ + readonly string directory = Path.Combine( + Path.GetTempPath(), + $"LuaBytes_\u1EBF_{Guid.NewGuid()}" + ); + + public ByteStringIntegrationTests() => Directory.CreateDirectory(directory); + + public void Dispose() => Directory.Delete(directory, true); + + LuaState CreateState(LuaStringMode stringMode = LuaStringMode.Bytes) + { + var state = LuaState.Create( + ( + stringMode == LuaStringMode.Bytes + ? LuaPlatform.DefaultByteStrings + : LuaPlatform.Default + ) with + { + FileSystem = new FileSystem(directory, stringMode), + } + ); + state.OpenStandardLibraries(); + return state; + } + + [Test] + public void ExistingConstructorSignaturesRemainAvailable() + { + Assert.That(typeof(FileSystem).GetConstructor([typeof(string)]), Is.Not.Null); + var baseDirectory = typeof(FileSystem).GetConstructor([typeof(string)])!.GetParameters()[0]; + Assert.That(baseDirectory.IsOptional, Is.True); + Assert.That(baseDirectory.DefaultValue, Is.Null); + Assert.That( + typeof(LuaStream).GetConstructor([typeof(LuaFileOpenMode), typeof(Stream)]), + Is.Not.Null + ); + Assert.That(typeof(ConsoleStandardIO).GetConstructor(Type.EmptyTypes), Is.Not.Null); + } + + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task XPCallHostErrorsPreserveDefaultBehaviorAndEncodeByteMode( + LuaStringMode stringMode + ) + { + using var state = CreateState(stringMode); + const string message = "host error ế"; + state.Environment["fail"] = new LuaFunction( + "fail", + (_, _) => throw new IOException(message) + ); + var result = await state.DoStringAsync( + "return xpcall(function() fail() end,function(e) return e end)" + ); + Assert.That(result[0], Is.EqualTo(new LuaValue(false))); + Assert.That( + result[1], + Is.EqualTo( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(message) : LuaValue.Nil + ) + ); + } + + [Test] + public async Task TemporaryFilesPreserveRawBytesAndPositions() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local bytes = string.char(239, 187, 191, 0, 128, 255) + local f = assert(io.tmpfile()) + f:write(bytes) + assert(f:seek("end") == #bytes) + f:seek("set", 0) + assert(f:read(1):byte() == 239) + assert(f:seek() == 1) + f:seek("set", 0) + assert(f:read("*a") == bytes) + f:close() + """ + ); + } + + [TestCase("local f,e=loadfile(path); return f~=nil,e")] + [TestCase("return pcall(io.input,path)")] + [TestCase("return xpcall(function() dofile(path) end,function(e) return e end)")] + public async Task FileErrorsContainTheOriginalUtf8Path(string script) + { + using var state = CreateState(); + var path = LuaByteString.FromString(Path.Combine(directory, "missing_\u1EBF", "file.lua")); + state.Environment["path"] = path; + var result = await state.DoStringAsync(script, "caller_\u1EBF"); + + Assert.That(result[0], Is.EqualTo(new LuaValue(false))); + var message = result[1].Read(); + Assert.That(message.All(c => c <= 255), Is.True); + Assert.That(message, Does.Contain(path)); + } + + [Test] + public async Task LuaErrorsAndSyntaxDiagnosticsAreNotReencoded() + { + using var state = CreateState(); + var text = LuaByteString.FromString("Lua error \u1EBF"); + state.Environment["message"] = text; + state.Environment["chunk"] = LuaByteString.FromString("chunk_\u1EBF"); + var result = await state.DoStringAsync( + """ + local _,p = pcall(error,message,0) + local _,x = xpcall(function() error(message,0) end,function(e) return e end) + local _,s = load("return )",chunk) + local payload = {} + local ok,value = xpcall(function() error(payload) end,function(e) return e end) + assert(not ok and value == payload) + local raw = string.char(255,128) + local f = assert(load("return '"..raw.."'",raw)) + assert(f() == raw and debug.getinfo(f).source == raw) + return p,x,s + """ + ); + Assert.That(result[0].Read(), Is.EqualTo(text)); + Assert.That(result[1].Read(), Is.EqualTo(text)); + Assert.That( + result[2].Read(), + Does.Contain(state.Environment["chunk"].Read()) + ); + } + + [TestCase("read('*a')")] + [TestCase("write('x')")] + [TestCase("flush()")] + public async Task FileHandleErrorsAreUtf8Bytes(string operation) + { + using var state = CreateState(); + state.Environment["file"] = LuaValue.FromUserData(new FileHandle(new FailingStream())); + var result = await state.DoStringAsync($"return file:{operation}"); + Assert.That(result[0], Is.EqualTo(LuaValue.Nil)); + Assert.That( + result[1].Read(), + Is.EqualTo(LuaByteString.FromString(FailingStream.Error)) + ); + } + + [TestCase(1)] + [TestCase(8193)] + public async Task BinaryPacketsRoundTripAcrossBufferBoundaries(int chunkSize) + { + var bytes = Enumerable.Range(0, 20000).Select(i => (byte)i).ToArray(); + using var memory = new MemoryStream(); + using var stream = new LuaStream(LuaFileOpenMode.WriteUpdate, memory, LuaStringMode.Bytes); + await stream.WriteAsync(LuaByteString.FromBytes(bytes).AsMemory(), default); + Assert.That(memory.ToArray(), Is.EqualTo(bytes)); + stream.Seek(SeekOrigin.Begin, 0); + var result = new StringBuilder(); + string? part; + while ((part = await stream.ReadAsync(chunkSize, default)) != null) + result.Append(part); + Assert.That(LuaByteString.ToBytes(result.ToString()), Is.EqualTo(bytes)); + Assert.That(stream.Seek(SeekOrigin.Current, 0), Is.EqualTo(bytes.Length)); + } + + [Test] + public async Task RawLinesPreserveCarriageReturnsIncludingAtBufferEdges() + { + var first = new string('a', 1023) + "\r"; + var bytes = LuaByteString.ToBytes(first + "\nnext\rpart\nlast\r"); + using var stream = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(bytes), + LuaStringMode.Bytes + ); + Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo(first)); + Assert.That(await stream.ReadLineAsync(true, default), Is.EqualTo("next\rpart\n")); + Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo("last\r")); + Assert.That(await stream.ReadLineAsync(false, default), Is.Null); + } + + [TestCase(LuaStringMode.Utf16, false)] + [TestCase(LuaStringMode.Bytes, false)] + [TestCase(LuaStringMode.Bytes, true)] + public async Task HostModuleLoadersReceiveDotNetNamesAndReturnTextOrBytes( + LuaStringMode stringMode, + bool bytes + ) + { + var loader = new MemoryModuleLoader(bytes); + using var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.ModuleLoader = loader; + state.OpenStandardLibraries(); + state.Environment["name"] = + stringMode == LuaStringMode.Bytes + ? LuaByteString.FromString(MemoryModuleLoader.Name) + : MemoryModuleLoader.Name; + var result = await state.DoStringAsync("local a=require(name); return a.text,a.size"); + Assert.That( + result[0].Read(), + Is.EqualTo( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString("\u1EBF") : "\u1EBF" + ) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(stringMode == LuaStringMode.Bytes ? 3 : 1))); + } + + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task HostSourceEntryPointsAcceptDotNetTextAndChunkNames(LuaStringMode stringMode) + { + using var state = CreateState(stringMode); + const string text = "こんにちはé😀"; + const string source = "return 'こんにちはé😀', #'こんにちはé😀', debug.getinfo(1).source"; + const string name = "@日本_é_😀.lua"; + LuaValue[] expected = + [ + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(text) : text, + stringMode == LuaStringMode.Bytes ? Encoding.UTF8.GetByteCount(text) : text.Length, + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(name) : name, + ]; + + Assert.That(await state.ExecuteAsync(state.Load(source, name)), Is.EqualTo(expected)); + if (stringMode == LuaStringMode.Bytes) + { + Assert.That( + await state.ExecuteAsync(state.Load("\uFEFF" + source, name)), + Is.EqualTo(expected) + ); + } + var buffer = new LuaValue[expected.Length]; + Assert.That(await state.DoStringAsync(source, buffer, name), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + + var bytes = Encoding.UTF8.GetBytes(source); + Assert.That(await state.ExecuteAsync(bytes, name), Is.EqualTo(expected)); + Assert.That(await state.ExecuteAsync(bytes, buffer, name), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + } + + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task HostFileEntryPointsAcceptDotNetPathsWhileLuaUsesLuaStrings( + LuaStringMode stringMode + ) + { + using var state = CreateState(stringMode); + const string fileName = "日本_é_😀.lua"; + const string text = "こんにちは"; + var path = Path.Combine(directory, fileName); + await File.WriteAllTextAsync(path, "return 'こんにちは', #'こんにちは'", Encoding.UTF8); + Assert.That(state.Platform.FileSystem.IsReadable(path), Is.True); + LuaValue[] expected = + [ + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(text) : text, + stringMode == LuaStringMode.Bytes ? Encoding.UTF8.GetByteCount(text) : text.Length, + ]; + + Assert.That(await state.DoFileAsync(path), Is.EqualTo(expected)); + var buffer = new LuaValue[expected.Length]; + Assert.That(await state.DoFileAsync(fileName, buffer), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + var closure = await state.LoadFileAsync(path, "t", null, default); + Assert.That(await state.ExecuteAsync(closure), Is.EqualTo(expected)); + + state.Environment["path"] = + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(fileName) : fileName; + var module = Path.GetFileNameWithoutExtension(fileName); + state.Environment["module"] = + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(module) : module; + state.Environment["text"] = expected[0]; + await state.DoStringAsync( + """ + assert(assert(loadfile(path))() == text) + assert(dofile(path) == text) + assert(require(module) == text) + """ + ); + + state.Platform = state.Platform with + { + FileSystem = CompositeLoaderFileSystem.Create( + state.Platform.FileSystem, + new MemoryFileLoader(stringMode) + ), + }; + await state.DoStringAsync("assert(require('virtual_日本') == text)"); + } + + sealed class MemoryFileLoader(LuaStringMode stringMode) : ILuaFileLoader + { + public bool Exists(string path) => path == "virtual_日本.lua"; + + public ValueTask LoadAsync(string path, CancellationToken cancellationToken) + { + Assert.That(path, Is.EqualTo("virtual_日本.lua")); + const string source = "return 'こんにちは'"; + return new( + ILuaStream.CreateFromString( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(source) : source + ) + ); + } + } + + sealed class MemoryModuleLoader(bool bytes) : ILuaModuleLoader + { + public const string Name = "memory_\u1EBF"; + + public bool Exists(string moduleName) => moduleName == Name; + + public async ValueTask LoadAsync( + string moduleName, + CancellationToken ct = default + ) + { + Assert.That(moduleName, Is.EqualTo(Name)); + await Task.Yield(); + const string source = "return {text='\u1EBF',size=#'\u1EBF'}"; + return bytes + ? new LuaModule(moduleName, Encoding.UTF8.GetBytes(source)) + : new LuaModule(moduleName, source); + } + } + + sealed class FailingStream : ILuaStream + { + public const string Error = "I/O failure \u1EBF"; + public bool IsOpen => true; + public LuaFileOpenMode Mode => LuaFileOpenMode.ReadUpdate; + + public ValueTask ReadAllAsync(CancellationToken ct) => throw new IOException(Error); + + public ValueTask WriteAsync(ReadOnlyMemory value, CancellationToken ct) => + throw new IOException(Error); + + public ValueTask FlushAsync(CancellationToken ct) => throw new IOException(Error); + + public void Dispose() { } + } +} diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs new file mode 100644 index 00000000..8c689470 --- /dev/null +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -0,0 +1,587 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +// Expected values in this file were produced by the reference Lua 5.2.4 interpreter. +public sealed class ByteStringTests : IDisposable +{ + // VI is "Tien Len" with accents (Vietnamese) as UTF-8 bytes; the sources use \x escapes so this file stays ASCII. + const string Prelude = """ + local VI = "Ti\xE1\xBA\xBFn L\xC3\xAAn" + local E = "\xE1\xBA\xBF" + local function hex(s) return (s:gsub(".", function(c) return string.format("%02x", c:byte()) end)) end + + """; + + readonly string testDirectory = Path.Combine( + Path.GetTempPath(), + $"LuaByteStringTests_{Guid.NewGuid()}" + ); + + public ByteStringTests() + { + Directory.CreateDirectory(testDirectory); + } + + public void Dispose() + { + if (Directory.Exists(testDirectory)) + { + Directory.Delete(testDirectory, true); + } + } + + LuaState CreateByteState() + { + var state = LuaState.Create( + LuaPlatform.DefaultByteStrings with + { + FileSystem = new FileSystem(testDirectory, LuaStringMode.Bytes), + } + ); + state.OpenStandardLibraries(); + return state; + } + + async Task RunAsync(string body) + { + using var state = CreateByteState(); + await state.DoStringAsync(Prelude + body); + } + + [Test] + public void DefaultPlatform_KeepsUtf16Strings() + { + Assert.That(LuaPlatform.Default.StringMode, Is.EqualTo(LuaStringMode.Utf16)); + Assert.That(LuaPlatform.DefaultByteStrings.StringMode, Is.EqualTo(LuaStringMode.Bytes)); + } + + [Test] + public async Task DefaultMode_StringLenCountsUtf16Chars() + { + var state = LuaState.Create(); + state.OpenStandardLibraries(); + var result = await state.DoStringAsync( + "return string.len('\u3042\u3044\u3046\u3048\u304A'), #'\u1EBF', ('\u1EBF'):find('%a')" + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(5))); + Assert.That(result[1], Is.EqualTo(new LuaValue(1))); + Assert.That(result[2], Is.EqualTo(new LuaValue(1))); + } + + [Test] + public Task Length_Byte_Char_Sub() => + RunAsync( + """ + assert(#VI == 11 and VI:len() == 11) + assert(#E == 3) + assert(#"\xE3\x81\x82\xE3\x81\x84\xE3\x81\x86\xE3\x81\x88\xE3\x81\x8A" == 15) + assert(#"\xF0\x9F\x98\x80" == 4) + assert(select("#", E:byte(1, -1)) == 3) + local a, b, c = E:byte(1, -1) + assert(a == 225 and b == 186 and c == 191) + assert(string.char(225, 186, 191) == E) + assert(hex(string.char(255, 0, 128)) == "ff0080") + assert(hex(VI:sub(3, 4)) == "e1ba") + assert(hex(VI:sub(-3)) == "c3aa6e") + assert(hex(E:sub(1, 1)) == "e1" and hex(E:sub(2)) == "babf") + assert(hex(E:reverse()) == "bfbae1") + assert(hex(E:rep(2, "|")) == "e1babf7ce1babf") + assert(hex(E .. "-" .. E) == "e1babf2de1babf") + """ + ); + + [Test] + public Task StringChar_RejectsValuesAboveByteRange() => + RunAsync( + """ + assert(not pcall(string.char, 256)) + assert(not pcall(string.char, -1)) + assert(not pcall(string.char, 65 + 256)) + local ok, err = pcall(string.char, 256) + assert(err:find("value out of range", 1, true)) + assert(hex(string.format("%c", 256 + 65)) == "41") + assert(hex(string.format("%c%c%c", 225, 186, 191)) == "e1babf") + """ + ); + + [Test] + public Task UpperLower_OnlyChangeAscii() => + RunAsync( + """ + assert(hex(("a\xC3\xA0b\xE1\xBA\xBF"):upper()) == "41c3a042e1babf") + assert(hex(("A\xC3\x80B\xE1\xBA\xBE"):lower()) == "61c38062e1babe") + local all = {} + for c = 0, 255 do all[#all + 1] = string.char(c) end + all = table.concat(all) + local up, low = all:upper(), all:lower() + assert(#up == 256 and #low == 256) + for c = 0, 255 do + local ch = string.char(c) + local wantUp = (c >= 97 and c <= 122) and c - 32 or c + local wantLow = (c >= 65 and c <= 90) and c + 32 or c + assert(up:byte(c + 1) == wantUp, "upper " .. c) + assert(low:byte(c + 1) == wantLow, "lower " .. c) + end + """ + ); + + // C locale ctype for every byte, written independently of the implementation. + static bool CType(char cls, int c) + { + var upper = c >= 'A' && c <= 'Z'; + var lower = c >= 'a' && c <= 'z'; + var digit = c >= '0' && c <= '9'; + return cls switch + { + 'a' => upper || lower, + 'c' => c < 32 || c == 127, + 'd' => digit, + 'g' => c > 32 && c < 127, + 'l' => lower, + 'p' => c > 32 && c < 127 && !(upper || lower || digit), + 's' => c == 32 || (c >= 9 && c <= 13), + 'u' => upper, + 'w' => upper || lower || digit, + 'x' => digit || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), + _ => throw new ArgumentOutOfRangeException(nameof(cls)), + }; + } + + [TestCase('a')] + [TestCase('c')] + [TestCase('d')] + [TestCase('g')] + [TestCase('l')] + [TestCase('p')] + [TestCase('s')] + [TestCase('u')] + [TestCase('w')] + [TestCase('x')] + public async Task CharacterClasses_FollowCLocaleForEveryByte(char cls) + { + var expected = new StringBuilder(); + var expectedComplement = new StringBuilder(); + for (var c = 0; c < 256; c++) + { + expected.Append(CType(cls, c) ? '1' : '0'); + expectedComplement.Append(CType(cls, c) ? '0' : '1'); + } + + var upperCls = char.ToUpperInvariant(cls); + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + local function bits(pattern) + local t = {} + for c = 0, 255 do t[#t + 1] = string.char(c):find(pattern) and "1" or "0" end + return table.concat(t) + end + return bits("%{{cls}}"), bits("%{{upperCls}}"), bits("[%{{cls}}]"), bits("[^%{{cls}}]") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[1].Read(), Is.EqualTo(expectedComplement.ToString())); + Assert.That(result[2].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[3].Read(), Is.EqualTo(expectedComplement.ToString())); + } + + [Test] + public Task Patterns_WorkOnBytes() => + RunAsync( + """ + local s, n = VI:gsub("%a+", "X") + assert(hex(s) == hex("X\xE1\xBA\xBFX X\xC3\xAAX") and n == 4) + assert(select(2, VI:gsub("%a+", "X")) == 4) + assert(E:find("%a") == nil) + assert(VI:find(E, 1, true) == 3) + assert(VI:find("L") == 8) + assert(VI:find("n", 7) == 11) + local c1, c2 = VI:match("^(.)(.)") + assert(hex(c1) == "54" and hex(c2) == "69") + assert(VI:match("()L") == 8) + assert(hex((E:gsub(".", "<%0>"))) == "3ce13e3cba3e3cbf3e") + local words = {} + for w in VI:gmatch("%S+") do words[#words + 1] = hex(w) end + assert(table.concat(words, ",") == "5469e1babf6e,4cc3aa6e") + local bytes = {} + for ch in E:gmatch(".") do bytes[#bytes + 1] = hex(ch) end + assert(table.concat(bytes, ",") == "e1,ba,bf") + assert(VI:gsub("%f[%a]%a+", "<%0>") == "\xE1\xBA\xBF \xC3\xAA") + assert(("(\xE1\xBA\xBF (x)) y"):match("%b()") == "(\xE1\xBA\xBF (x))") + """ + ); + + [Test] + public Task Format_PadsByBytes() => + RunAsync( + """ + assert(hex(string.format("%5s|%-5s|", E, E)) == "2020e1babf7c" .. "e1babf20207c") + assert(hex(string.format("%.2s", E)) == "e1ba") + assert(#string.format("%s", VI) == 11) + """ + ); + + [Test] + public Task Comparison_IsByteWise() => + RunAsync( + """ + assert("\x80" > "\x7F" and "\xFF" > "\x01" and "\x7F" < "\x80") + local w = { "\xC3\xA9", "e", "z", "\xE1\xBA\xBF", "a", "\xC3\x80", "Z", "\xC3\xAA" } + table.sort(w) + local h = {} + for i, x in ipairs(w) do h[i] = hex(x) end + assert(table.concat(h, ",") == "5a,61,65,7a,c380,c3a9,c3aa,e1babf") + """ + ); + + [Test] + public async Task DoString_EncodesDotNetSourceAsUtf8() + { + using var state = CreateByteState(); + var source = "return 'Ti\u1EBFn', #'Ti\u1EBFn', '\u1EBF'"; + var result = await state.DoStringAsync(source); + + Assert.That(result[0].Read(), Is.EqualTo(LuaByteString.FromString("Ti\u1EBFn"))); + Assert.That(result[1], Is.EqualTo(new LuaValue(6))); + Assert.That( + LuaByteString.ToBytes(result[2].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + } + + [Test] + public async Task LoadBytes_SkipsOnlyUtf8Bom() + { + using var state = CreateByteState(); + var withBom = new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray(); + var result = await state.ExecuteAsync(state.Load(withBom, "bom", "t")); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task LoadFile_ReadsUtf8SourceAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "chunk.lua"), + new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray() + ); + + using var state = CreateByteState(); + var closure = await state.LoadFileAsync("chunk.lua", "bt", null, CancellationToken.None); + var result = await state.ExecuteAsync(closure); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task Require_LoadsUtf8ModuleAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "viet.lua"), + Encoding.UTF8.GetBytes("return { s = 'Ti\u1EBFn L\u00EAn' }") + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{testDirectory.Replace("\\", "/")}}/?.lua" + local m = require("viet") + return #m.s + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(11))); + } + + [Test] + public async Task Io_WritesAndReadsRawBytes() + { + var data = new byte[] + { + 0x54, + 0x69, + 0xE1, + 0xBA, + 0xBF, + 0x6E, + 0x0A, + 0xE1, + 0xBA, + 0xBF, + 0xFF, + 0x00, + 0x80, + 0x7A, + 0x0D, + 0x0A, + 0x65, + 0x6E, + 0x64, + }; + using var state = CreateByteState(); + state.Environment["data"] = LuaByteString.FromBytes(data); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("raw.bin", "wb")) + f:write(data) + f:close() + f = assert(io.open("raw.bin", "rb")) + local size = f:seek("end") + f:seek("set", 0) + local all = f:read("*a") + f:close() + f = assert(io.open("raw.bin", "rb")) + local l1 = f:read("*l") + local n2 = f:read(2) + f:close() + return size, all == data, #l1, #n2 + """ + ); + + Assert.That(File.ReadAllBytes(Path.Combine(testDirectory, "raw.bin")), Is.EqualTo(data)); + Assert.That(result[0], Is.EqualTo(new LuaValue(data.Length))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(6))); + Assert.That(result[3], Is.EqualTo(new LuaValue(2))); + } + + [Test] + public async Task FileNames_AreDecodedFromUtf8Bytes() + { + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("t\xE1\xBA\xBFn.bin", "wb")) + f:write("x") + f:close() + local g = assert(io.open("r\xC3\xAA.bin", "wb")) + g:close() + local renamed = os.rename("r\xC3\xAA.bin", "r2\xC3\xAA.bin") + local gone = io.open("r\xC3\xAA.bin", "rb") == nil + local removed = os.remove("r2\xC3\xAA.bin") + local ok, msg = io.open("missing_\xE1\xBA\xBF.bin", "rb") + return renamed, gone, removed, ok == nil, type(msg) + """ + ); + + Assert.That(File.Exists(Path.Combine(testDirectory, "t\u1EBFn.bin")), Is.True); + Assert.That(File.Exists(Path.Combine(testDirectory, "r\u00EA.bin")), Is.False); + Assert.That(File.Exists(Path.Combine(testDirectory, "r2\u00EA.bin")), Is.False); + Assert.That(result[0], Is.EqualTo(new LuaValue(true))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(true))); + Assert.That(result[3], Is.EqualTo(new LuaValue(true))); + Assert.That(result[4].Read(), Is.EqualTo("string")); + } + + [Test] + public async Task Require_FindsModuleWithNonAsciiName() + { + await File.WriteAllTextAsync(Path.Combine(testDirectory, "m\u1EBFd.lua"), "return 'ok'"); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{testDirectory.Replace("\\", "/")}}/?.lua" + return require("m\xE1\xBA\xBFd") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo("ok")); + } + + [Test] + public async Task OsGetenv_ReturnsUtf8Bytes() + { + const string name = "LUA_CSHARP_BYTE_STRINGS_TEST"; + Environment.SetEnvironmentVariable(name, "Ti\u1EBFn"); + try + { + using var state = CreateByteState(); + var result = await state.DoStringAsync($"local v = os.getenv('{name}'); return #v, v"); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(LuaByteString.ToString(result[1].Read()), Is.EqualTo("Ti\u1EBFn")); + } + finally + { + Environment.SetEnvironmentVariable(name, null); + } + } + + [Test] + public async Task Io_DoesNotSkipBomOnRead() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "bom.bin"), + new byte[] { 0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63 } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("bom.bin", "rb")) + local all = f:read("*a") + f:close() + f = assert(io.open("bom.bin", "rb")) + local n3 = f:read(3) + local rest = f:read("*l") + f:close() + return #all, #n3, rest + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + Assert.That(result[2].Read(), Is.EqualTo("abc")); + } + + [Test] + public async Task Io_NumberReadingTreatsHighBytesAsNotWhitespace() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "num.bin"), + new byte[] + { + 0x20, + 0x20, + 0x31, + 0x32, + 0x20, + 0x30, + 0x78, + 0x31, + 0x46, + 0x20, + 0xA0, + 0x37, + 0x0A, + } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("num.bin", "rb")) + local a, b, c = f:read("*n", "*n", "*n") + f:close() + return a, b, c + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(12))); + Assert.That(result[1], Is.EqualTo(new LuaValue(31))); + Assert.That(result[2], Is.EqualTo(LuaValue.Nil)); + } + + [Test] + public async Task LuaStream_ByteMode_RoundTripsEveryByte() + { + var all = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray(); + using var memory = new MemoryStream(); + using (var writer = new LuaStream(LuaFileOpenMode.Write, memory, LuaStringMode.Bytes)) + { + await writer.WriteAsync( + LuaByteString.FromBytes(all).AsMemory(), + CancellationToken.None + ); + await writer.FlushAsync(CancellationToken.None); + Assert.That(memory.ToArray(), Is.EqualTo(all)); + } + + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(all), + LuaStringMode.Bytes + ); + var text = await reader.ReadAllAsync(CancellationToken.None); + Assert.That(LuaByteString.ToBytes(text), Is.EqualTo(all)); + } + + [Test] + public async Task LuaStream_DefaultMode_StillEncodesUtf8() + { + using var memory = new MemoryStream(); + var writer = new LuaStream(LuaFileOpenMode.Write, memory); + await writer.WriteAsync("\u1EBF".AsMemory(), CancellationToken.None); + await writer.FlushAsync(CancellationToken.None); + + Assert.That(memory.ToArray(), Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF })); + } + + [Test] + public async Task LuaStream_ByteMode_ReadCountCountsBytes() + { + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(new byte[] { 0xE1, 0xBA, 0xBF, 0x61 }), + LuaStringMode.Bytes + ); + + var first = await reader.ReadAsync(2, CancellationToken.None); + var second = await reader.ReadAsync(2, CancellationToken.None); + + Assert.That(LuaByteString.ToBytes(first!), Is.EqualTo(new byte[] { 0xE1, 0xBA })); + Assert.That(LuaByteString.ToBytes(second!), Is.EqualTo(new byte[] { 0xBF, 0x61 })); + } + + [Test] + public void Utf8EncodingMatchesDotNetForSurrogateEdgeCases() + { + var source = new StringBuilder(); + for (var value = 0; value <= char.MaxValue; value++) + source.Append((char)value); + for (var value = 0x10000; value <= 0x10FFFF; value += 997) + source.Append(char.ConvertFromUtf32(value)); + source.Append("\uD800\uDC00\uDBFF\uDFFF\uD800\uD800\uDC00\uDC00\uD800x\uDC00"); + var text = source.ToString(); + var actual = LuaByteString.FromString(text); + Assert.That(actual.All(c => c <= 255), Is.True); + Assert.That(LuaByteString.ToBytes(actual), Is.EqualTo(Encoding.UTF8.GetBytes(text))); + for (var length = 0; length < 65; length++) + { + text = new string('a', length) + "ế😀\uD800x" + new string('z', length); + Assert.That( + LuaByteString.ToBytes(LuaByteString.FromString(text)), + Is.EqualTo(Encoding.UTF8.GetBytes(text)) + ); + } + } + + [Test] + public void LuaByteString_ConvertsBetweenDotNetStringsAndBytes() + { + const string text = "Ti\u1EBFn L\u00EAn \U0001F600"; + var byteString = LuaByteString.FromString(text); + + Assert.That(byteString.Length, Is.EqualTo(Encoding.UTF8.GetByteCount(text))); + Assert.That(byteString.All(c => c <= 0xFF), Is.True); + Assert.That(LuaByteString.ToString(byteString), Is.EqualTo(text)); + Assert.That(LuaByteString.ToBytes(byteString), Is.EqualTo(Encoding.UTF8.GetBytes(text))); + Assert.That( + LuaByteString.FromBytes(new byte[] { 0xFF, 0x00, 0x80 }), + Is.EqualTo("\u00FF\0\u0080") + ); + Assert.That(LuaByteString.ToString("\u00FF"), Is.EqualTo("\uFFFD")); + Assert.That(LuaByteString.ToBytes("\u1EBF"), Is.EqualTo(new byte[] { (byte)'?' })); + } +} diff --git a/tests/Lua.Tests/Helpers/TestStandardIO.cs b/tests/Lua.Tests/Helpers/TestStandardIO.cs index 89a135d2..ad867851 100644 --- a/tests/Lua.Tests/Helpers/TestStandardIO.cs +++ b/tests/Lua.Tests/Helpers/TestStandardIO.cs @@ -2,9 +2,9 @@ namespace Lua.Tests.Helpers; -public class TestStandardIO : ILuaStandardIO +public class TestStandardIO(LuaStringMode stringMode = LuaStringMode.Utf16) : ILuaStandardIO { - readonly ConsoleStandardIO consoleStandardIO = new(); + readonly ConsoleStandardIO consoleStandardIO = new(stringMode); public ILuaStream Input { @@ -18,7 +18,11 @@ public ILuaStream Input new BufferedOutputStream( (memory) => { - Console.WriteLine(memory.ToString()); + Console.WriteLine( + stringMode == LuaStringMode.Bytes + ? LuaByteString.ToString(memory.ToString()) + : memory.ToString() + ); } ) ); diff --git a/tests/Lua.Tests/LuaTests.cs b/tests/Lua.Tests/LuaTests.cs index 4332568a..08483a8c 100644 --- a/tests/Lua.Tests/LuaTests.cs +++ b/tests/Lua.Tests/LuaTests.cs @@ -1,13 +1,16 @@ using System.Text; using Lua.IO; +using Lua.Platforms; using Lua.Standard; using Lua.Tests.Helpers; namespace Lua.Tests; -public class LuaTests +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class LuaTests(LuaStringMode stringMode) { - static string PatchFilesLuaSource(string source) + static string PatchFilesLuaSource(string source, LuaStringMode stringMode) { const char utf8Bom = '\uFEFF'; var lines = source.Replace("\r\n", "\n").Split('\n'); @@ -19,12 +22,15 @@ static string PatchFilesLuaSource(string source) "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; lines[296] = "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - lines[330] = - $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[331] = - $"testloadfile(\"{utf8Bom}return 239\", 239) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[332] = - $"testloadfile(\"{utf8Bom}\", nil) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + if (stringMode != LuaStringMode.Bytes) + { + lines[330] = + $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + lines[331] = + $"testloadfile(\"{utf8Bom}return 239\", 239) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + lines[332] = + $"testloadfile(\"{utf8Bom}\", nil) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + } for (var i = 341; i <= 346; i++) { lines[i] = @@ -77,11 +83,13 @@ public async Task Test_Lua(string file) { var path = FileHelper.GetAbsolutePath(file); var baseDirectory = Path.GetDirectoryName(path)!; - var state = LuaState.Create(); + using var state = LuaState.Create( + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); state.Platform = state.Platform with { - StandardIO = new TestStandardIO(), - FileSystem = new FileSystem(baseDirectory), + StandardIO = new TestStandardIO(stringMode), + FileSystem = new FileSystem(baseDirectory, stringMode), }; state.OpenStandardLibraries(); if (file == "tests-lua/errors.lua") @@ -91,11 +99,18 @@ public async Task Test_Lua(string file) if (file == "tests-lua/files.lua") { // files.lua contains raw 8-bit source literals. Decode its bytes 1:1 - // and patch only the byte-sensitive assertions in memory so the + // and apply compatibility patches in memory so the // original Lua test file stays untouched on disk. var sourceBytes = await File.ReadAllBytesAsync(path); - var source = PatchFilesLuaSource(Encoding.Latin1.GetString(sourceBytes)); - var closure = state.Load(source, "@" + Path.GetFileName(file)); + var source = PatchFilesLuaSource( + Encoding.Latin1.GetString(sourceBytes), + stringMode + ); + var name = "@" + Path.GetFileName(file); + var closure = + stringMode == LuaStringMode.Bytes + ? state.Load(LuaByteString.ToBytes(source), name) + : state.Load(source, name); await state.ExecuteAsync(closure); } else