From 13f58a8a7d99ff4be3e34d0b815a51718642a676 Mon Sep 17 00:00:00 2001 From: DEVN <10086005+billythekidz@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:35:01 +0700 Subject: [PATCH 01/11] Add opt-in byte strings (LuaPlatform.ByteStrings) Lua strings are UTF-16 by default, so #"x" and string.sub differ from the reference implementation for non-ASCII text. With ByteStrings enabled every char holds one byte, files and standard I/O are read and written as raw bytes, character classes, upper and lower follow the C locale and string.char rejects values above 255. Default behavior is unchanged. --- .../CodeAnalysis/Compilation/BomUtility.cs | 5 + src/Lua/IO/ConsoleStandardIO.cs | 24 +- src/Lua/IO/FileSystem.cs | 10 +- src/Lua/IO/ILuaStream.cs | 5 + src/Lua/IO/LuaByteEncoding.cs | 88 +++ src/Lua/IO/LuaStream.cs | 14 +- src/Lua/Internal/StringHelper.cs | 60 +++ src/Lua/Internal/Utf8Reader.cs | 30 +- src/Lua/LuaByteString.cs | 44 ++ src/Lua/LuaState.cs | 14 +- src/Lua/Platforms/LuaPlatform.cs | 27 + src/Lua/Standard/Internal/MatchState.cs | 62 ++- src/Lua/Standard/StringLibrary.cs | 24 +- tests/Lua.Tests/ByteStringTests.cs | 500 ++++++++++++++++++ 14 files changed, 880 insertions(+), 27 deletions(-) create mode 100644 src/Lua/IO/LuaByteEncoding.cs create mode 100644 src/Lua/LuaByteString.cs create mode 100644 tests/Lua.Tests/ByteStringTests.cs diff --git a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs index 074c1caa..6cd18cfd 100644 --- a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs +++ b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs @@ -12,6 +12,11 @@ static class BomUtility static ReadOnlySpan BomUtf32Little => [0xFF, 0xFE, 0x00, 0x00]; + public static ReadOnlySpan SkipUtf8Bom(ReadOnlySpan text) + { + return text.StartsWith(BomUtf8) ? text.Slice(BomUtf8.Length) : text; + } + /// /// Removes the BOM from the beginning of the text and returns the encoding. /// Supported encodings are UTF-8, UTF-16 (little and big endian), and UTF-32 (little endian). diff --git a/src/Lua/IO/ConsoleStandardIO.cs b/src/Lua/IO/ConsoleStandardIO.cs index 8d532ecb..91cff485 100644 --- a/src/Lua/IO/ConsoleStandardIO.cs +++ b/src/Lua/IO/ConsoleStandardIO.cs @@ -5,7 +5,11 @@ namespace Lua.IO; /// /// Default implementation of ILuaStandardIO using Console /// -public sealed class ConsoleStandardIO : ILuaStandardIO +/// +/// When true, standard input and output are read and written as raw bytes (one byte per char) +/// instead of UTF-8 text. Use together with . +/// +public sealed class ConsoleStandardIO(bool byteStrings = false) : ILuaStandardIO { ILuaStream? standardInput; ILuaStream? standardOutput; @@ -13,16 +17,28 @@ public sealed class ConsoleStandardIO : ILuaStandardIO public ILuaStream Input => standardInput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardInput(), LuaFileOpenMode.Read) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardInput(), + LuaFileOpenMode.Read, + byteStrings + ) ); public ILuaStream Output => standardOutput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardOutput(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardOutput(), + LuaFileOpenMode.Write, + byteStrings + ) ); public ILuaStream Error => standardError ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardError(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardError(), + LuaFileOpenMode.Write, + byteStrings + ) ); } diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 161d3c8b..7843985c 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -1,6 +1,12 @@ namespace Lua.IO; -public sealed class FileSystem(string? baseDirectory = null) : ILuaFileSystem +/// Directory that relative paths are resolved against. +/// +/// When true, files are read and written as raw bytes (one byte per char) instead of UTF-8 text. +/// Use together with . +/// +public sealed class FileSystem(string? baseDirectory = null, bool byteStrings = false) + : ILuaFileSystem { public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory(); @@ -60,7 +66,7 @@ CancellationToken cancellationToken stream = File.Open(path, mode, access, FileShare.ReadWrite | FileShare.Delete); } - ILuaStream wrapper = new LuaStream(openMode, stream); + ILuaStream wrapper = new LuaStream(openMode, stream, byteStrings); if (openMode == LuaFileOpenMode.AppendUpdate) { diff --git a/src/Lua/IO/ILuaStream.cs b/src/Lua/IO/ILuaStream.cs index ed6ab400..63341090 100644 --- a/src/Lua/IO/ILuaStream.cs +++ b/src/Lua/IO/ILuaStream.cs @@ -74,6 +74,11 @@ static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode) return new LuaStream(openMode, stream); } + static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode, bool byteStrings) + { + return new LuaStream(openMode, stream, byteStrings); + } + static ILuaStream CreateFromString(string content) { return new StringStream(content); diff --git a/src/Lua/IO/LuaByteEncoding.cs b/src/Lua/IO/LuaByteEncoding.cs new file mode 100644 index 00000000..bdc03d83 --- /dev/null +++ b/src/Lua/IO/LuaByteEncoding.cs @@ -0,0 +1,88 @@ +using System.Text; + +namespace Lua.IO; + +/// +/// Maps one byte to one (0x00-0xFF) and back, so that a Lua string can carry +/// arbitrary bytes (for example UTF-8 text) with the same #s, string.sub and +/// string.byte results as the reference C implementation. +/// Characters above 0xFF cannot be represented and are written as '?'. +/// +sealed class LuaByteEncoding : Encoding +{ + public static LuaByteEncoding Instance { get; } = new(); + + public override int GetByteCount(char[] chars, int index, int count) => count; + + public override int GetByteCount(string s) => s.Length; + + public override int GetByteCount(ReadOnlySpan chars) => chars.Length; + + public override int GetBytes( + char[] chars, + int charIndex, + int charCount, + byte[] bytes, + int byteIndex + ) => GetBytes(chars.AsSpan(charIndex, charCount), bytes.AsSpan(byteIndex)); + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + if (bytes.Length < chars.Length) + { + throw new ArgumentException("Destination is too small.", nameof(bytes)); + } + + for (var i = 0; i < chars.Length; i++) + { + var c = chars[i]; + bytes[i] = c <= 0xFF ? (byte)c : (byte)'?'; + } + + return chars.Length; + } + + public override int GetCharCount(byte[] bytes, int index, int count) => count; + + public override int GetCharCount(ReadOnlySpan bytes) => bytes.Length; + + public override int GetChars( + byte[] bytes, + int byteIndex, + int byteCount, + char[] chars, + int charIndex + ) => GetChars(bytes.AsSpan(byteIndex, byteCount), chars.AsSpan(charIndex)); + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + if (chars.Length < bytes.Length) + { + throw new ArgumentException("Destination is too small.", nameof(chars)); + } + + for (var i = 0; i < bytes.Length; i++) + { + chars[i] = (char)bytes[i]; + } + + return bytes.Length; + } + + public override string GetString(byte[] bytes, int index, int count) => + string.Create( + count, + (bytes, index), + static (span, state) => + { + for (var i = 0; i < span.Length; i++) + { + span[i] = (char)state.bytes[state.index + i]; + } + } + ); + + public override int GetMaxByteCount(int charCount) => charCount; + + public override int GetMaxCharCount(int byteCount) => byteCount; +} diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index a7c77369..80f82a4d 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -4,7 +4,9 @@ namespace Lua.IO; -public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaStream, ILuaByteStream +public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings = false) + : ILuaStream, + ILuaByteStream { Utf8Reader? reader; ulong flushSize = ulong.MaxValue; @@ -19,14 +21,14 @@ public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaSt public ValueTask ReadLineAsync(bool keepEol, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); return new(reader.ReadLine(innerStream, keepEol)); } public ValueTask ReadAllAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); var text = reader.ReadToEnd(innerStream); return new(text); } @@ -57,7 +59,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadAsync(int count, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); if (count == 0) { return new(reader.IsEndOfStream(innerStream) ? null : string.Empty); @@ -69,7 +71,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadNumberAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); // Use the Utf8Reader's ReadNumber method which handles positioning correctly var numberStr = reader.ReadNumber(innerStream); @@ -93,7 +95,7 @@ public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cance } using PooledArray byteBuffer = new(4096); - var encoder = Encoding.UTF8.GetEncoder(); + var encoder = (byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8).GetEncoder(); var remainingChars = buffer.Span; var totalBytes = 0; while (!remainingChars.IsEmpty) diff --git a/src/Lua/Internal/StringHelper.cs b/src/Lua/Internal/StringHelper.cs index 25fe5deb..bf543bf9 100644 --- a/src/Lua/Internal/StringHelper.cs +++ b/src/Lua/Internal/StringHelper.cs @@ -379,6 +379,66 @@ public static Regex ToRegex(ReadOnlySpan pattern) return new(builder.ToString()); } + /// + /// toupper in the C locale: only 'a'-'z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToUpperAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'a' && s[i] <= 'z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'a' && c <= 'z' ? (char)(c - 32) : c; + } + } + ); + } + + /// + /// tolower in the C locale: only 'A'-'Z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToLowerAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'A' && s[i] <= 'Z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'A' && c <= 'Z' ? (char)(c + 32) : c; + } + } + ); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNumber(char c) { diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index 6c23e0e2..e6a4cc26 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; using Lua.CodeAnalysis.Compilation; +using Lua.IO; namespace Lua.Internal; @@ -13,14 +14,23 @@ sealed class Utf8Reader internal static bool scratchBufferUsed; readonly byte[] buffer; + readonly Encoding encoding; + readonly bool byteStrings; int bufPos, bufLen; Decoder? decoder; const int ThreadStaticBufferSize = 1024; - public Utf8Reader() + /// + /// When true, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, + /// like the reference C implementation. Otherwise the stream is decoded as UTF-8. + /// + public Utf8Reader(bool byteStrings = false) { + this.byteStrings = byteStrings; + encoding = byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8; + if (scratchBufferUsed) { buffer = new byte[ThreadStaticBufferSize]; @@ -46,7 +56,7 @@ public Utf8Reader() if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -92,7 +102,7 @@ ref lineLen } } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } else { @@ -107,7 +117,7 @@ ref lineLen return null; } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } finally { @@ -126,7 +136,7 @@ public string ReadToEnd(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -143,7 +153,7 @@ public string ReadToEnd(Stream stream) return ""; } - return Encoding.UTF8.GetString(resultBuffer, 0, len); + return encoding.GetString(resultBuffer, 0, len); } finally { @@ -209,7 +219,7 @@ public byte ReadByte(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -218,7 +228,7 @@ public byte ReadByte(Stream stream) ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); Span charSpan = new(resultBuffer, len, charCount - len); - decoder ??= Encoding.UTF8.GetDecoder(); + decoder ??= encoding.GetDecoder(); decoder.Convert( byteSpan, charSpan, @@ -269,6 +279,8 @@ static void AppendToBuffer(ref byte[] buffer, ReadOnlySpan segment, ref in length += segment.Length; } + static bool IsAsciiSpace(char c) => c == ' ' || (c >= '\t' && c <= '\r'); + public void Clear() { bufPos = 0; @@ -296,7 +308,7 @@ public void Clear() } var c = (char)b; - if (!char.IsWhiteSpace(c)) + if (byteStrings ? !IsAsciiSpace(c) : !char.IsWhiteSpace(c)) { break; } diff --git a/src/Lua/LuaByteString.cs b/src/Lua/LuaByteString.cs new file mode 100644 index 00000000..5f238aae --- /dev/null +++ b/src/Lua/LuaByteString.cs @@ -0,0 +1,44 @@ +using System.Text; +using Lua.IO; + +namespace Lua; + +/// +/// Conversions between .NET strings and Lua byte strings (see +/// ), where every holds one byte. +/// +public static class LuaByteString +{ + /// + /// Converts raw bytes (for example UTF-8 text) to a byte string. + /// + public static string FromBytes(ReadOnlySpan bytes) + { + return LuaByteEncoding.Instance.GetString(bytes); + } + + /// + /// Converts a byte string back to raw bytes. Chars above 0xFF are written as '?'. + /// + public static byte[] ToBytes(string byteString) + { + return LuaByteEncoding.Instance.GetBytes(byteString); + } + + /// + /// Converts a .NET string to a byte string holding its UTF-8 bytes. + /// + public static string FromString(string value) + { + return FromBytes(Encoding.UTF8.GetBytes(value)); + } + + /// + /// Decodes the UTF-8 bytes held by a byte string into a .NET string. + /// Invalid UTF-8 is replaced with U+FFFD. + /// + public static string ToString(string byteString) + { + return Encoding.UTF8.GetString(ToBytes(byteString)); + } +} diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 290ba387..d43c6dc9 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -1,8 +1,10 @@ using System.Buffers; using System.Runtime.CompilerServices; +using System.Text; using Lua.CodeAnalysis.Compilation; using Lua.Internal; using Lua.Internal.CompilerServices; +using Lua.IO; using Lua.Platforms; using Lua.Runtime; @@ -637,7 +639,17 @@ static bool AllowsMode(string mode, char chunkMode) throw new Exception("attempt to load a text chunk (mode is 'b')"); } - chunk = BomUtility.GetEncodingFromBytes(chunk, out var encoding); + Encoding encoding; + if (Platform.ByteStrings) + { + // Byte strings: like luaL_loadfile, skip a UTF-8 BOM and keep every other byte as one char. + chunk = BomUtility.SkipUtf8Bom(chunk); + encoding = LuaByteEncoding.Instance; + } + else + { + chunk = BomUtility.GetEncodingFromBytes(chunk, out encoding); + } var charCount = encoding.GetCharCount(chunk); var pooled = ArrayPool.Shared.Rent(charCount); diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index 009082cb..1332e268 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -16,6 +16,18 @@ public record LuaPlatform( TimeProvider TimeProvider ) { + /// + /// When true, Lua strings are byte strings: every holds one byte (0x00-0xFF), + /// so #s, string.len, string.sub and string.byte count bytes exactly + /// like the reference C implementation (UTF-8 text such as "ế" has length 3 instead of 1). + /// Character classes (%a, %w, ...), string.upper and string.lower follow the + /// C locale (ASCII only) and string.char rejects values above 255. + /// The and of the platform must read and write + /// bytes as well, which does. The default is false (UTF-16 strings). + /// Use to convert between .NET strings and byte strings. + /// + public bool ByteStrings { get; init; } + /// /// Standard console platform implementation. /// Uses real file system, console I/O, and system operations. @@ -27,4 +39,19 @@ TimeProvider TimeProvider StandardIO: new ConsoleStandardIO(), TimeProvider: TimeProvider.System ); + + /// + /// Same as , but with enabled: files and standard I/O + /// are read and written as raw bytes. + /// + public static LuaPlatform DefaultByteStrings { get; } = + new( + FileSystem: new FileSystem(byteStrings: true), + OsEnvironment: new SystemOsEnvironment(), + StandardIO: new ConsoleStandardIO(byteStrings: true), + TimeProvider: TimeProvider.System + ) + { + ByteStrings = true, + }; } diff --git a/src/Lua/Standard/Internal/MatchState.cs b/src/Lua/Standard/Internal/MatchState.cs index 3dc20f82..a56e3bf7 100644 --- a/src/Lua/Standard/Internal/MatchState.cs +++ b/src/Lua/Standard/Internal/MatchState.cs @@ -17,6 +17,9 @@ internal struct Capture public bool IsPosition => Len == CapPosition; } + // Byte strings use the C locale: classes only match ASCII, bytes 0x80-0xFF never belong to a class. + readonly bool byteStrings = state.Platform.ByteStrings; + public readonly LuaState State = state; public readonly string Source = source; public readonly string Pattern = pattern; @@ -444,8 +447,13 @@ int ClassEnd(ReadOnlySpan pattern, int pIdx) } } - static bool MatchClass(char c, char cl) + bool MatchClass(char c, char cl) { + if (byteStrings) + { + return MatchClassAscii(c, cl); + } + bool res; switch (char.ToLower(cl)) { @@ -495,12 +503,62 @@ static bool MatchClass(char c, char cl) return char.IsLower(cl) ? res : !res; } + static bool MatchClassAscii(char c, char cl) + { + bool res; + switch (char.ToLowerInvariant(cl)) + { + case 'a': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + break; + case 'c': + res = c < ' ' || c == 127; + break; + case 'd': + res = c >= '0' && c <= '9'; + break; + case 'g': + res = c > ' ' && c < 127; + break; + case 'l': + res = c >= 'a' && c <= 'z'; + break; + case 'p': + res = + c + is (>= '!' and <= '/') + or (>= ':' and <= '@') + or (>= '[' and <= '`') + or (>= '{' and <= '~'); + break; + case 's': + res = c == ' ' || (c >= '\t' && c <= '\r'); + break; + case 'u': + res = c >= 'A' && c <= 'Z'; + break; + case 'w': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + break; + case 'x': + res = IsHexDigit(c); + break; + case 'z': + res = c == '\0'; + break; + default: + return cl == c; + } + + return cl is >= 'a' and <= 'z' ? res : !res; + } + static bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } - static bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) + bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) { var sig = true; if (pIdx + 1 < pattern.Length && pattern[pIdx + 1] == '^') diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index 53bf4aa6..099f9c63 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -69,11 +69,17 @@ CancellationToken cancellationToken return new(context.Return("")); } + var byteStrings = context.State.Platform.ByteStrings; ValueStringBuilder builder = new(context.ArgumentCount); for (var i = 0; i < context.ArgumentCount; i++) { var arg = context.GetArgument(i); LuaRuntimeException.ThrowBadArgumentIfNumberIsNotInteger(context.State, i + 1, arg); + if (byteStrings && (arg < 0 || arg > 255)) + { + LuaRuntimeException.BadArgument(context.State, i + 1, "value out of range"); + } + builder.Append((char)arg); } @@ -412,7 +418,11 @@ context with } break; case 'c': - formattedValue = ((char)(int)x).ToString(); + formattedValue = ( + context.State.Platform.ByteStrings + ? (char)((int)x & 0xFF) + : (char)(int)x + ).ToString(); break; case 'x': { @@ -827,7 +837,11 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToLower())); + return new( + context.Return( + context.State.Platform.ByteStrings ? StringHelper.ToLowerAscii(s) : s.ToLower() + ) + ); } public ValueTask Match( @@ -1021,6 +1035,10 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToUpper())); + return new( + context.Return( + context.State.Platform.ByteStrings ? StringHelper.ToUpperAscii(s) : s.ToUpper() + ) + ); } } diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs new file mode 100644 index 00000000..c88e869c --- /dev/null +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -0,0 +1,500 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +// Expected values in this file were produced by the reference Lua 5.2.4 interpreter. +public sealed class ByteStringTests : IDisposable +{ + // VI is "Tien Len" with accents (Vietnamese) as UTF-8 bytes; the sources use \x escapes so this file stays ASCII. + const string Prelude = """ + local VI = "Ti\xE1\xBA\xBFn L\xC3\xAAn" + local E = "\xE1\xBA\xBF" + local function hex(s) return (s:gsub(".", function(c) return string.format("%02x", c:byte()) end)) end + + """; + + readonly string testDirectory = Path.Combine( + Path.GetTempPath(), + $"LuaByteStringTests_{Guid.NewGuid()}" + ); + + public ByteStringTests() + { + Directory.CreateDirectory(testDirectory); + } + + public void Dispose() + { + if (Directory.Exists(testDirectory)) + { + Directory.Delete(testDirectory, true); + } + } + + LuaState CreateByteState() + { + var state = LuaState.Create( + LuaPlatform.DefaultByteStrings with + { + FileSystem = new FileSystem(testDirectory, byteStrings: true), + } + ); + state.OpenStandardLibraries(); + return state; + } + + async Task RunAsync(string body) + { + using var state = CreateByteState(); + await state.DoStringAsync(Prelude + body); + } + + [Test] + public void DefaultPlatform_KeepsUtf16Strings() + { + Assert.That(LuaPlatform.Default.ByteStrings, Is.False); + Assert.That(LuaPlatform.DefaultByteStrings.ByteStrings, Is.True); + } + + [Test] + public async Task DefaultMode_StringLenCountsUtf16Chars() + { + var state = LuaState.Create(); + state.OpenStandardLibraries(); + var result = await state.DoStringAsync( + "return string.len('\u3042\u3044\u3046\u3048\u304A'), #'\u1EBF', ('\u1EBF'):find('%a')" + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(5))); + Assert.That(result[1], Is.EqualTo(new LuaValue(1))); + Assert.That(result[2], Is.EqualTo(new LuaValue(1))); + } + + [Test] + public Task Length_Byte_Char_Sub() => + RunAsync( + """ + assert(#VI == 11 and VI:len() == 11) + assert(#E == 3) + assert(#"\xE3\x81\x82\xE3\x81\x84\xE3\x81\x86\xE3\x81\x88\xE3\x81\x8A" == 15) + assert(#"\xF0\x9F\x98\x80" == 4) + assert(select("#", E:byte(1, -1)) == 3) + local a, b, c = E:byte(1, -1) + assert(a == 225 and b == 186 and c == 191) + assert(string.char(225, 186, 191) == E) + assert(hex(string.char(255, 0, 128)) == "ff0080") + assert(hex(VI:sub(3, 4)) == "e1ba") + assert(hex(VI:sub(-3)) == "c3aa6e") + assert(hex(E:sub(1, 1)) == "e1" and hex(E:sub(2)) == "babf") + assert(hex(E:reverse()) == "bfbae1") + assert(hex(E:rep(2, "|")) == "e1babf7ce1babf") + assert(hex(E .. "-" .. E) == "e1babf2de1babf") + """ + ); + + [Test] + public Task StringChar_RejectsValuesAboveByteRange() => + RunAsync( + """ + assert(not pcall(string.char, 256)) + assert(not pcall(string.char, -1)) + assert(not pcall(string.char, 65 + 256)) + local ok, err = pcall(string.char, 256) + assert(err:find("value out of range", 1, true)) + assert(hex(string.format("%c", 256 + 65)) == "41") + assert(hex(string.format("%c%c%c", 225, 186, 191)) == "e1babf") + """ + ); + + [Test] + public Task UpperLower_OnlyChangeAscii() => + RunAsync( + """ + assert(hex(("a\xC3\xA0b\xE1\xBA\xBF"):upper()) == "41c3a042e1babf") + assert(hex(("A\xC3\x80B\xE1\xBA\xBE"):lower()) == "61c38062e1babe") + local all = {} + for c = 0, 255 do all[#all + 1] = string.char(c) end + all = table.concat(all) + local up, low = all:upper(), all:lower() + assert(#up == 256 and #low == 256) + for c = 0, 255 do + local ch = string.char(c) + local wantUp = (c >= 97 and c <= 122) and c - 32 or c + local wantLow = (c >= 65 and c <= 90) and c + 32 or c + assert(up:byte(c + 1) == wantUp, "upper " .. c) + assert(low:byte(c + 1) == wantLow, "lower " .. c) + end + """ + ); + + // C locale ctype for every byte, written independently of the implementation. + static bool CType(char cls, int c) + { + var upper = c >= 'A' && c <= 'Z'; + var lower = c >= 'a' && c <= 'z'; + var digit = c >= '0' && c <= '9'; + return cls switch + { + 'a' => upper || lower, + 'c' => c < 32 || c == 127, + 'd' => digit, + 'g' => c > 32 && c < 127, + 'l' => lower, + 'p' => c > 32 && c < 127 && !(upper || lower || digit), + 's' => c == 32 || (c >= 9 && c <= 13), + 'u' => upper, + 'w' => upper || lower || digit, + 'x' => digit || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), + _ => throw new ArgumentOutOfRangeException(nameof(cls)), + }; + } + + [TestCase('a')] + [TestCase('c')] + [TestCase('d')] + [TestCase('g')] + [TestCase('l')] + [TestCase('p')] + [TestCase('s')] + [TestCase('u')] + [TestCase('w')] + [TestCase('x')] + public async Task CharacterClasses_FollowCLocaleForEveryByte(char cls) + { + var expected = new StringBuilder(); + var expectedComplement = new StringBuilder(); + for (var c = 0; c < 256; c++) + { + expected.Append(CType(cls, c) ? '1' : '0'); + expectedComplement.Append(CType(cls, c) ? '0' : '1'); + } + + var upperCls = char.ToUpperInvariant(cls); + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + local function bits(pattern) + local t = {} + for c = 0, 255 do t[#t + 1] = string.char(c):find(pattern) and "1" or "0" end + return table.concat(t) + end + return bits("%{{cls}}"), bits("%{{upperCls}}"), bits("[%{{cls}}]"), bits("[^%{{cls}}]") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[1].Read(), Is.EqualTo(expectedComplement.ToString())); + Assert.That(result[2].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[3].Read(), Is.EqualTo(expectedComplement.ToString())); + } + + [Test] + public Task Patterns_WorkOnBytes() => + RunAsync( + """ + local s, n = VI:gsub("%a+", "X") + assert(hex(s) == hex("X\xE1\xBA\xBFX X\xC3\xAAX") and n == 4) + assert(select(2, VI:gsub("%a+", "X")) == 4) + assert(E:find("%a") == nil) + assert(VI:find(E, 1, true) == 3) + assert(VI:find("L") == 8) + assert(VI:find("n", 7) == 11) + local c1, c2 = VI:match("^(.)(.)") + assert(hex(c1) == "54" and hex(c2) == "69") + assert(VI:match("()L") == 8) + assert(hex((E:gsub(".", "<%0>"))) == "3ce13e3cba3e3cbf3e") + local words = {} + for w in VI:gmatch("%S+") do words[#words + 1] = hex(w) end + assert(table.concat(words, ",") == "5469e1babf6e,4cc3aa6e") + local bytes = {} + for ch in E:gmatch(".") do bytes[#bytes + 1] = hex(ch) end + assert(table.concat(bytes, ",") == "e1,ba,bf") + assert(VI:gsub("%f[%a]%a+", "<%0>") == "\xE1\xBA\xBF \xC3\xAA") + assert(("(\xE1\xBA\xBF (x)) y"):match("%b()") == "(\xE1\xBA\xBF (x))") + """ + ); + + [Test] + public Task Format_PadsByBytes() => + RunAsync( + """ + assert(hex(string.format("%5s|%-5s|", E, E)) == "2020e1babf7c" .. "e1babf20207c") + assert(hex(string.format("%.2s", E)) == "e1ba") + assert(#string.format("%s", VI) == 11) + """ + ); + + [Test] + public Task Comparison_IsByteWise() => + RunAsync( + """ + assert("\x80" > "\x7F" and "\xFF" > "\x01" and "\x7F" < "\x80") + local w = { "\xC3\xA9", "e", "z", "\xE1\xBA\xBF", "a", "\xC3\x80", "Z", "\xC3\xAA" } + table.sort(w) + local h = {} + for i, x in ipairs(w) do h[i] = hex(x) end + assert(table.concat(h, ",") == "5a,61,65,7a,c380,c3a9,c3aa,e1babf") + """ + ); + + [Test] + public async Task LoadString_KeepsSourceBytes() + { + using var state = CreateByteState(); + var source = LuaByteString.FromString("return 'Ti\u1EBFn', #'Ti\u1EBFn', '\u1EBF'"); + var result = await state.DoStringAsync(source); + + Assert.That(result[0].Read(), Is.EqualTo(LuaByteString.FromString("Ti\u1EBFn"))); + Assert.That(result[1], Is.EqualTo(new LuaValue(6))); + Assert.That( + LuaByteString.ToBytes(result[2].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + } + + [Test] + public async Task LoadBytes_SkipsOnlyUtf8Bom() + { + using var state = CreateByteState(); + var withBom = new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray(); + var result = await state.ExecuteAsync(state.Load(withBom, "bom", "t")); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task LoadFile_ReadsUtf8SourceAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "chunk.lua"), + new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray() + ); + + using var state = CreateByteState(); + var closure = await state.LoadFileAsync("chunk.lua", "bt", null, CancellationToken.None); + var result = await state.ExecuteAsync(closure); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task Require_LoadsUtf8ModuleAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "viet.lua"), + Encoding.UTF8.GetBytes("return { s = 'Ti\u1EBFn L\u00EAn' }") + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{testDirectory.Replace("\\", "/")}}/?.lua" + local m = require("viet") + return #m.s + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(11))); + } + + [Test] + public async Task Io_WritesAndReadsRawBytes() + { + var data = new byte[] + { + 0x54, + 0x69, + 0xE1, + 0xBA, + 0xBF, + 0x6E, + 0x0A, + 0xE1, + 0xBA, + 0xBF, + 0xFF, + 0x00, + 0x80, + 0x7A, + 0x0D, + 0x0A, + 0x65, + 0x6E, + 0x64, + }; + using var state = CreateByteState(); + state.Environment["data"] = LuaByteString.FromBytes(data); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("raw.bin", "wb")) + f:write(data) + f:close() + f = assert(io.open("raw.bin", "rb")) + local size = f:seek("end") + f:seek("set", 0) + local all = f:read("*a") + f:close() + f = assert(io.open("raw.bin", "rb")) + local l1 = f:read("*l") + local n2 = f:read(2) + f:close() + return size, all == data, #l1, #n2 + """ + ); + + Assert.That(File.ReadAllBytes(Path.Combine(testDirectory, "raw.bin")), Is.EqualTo(data)); + Assert.That(result[0], Is.EqualTo(new LuaValue(data.Length))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(6))); + Assert.That(result[3], Is.EqualTo(new LuaValue(2))); + } + + [Test] + public async Task Io_DoesNotSkipBomOnRead() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "bom.bin"), + new byte[] { 0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63 } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("bom.bin", "rb")) + local all = f:read("*a") + f:close() + f = assert(io.open("bom.bin", "rb")) + local n3 = f:read(3) + local rest = f:read("*l") + f:close() + return #all, #n3, rest + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + Assert.That(result[2].Read(), Is.EqualTo("abc")); + } + + [Test] + public async Task Io_NumberReadingTreatsHighBytesAsNotWhitespace() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "num.bin"), + new byte[] + { + 0x20, + 0x20, + 0x31, + 0x32, + 0x20, + 0x30, + 0x78, + 0x31, + 0x46, + 0x20, + 0xA0, + 0x37, + 0x0A, + } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("num.bin", "rb")) + local a, b, c = f:read("*n", "*n", "*n") + f:close() + return a, b, c + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(12))); + Assert.That(result[1], Is.EqualTo(new LuaValue(31))); + Assert.That(result[2], Is.EqualTo(LuaValue.Nil)); + } + + [Test] + public async Task LuaStream_ByteMode_RoundTripsEveryByte() + { + var all = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray(); + using var memory = new MemoryStream(); + using (var writer = new LuaStream(LuaFileOpenMode.Write, memory, byteStrings: true)) + { + await writer.WriteAsync( + LuaByteString.FromBytes(all).AsMemory(), + CancellationToken.None + ); + await writer.FlushAsync(CancellationToken.None); + Assert.That(memory.ToArray(), Is.EqualTo(all)); + } + + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(all), + byteStrings: true + ); + var text = await reader.ReadAllAsync(CancellationToken.None); + Assert.That(LuaByteString.ToBytes(text), Is.EqualTo(all)); + } + + [Test] + public async Task LuaStream_DefaultMode_StillEncodesUtf8() + { + using var memory = new MemoryStream(); + var writer = new LuaStream(LuaFileOpenMode.Write, memory); + await writer.WriteAsync("\u1EBF".AsMemory(), CancellationToken.None); + await writer.FlushAsync(CancellationToken.None); + + Assert.That(memory.ToArray(), Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF })); + } + + [Test] + public async Task LuaStream_ByteMode_ReadCountCountsBytes() + { + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(new byte[] { 0xE1, 0xBA, 0xBF, 0x61 }), + byteStrings: true + ); + + var first = await reader.ReadAsync(2, CancellationToken.None); + var second = await reader.ReadAsync(2, CancellationToken.None); + + Assert.That(LuaByteString.ToBytes(first!), Is.EqualTo(new byte[] { 0xE1, 0xBA })); + Assert.That(LuaByteString.ToBytes(second!), Is.EqualTo(new byte[] { 0xBF, 0x61 })); + } + + [Test] + public void LuaByteString_ConvertsBetweenDotNetStringsAndBytes() + { + const string text = "Ti\u1EBFn L\u00EAn \U0001F600"; + var byteString = LuaByteString.FromString(text); + + Assert.That(byteString.Length, Is.EqualTo(Encoding.UTF8.GetByteCount(text))); + Assert.That(byteString.All(c => c <= 0xFF), Is.True); + Assert.That(LuaByteString.ToString(byteString), Is.EqualTo(text)); + Assert.That(LuaByteString.ToBytes(byteString), Is.EqualTo(Encoding.UTF8.GetBytes(text))); + Assert.That( + LuaByteString.FromBytes(new byte[] { 0xFF, 0x00, 0x80 }), + Is.EqualTo("\u00FF\0\u0080") + ); + Assert.That(LuaByteString.ToString("\u00FF"), Is.EqualTo("\uFFFD")); + Assert.That(LuaByteString.ToBytes("\u1EBF"), Is.EqualTo(new byte[] { (byte)'?' })); + } +} From e8c6c72ff349f78a45db973c9a7e9aec68caa5da Mon Sep 17 00:00:00 2001 From: DEVN <10086005+billythekidz@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:35:01 +0700 Subject: [PATCH 02/11] Document opt-in byte strings in the README --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 5909f922..9526ddf0 100644 --- a/README.md +++ b/README.md @@ -684,6 +684,26 @@ print(l) All string library functions handle strings as UTF-16. +#### Byte strings (opt-in) + +If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), enable `LuaPlatform.ByteStrings`. `LuaPlatform.DefaultByteStrings` is the default platform with byte strings and byte-oriented file and standard I/O: + +```cs +using Lua; +using Lua.Platforms; +using Lua.Standard; + +var state = LuaState.Create(LuaPlatform.DefaultByteStrings); +state.OpenStandardLibraries(); + +// Every char of a Lua string holds one byte, so convert .NET strings at the boundary. +state.Environment["name"] = LuaByteString.FromString("Tiến Lên"); // UTF-8 bytes, #name == 11 +var results = await state.DoStringAsync("return #name, name:upper()"); +var upper = LuaByteString.ToString(results[1].Read()); // back to a .NET string +``` + +In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16. + ### Garbage Collection Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua. From 949a9e4dfd3767b2b4274a76e71ce3a4d6ed9588 Mon Sep 17 00:00:00 2001 From: DEVN <10086005+billythekidz@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:35:01 +0700 Subject: [PATCH 03/11] Byte strings: decode file names and encode environment values and OS error text --- src/Lua/IO/FileSystem.cs | 19 +++++-- src/Lua/Platforms/LuaPlatform.cs | 6 ++ src/Lua/Standard/IOLibrary.cs | 18 +++++- src/Lua/Standard/OperatingSystemLibrary.cs | 21 +++++-- tests/Lua.Tests/ByteStringTests.cs | 64 ++++++++++++++++++++++ 5 files changed, 115 insertions(+), 13 deletions(-) diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 7843985c..1539d95b 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -38,9 +38,15 @@ public string GetFullPath(string path) return Path.Combine(baseDirectory, path); } + // Paths come from Lua as byte strings in byte mode: decode the UTF-8 bytes into the real file name. + string ResolvePath(string path) + { + return GetFullPath(byteStrings ? LuaByteString.ToString(path) : path); + } + public bool IsReadable(string path) { - return File.Exists(GetFullPath(path)); + return File.Exists(ResolvePath(path)); } public ValueTask Open( @@ -51,7 +57,7 @@ CancellationToken cancellationToken { var (mode, access) = GetFileMode(openMode); Stream stream; - path = GetFullPath(path); + path = ResolvePath(path); if (openMode == LuaFileOpenMode.AppendUpdate) { stream = File.Open( @@ -78,8 +84,8 @@ CancellationToken cancellationToken public ValueTask Rename(string oldName, string newName, CancellationToken cancellationToken) { - oldName = GetFullPath(oldName); - newName = GetFullPath(newName); + oldName = ResolvePath(oldName); + newName = ResolvePath(newName); if (oldName == newName) { return default; @@ -97,7 +103,7 @@ public ValueTask Rename(string oldName, string newName, CancellationToken cancel public ValueTask Remove(string path, CancellationToken cancellationToken) { - path = GetFullPath(path); + path = ResolvePath(path); if (!File.Exists(path)) { throw new FileNotFoundException("No such file or directory", path); @@ -113,7 +119,8 @@ public ValueTask Remove(string path, CancellationToken cancellationToken) public string GetTempFileName() { - return Path.GetTempFileName(); + var name = Path.GetTempFileName(); + return byteStrings ? LuaByteString.FromString(name) : name; } public ValueTask OpenTempFileStream(CancellationToken cancellationToken) diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index 1332e268..70befa6c 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -28,6 +28,12 @@ TimeProvider TimeProvider /// public bool ByteStrings { get; init; } + // Converts a .NET string (file name, environment value, error text) into a Lua string for this platform. + internal string ToLuaString(string value) + { + return ByteStrings ? LuaByteString.FromString(value) : value; + } + /// /// Standard console platform implementation. /// Uses real file system, console I/O, and system operations. diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index c8ed4f3e..04ec0ecd 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -44,7 +44,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -62,7 +66,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -213,7 +221,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } diff --git a/src/Lua/Standard/OperatingSystemLibrary.cs b/src/Lua/Standard/OperatingSystemLibrary.cs index 318c4542..3d1870ab 100644 --- a/src/Lua/Standard/OperatingSystemLibrary.cs +++ b/src/Lua/Standard/OperatingSystemLibrary.cs @@ -156,10 +156,15 @@ CancellationToken cancellationToken ) { var variable = context.GetArgument(0); + var byteStrings = context.GlobalState.Platform.ByteStrings; + var value = context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable( + byteStrings ? LuaByteString.ToString(variable) : variable + ); return new( context.Return( - context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable(variable) - ?? LuaValue.Nil + value is not null && byteStrings ? LuaByteString.FromString(value) + : value is not null ? value + : LuaValue.Nil ) ); } @@ -177,7 +182,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -199,7 +208,11 @@ await context.GlobalState.Platform.FileSystem.Rename( } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs index c88e869c..2ceb825e 100644 --- a/tests/Lua.Tests/ByteStringTests.cs +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -364,6 +364,70 @@ public async Task Io_WritesAndReadsRawBytes() Assert.That(result[3], Is.EqualTo(new LuaValue(2))); } + [Test] + public async Task FileNames_AreDecodedFromUtf8Bytes() + { + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("t\xE1\xBA\xBFn.bin", "wb")) + f:write("x") + f:close() + local g = assert(io.open("r\xC3\xAA.bin", "wb")) + g:close() + local renamed = os.rename("r\xC3\xAA.bin", "r2\xC3\xAA.bin") + local gone = io.open("r\xC3\xAA.bin", "rb") == nil + local removed = os.remove("r2\xC3\xAA.bin") + local ok, msg = io.open("missing_\xE1\xBA\xBF.bin", "rb") + return renamed, gone, removed, ok == nil, type(msg) + """ + ); + + Assert.That(File.Exists(Path.Combine(testDirectory, "t\u1EBFn.bin")), Is.True); + Assert.That(File.Exists(Path.Combine(testDirectory, "r\u00EA.bin")), Is.False); + Assert.That(File.Exists(Path.Combine(testDirectory, "r2\u00EA.bin")), Is.False); + Assert.That(result[0], Is.EqualTo(new LuaValue(true))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(true))); + Assert.That(result[3], Is.EqualTo(new LuaValue(true))); + Assert.That(result[4].Read(), Is.EqualTo("string")); + } + + [Test] + public async Task Require_FindsModuleWithNonAsciiName() + { + await File.WriteAllTextAsync(Path.Combine(testDirectory, "m\u1EBFd.lua"), "return 'ok'"); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{LuaByteString.FromString(testDirectory.Replace("\\", "/"))}}/?.lua" + return require("m\xE1\xBA\xBFd") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo("ok")); + } + + [Test] + public async Task OsGetenv_ReturnsUtf8Bytes() + { + const string name = "LUA_CSHARP_BYTE_STRINGS_TEST"; + Environment.SetEnvironmentVariable(name, "Ti\u1EBFn"); + try + { + using var state = CreateByteState(); + var result = await state.DoStringAsync($"local v = os.getenv('{name}'); return #v, v"); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(LuaByteString.ToString(result[1].Read()), Is.EqualTo("Ti\u1EBFn")); + } + finally + { + Environment.SetEnvironmentVariable(name, null); + } + } + [Test] public async Task Io_DoesNotSkipBomOnRead() { From 0f4a8a285b8c8f99e10a2ef053085e8c5ab8603f Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:00:12 +0900 Subject: [PATCH 04/11] Complete byte-string I/O, compatibility, and host module boundaries --- README.md | 27 +++ sandbox/Benchmark/ByteStringIoBenchmark.cs | 49 ++++ .../Runtime/LuaThreadAssetExtensions.cs | 13 +- src/Lua/Exceptions.cs | 9 +- src/Lua/ILuaModuleLoader.cs | 5 + src/Lua/IO/ConsoleStandardIO.cs | 5 +- src/Lua/IO/FileSystem.cs | 6 +- src/Lua/IO/LuaByteEncoding.cs | 34 +++ src/Lua/IO/LuaStream.cs | 36 ++- src/Lua/Internal/Utf8Reader.cs | 15 +- src/Lua/LuaStateExtensions.cs | 24 ++ src/Lua/Platforms/LuaPlatform.cs | 9 + src/Lua/Standard/BasicLibrary.cs | 15 +- src/Lua/Standard/FileHandle.cs | 20 +- src/Lua/Standard/Internal/IOHelper.cs | 6 +- src/Lua/Standard/ModuleLibrary.cs | 13 +- tests/Lua.Tests/ByteStringIntegrationTests.cs | 209 ++++++++++++++++++ 17 files changed, 456 insertions(+), 39 deletions(-) create mode 100644 sandbox/Benchmark/ByteStringIoBenchmark.cs create mode 100644 tests/Lua.Tests/ByteStringIntegrationTests.cs diff --git a/README.md b/README.md index 9526ddf0..6b242984 100644 --- a/README.md +++ b/README.md @@ -704,6 +704,33 @@ var upper = LuaByteString.ToString(results[1].Read()); // back to a .N In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16. +Choose one representation for a state and configure it before executing scripts. Setting `ByteStrings = true` on an otherwise unchanged platform does not switch its streams to raw bytes. When configuring a file root, keep that root as a normal .NET path: + +```cs +var platform = LuaPlatform.DefaultByteStrings with +{ + FileSystem = new Lua.IO.FileSystem(baseDirectory: scriptsDirectory, byteStrings: true) +}; +using var byteState = LuaState.Create(platform); +byteState.OpenStandardLibraries(); + +// A LuaModule carries normal .NET text. LoadModule converts its source and name. +var module = new LuaModule("greeting.lua", "return 'こんにちは'"); +var values = await byteState.ExecuteAsync(byteState.LoadModule(module)); +var greeting = LuaByteString.ToString(values[0].Read()); + +// Raw packets can contain NUL, invalid UTF-8 and every value from 0 to 255. +byteState.Environment["packet"] = LuaByteString.FromBytes(new byte[] { 0xFF, 0, 0x80 }); +var packet = await byteState.DoStringAsync("return packet:reverse()"); +byte[] reversed = LuaByteString.ToBytes(packet[0].Read()); +``` + +Use `FromString`/`ToString` for text and `FromBytes`/`ToBytes` for binary data. This also applies to non-ASCII table keys and the arguments/results of C# functions and generated bindings; interop does not convert strings automatically. `DoStringAsync` and `Load` with character source accept Lua strings, so encode non-ASCII source with `FromString`, or use `LoadModule` for .NET text. Byte source passed to `Load` and source read from files are preserved directly. + +Names passed to `ILuaModuleLoader` are normal .NET strings. Text modules are treated as .NET text and encoded to UTF-8 in byte mode; byte modules remain bytes. Unity text assets use the same conversion. Custom file systems and standard-I/O providers must implement the selected stream representation themselves; use `ILuaStream.CreateFromStream(stream, mode, byteStrings: true)` for raw streams. Filenames passed directly to the byte-mode `FileSystem`, `DoFileAsync` or `LoadFileAsync` are Lua byte strings, while `FileSystem.BaseDirectory` is a .NET path. + +Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `ToString`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters. + ### Garbage Collection Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua. diff --git a/sandbox/Benchmark/ByteStringIoBenchmark.cs b/sandbox/Benchmark/ByteStringIoBenchmark.cs new file mode 100644 index 00000000..adbcd3ee --- /dev/null +++ b/sandbox/Benchmark/ByteStringIoBenchmark.cs @@ -0,0 +1,49 @@ +using BenchmarkDotNet.Attributes; +using Lua.IO; + +[Config(typeof(BenchmarkConfig))] +public class ByteStringIoBenchmark +{ + [Params(false, true)] + public bool ByteStrings { get; set; } + + [Params(1, 4096)] + public int Size { get; set; } + + MemoryStream input = default!; + MemoryStream output = default!; + LuaStream reader = default!; + LuaStream writer = default!; + string text = default!; + + [GlobalSetup] + public void Setup() + { + text = new string('a', Size); + input = new MemoryStream(Enumerable.Repeat((byte)'a', Size).ToArray()); + output = new MemoryStream(Size); + reader = new LuaStream(LuaFileOpenMode.Read, input, ByteStrings); + writer = new LuaStream(LuaFileOpenMode.Write, output, ByteStrings); + } + + [GlobalCleanup] + public void Cleanup() + { + reader.Dispose(); + writer.Dispose(); + } + + [Benchmark] + public string? Read() + { + reader.Seek(SeekOrigin.Begin, 0); + return reader.ReadAsync(Size, default).GetAwaiter().GetResult(); + } + + [Benchmark] + public void Write() + { + output.Position = 0; + writer.WriteAsync(text.AsMemory(), default).GetAwaiter().GetResult(); + } +} diff --git a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs index 5003d6fb..6e007250 100644 --- a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs +++ b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs @@ -21,10 +21,7 @@ public static ValueTask ExecuteAsync( } var module = luaAssetBase.GetModule(name); - var closure = - module.Type == LuaModuleType.Bytes - ? state.Load(module.ReadBytes(), module.Name) - : state.Load(module.ReadText(), module.Name); + var closure = state.LoadModule(module); return state.ExecuteAsync(closure, buffer, cancellationToken); } @@ -35,7 +32,13 @@ public static ValueTask ExecuteAsync( CancellationToken cancellationToken = default ) { - return state.ExecuteAsync(luaAssetBase, name, cancellationToken); + if (luaAssetBase == null) + { + throw new ArgumentNullException(nameof(luaAssetBase)); + } + + var module = luaAssetBase.GetModule(name); + return state.ExecuteAsync(state.LoadModule(module), cancellationToken); } } } diff --git a/src/Lua/Exceptions.cs b/src/Lua/Exceptions.cs index a95a2b79..cf53a76f 100644 --- a/src/Lua/Exceptions.cs +++ b/src/Lua/Exceptions.cs @@ -2,6 +2,7 @@ using Lua.CodeAnalysis; using Lua.CodeAnalysis.Syntax; using Lua.Internal; +using Lua.Platforms; using Lua.Runtime; namespace Lua; @@ -490,9 +491,13 @@ internal void Forget() State = null; } - internal string MinimalMessage() + internal string MinimalMessage(LuaPlatform platform) { - var message = InnerException?.ToString() ?? ErrorObject.ToString(); + var message = InnerException is { } inner + ? platform.ByteStrings + ? platform.GetErrorMessage(inner) + : inner.ToString() + : ErrorObject.ToString(); if (level <= 0) { return message; diff --git a/src/Lua/ILuaModuleLoader.cs b/src/Lua/ILuaModuleLoader.cs index db998eff..20cf4182 100644 --- a/src/Lua/ILuaModuleLoader.cs +++ b/src/Lua/ILuaModuleLoader.cs @@ -2,7 +2,12 @@ namespace Lua; public interface ILuaModuleLoader { + /// Checks a module name expressed as normal .NET text, including in byte-string mode. bool Exists(string moduleName); + + /// + /// Loads a host module. Names and text source use .NET text; byte source is kept raw. + /// ValueTask LoadAsync( string moduleName, CancellationToken cancellationToken = default diff --git a/src/Lua/IO/ConsoleStandardIO.cs b/src/Lua/IO/ConsoleStandardIO.cs index 91cff485..43e3da3b 100644 --- a/src/Lua/IO/ConsoleStandardIO.cs +++ b/src/Lua/IO/ConsoleStandardIO.cs @@ -9,8 +9,11 @@ namespace Lua.IO; /// When true, standard input and output are read and written as raw bytes (one byte per char) /// instead of UTF-8 text. Use together with . /// -public sealed class ConsoleStandardIO(bool byteStrings = false) : ILuaStandardIO +public sealed class ConsoleStandardIO(bool byteStrings) : ILuaStandardIO { + public ConsoleStandardIO() + : this(false) { } + ILuaStream? standardInput; ILuaStream? standardOutput; ILuaStream? standardError; diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 65fb0a05..dbc638f1 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -8,6 +8,9 @@ public sealed class FileSystem(string? baseDirectory = null, bool byteStrings = false) : ILuaFileSystem { + public FileSystem(string? baseDirectory) + : this(baseDirectory, false) { } + public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory(); public static (FileMode, FileAccess access) GetFileMode(LuaFileOpenMode luaFileOpenMode) @@ -149,7 +152,8 @@ public ValueTask OpenTempFileStream(CancellationToken cancellationTo return new( new LuaStream( LuaFileOpenMode.WriteUpdate, - File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite) + File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite), + byteStrings ) ); } diff --git a/src/Lua/IO/LuaByteEncoding.cs b/src/Lua/IO/LuaByteEncoding.cs index bdc03d83..bd27ca6c 100644 --- a/src/Lua/IO/LuaByteEncoding.cs +++ b/src/Lua/IO/LuaByteEncoding.cs @@ -18,6 +18,24 @@ sealed class LuaByteEncoding : Encoding public override int GetByteCount(ReadOnlySpan chars) => chars.Length; + // Encoding's pointer defaults copy through temporary arrays. Its string, + // encoder and decoder APIs use these overloads even when span overrides exist. + public override unsafe int GetByteCount(char* chars, int count) + { + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + return new ReadOnlySpan(chars, count).Length; + } + + public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) + { + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + return GetBytes(new ReadOnlySpan(chars, charCount), new Span(bytes, byteCount)); + } + public override int GetBytes( char[] chars, int charIndex, @@ -46,6 +64,22 @@ public override int GetBytes(ReadOnlySpan chars, Span bytes) public override int GetCharCount(ReadOnlySpan bytes) => bytes.Length; + public override unsafe int GetCharCount(byte* bytes, int count) + { + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + return new ReadOnlySpan(bytes, count).Length; + } + + public override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount) + { + if (bytes == null) + throw new ArgumentNullException(nameof(bytes)); + if (chars == null) + throw new ArgumentNullException(nameof(chars)); + return GetChars(new ReadOnlySpan(bytes, byteCount), new Span(chars, charCount)); + } + public override int GetChars( byte[] bytes, int byteIndex, diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index 80f82a4d..1907bd71 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -4,10 +4,13 @@ namespace Lua.IO; -public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings = false) +public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings) : ILuaStream, ILuaByteStream { + public LuaStream(LuaFileOpenMode mode, Stream innerStream) + : this(mode, innerStream, false) { } + Utf8Reader? reader; ulong flushSize = ulong.MaxValue; ulong nextFlushSize = ulong.MaxValue; @@ -95,23 +98,34 @@ public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cance } using PooledArray byteBuffer = new(4096); - var encoder = (byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8).GetEncoder(); + var bytes = byteBuffer.AsSpan(); + var encoder = byteStrings ? null : Encoding.UTF8.GetEncoder(); var remainingChars = buffer.Span; var totalBytes = 0; while (!remainingChars.IsEmpty) { - encoder.Convert( - remainingChars, - byteBuffer.AsSpan(), - flush: true, - out var charsUsed, - out var bytesUsed, - out _ - ); + int charsUsed, + bytesUsed; + if (byteStrings) + { + charsUsed = Math.Min(remainingChars.Length, bytes.Length); + bytesUsed = LuaByteEncoding.Instance.GetBytes(remainingChars[..charsUsed], bytes); + } + else + { + encoder!.Convert( + remainingChars, + bytes, + flush: true, + out charsUsed, + out bytesUsed, + out _ + ); + } if (bytesUsed > 0) { - innerStream.Write(byteBuffer.AsSpan()[..bytesUsed]); + innerStream.Write(bytes[..bytesUsed]); totalBytes += bytesUsed; } diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index e6a4cc26..83045718 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -64,7 +64,10 @@ public Utf8Reader(bool byteStrings = false) } Span span = new(buffer, bufPos, bufLen - bufPos); - var idx = span.IndexOfAny((byte)'\r', (byte)'\n'); + // Raw Lua byte streams end a line at LF; CR remains part of the data. + var idx = byteStrings + ? span.IndexOf((byte)'\n') + : span.IndexOfAny((byte)'\r', (byte)'\n'); if (idx >= 0) { @@ -228,6 +231,16 @@ public byte ReadByte(Stream stream) ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); Span charSpan = new(resultBuffer, len, charCount - len); + if (byteStrings) + { + var count = Math.Min(byteSpan.Length, charSpan.Length); + LuaByteEncoding.Instance.GetChars(byteSpan[..count], charSpan); + len += count; + bufPos += count; + dataRead = true; + continue; + } + decoder ??= encoding.GetDecoder(); decoder.Convert( byteSpan, diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index 1b751994..186b5a08 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -1,5 +1,6 @@ using System.Buffers; using System.Runtime.CompilerServices; +using System.Text; using Lua.Internal; using Lua.Internal.CompilerServices; using Lua.IO; @@ -11,6 +12,29 @@ namespace Lua; public static class LuaStateExtensions { + /// + /// Loads a host module. Its name and text source are .NET text; byte source is + /// preserved. In byte-string mode, text source is encoded as UTF-8 before parsing. + /// + public static LuaClosure LoadModule(this LuaState state, LuaModule module) + { + var name = state.Platform.ToLuaString(module.Name); + if (module.Type == LuaModuleType.Bytes) + { + return state.Load(module.ReadBytes(), name); + } + + if (!state.Platform.ByteStrings) + { + return state.Load(module.ReadText(), name); + } + + var source = module.ReadText(); + using PooledArray bytes = new(Encoding.UTF8.GetByteCount(source)); + var count = Encoding.UTF8.GetBytes(source, bytes.AsSpan()); + return state.Load(bytes.AsSpan()[..count], name); + } + public static async ValueTask LoadFileAsync( this LuaState state, string fileName, diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index 70befa6c..b0aaa2a0 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -34,6 +34,15 @@ internal string ToLuaString(string value) return ByteStrings ? LuaByteString.FromString(value) : value; } + internal string GetErrorMessage(Exception exception) + { + // Lua diagnostics already contain Lua strings (including chunk names). + // Only host exceptions carry .NET text that needs UTF-8 conversion. + return exception is LuaRuntimeException or LuaParseException or LuaCompileException + ? exception.Message + : ToLuaString(exception.Message); + } + /// /// Standard console platform implementation. /// Uses real file system, console I/O, and system operations. diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 5d42540d..5323305e 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -232,7 +232,7 @@ CancellationToken cancellationToken } catch (Exception ex) { - return context.Return(LuaValue.Nil, ex.Message); + return context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex)); } } @@ -286,7 +286,7 @@ CancellationToken cancellationToken } catch (Exception ex) { - return new(context.Return(LuaValue.Nil, ex.Message)); + return new(context.Return(LuaValue.Nil, context.State.Platform.GetErrorMessage(ex))); } } @@ -384,12 +384,12 @@ CancellationToken cancellationToken } using PooledList builder = new(); - var message = luaEx.MinimalMessage(); + var message = luaEx.MinimalMessage(context.State.Platform); luaEx.Forget(); return context.Return(false, message); } default: - return context.Return(false, ex.Message); + return context.Return(false, context.State.Platform.GetErrorMessage(ex)); } } } @@ -768,12 +768,15 @@ CancellationToken cancellationToken if (ex is LuaRuntimeException luaEx) { + var error = luaEx.InnerException is { } inner + ? state.Platform.GetErrorMessage(inner) + : luaEx.ErrorObject; luaEx.Forget(); - state.Push(luaEx.ErrorObject); + state.Push(error); } else { - state.Push(ex.Message); + state.Push(state.Platform.GetErrorMessage(ex)); } // invoke error handler diff --git a/src/Lua/Standard/FileHandle.cs b/src/Lua/Standard/FileHandle.cs index 3fc11ce2..835c2485 100644 --- a/src/Lua/Standard/FileHandle.cs +++ b/src/Lua/Standard/FileHandle.cs @@ -146,7 +146,11 @@ public async ValueTask Close(CancellationToken cancellationToken) } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } ); @@ -164,7 +168,11 @@ public async ValueTask Close(CancellationToken cancellationToken) } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } ); @@ -245,7 +253,13 @@ static async (context, cancellationToken) => } catch (IOException ex) { - return new(context.Return(LuaValue.Nil, ex.Message, ex.HResult)); + return new( + context.Return( + LuaValue.Nil, + context.State.Platform.ToLuaString(ex.Message), + ex.HResult + ) + ); } } ); diff --git a/src/Lua/Standard/Internal/IOHelper.cs b/src/Lua/Standard/Internal/IOHelper.cs index 45738bbd..82d00227 100644 --- a/src/Lua/Standard/Internal/IOHelper.cs +++ b/src/Lua/Standard/Internal/IOHelper.cs @@ -39,7 +39,7 @@ CancellationToken cancellationToken } state.Stack.Push(LuaValue.Nil); - state.Stack.Push(ex.Message); + state.Stack.Push(state.Platform.ToLuaString(ex.Message)); state.Stack.Push(ex.HResult); return 3; } @@ -88,7 +88,7 @@ await file.WriteAsync( context.State.Stack.PopUntil(context.ReturnFrameBase); var stack = context.State.Stack; stack.Push(LuaValue.Nil); - stack.Push(ex.Message); + stack.Push(context.State.Platform.ToLuaString(ex.Message)); stack.Push(ex.HResult); return 3; } @@ -200,7 +200,7 @@ CancellationToken cancellationToken stack.PopUntil(top); stack.Push(LuaValue.Nil); - stack.Push(ex.Message); + stack.Push(state.Platform.ToLuaString(ex.Message)); stack.Push(ex.HResult); return 3; } diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index 994be706..c1225fbf 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -31,13 +31,14 @@ CancellationToken cancellationToken { LuaFunction loader; var moduleLoader = context.GlobalState.ModuleLoader; - if (moduleLoader != null && moduleLoader.Exists(arg0)) + var hostName = + moduleLoader != null && context.State.Platform.ByteStrings + ? LuaByteString.ToString(arg0) + : arg0; + if (moduleLoader != null && moduleLoader.Exists(hostName)) { - var module = await moduleLoader.LoadAsync(arg0, cancellationToken); - loader = - module.Type == LuaModuleType.Bytes - ? context.State.Load(module.ReadBytes(), module.Name) - : context.State.Load(module.ReadText(), module.Name); + var module = await moduleLoader.LoadAsync(hostName, cancellationToken); + loader = context.State.LoadModule(module); } else { diff --git a/tests/Lua.Tests/ByteStringIntegrationTests.cs b/tests/Lua.Tests/ByteStringIntegrationTests.cs new file mode 100644 index 00000000..f9959d1c --- /dev/null +++ b/tests/Lua.Tests/ByteStringIntegrationTests.cs @@ -0,0 +1,209 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +public sealed class ByteStringIntegrationTests : IDisposable +{ + readonly string directory = Path.Combine( + Path.GetTempPath(), + $"LuaBytes_\u1EBF_{Guid.NewGuid()}" + ); + + public ByteStringIntegrationTests() => Directory.CreateDirectory(directory); + + public void Dispose() => Directory.Delete(directory, true); + + LuaState CreateState() + { + var state = LuaState.Create( + LuaPlatform.DefaultByteStrings with + { + FileSystem = new FileSystem(directory, true), + } + ); + state.OpenStandardLibraries(); + return state; + } + + [Test] + public void ExistingConstructorSignaturesRemainAvailable() + { + Assert.That(typeof(FileSystem).GetConstructor([typeof(string)]), Is.Not.Null); + Assert.That( + typeof(LuaStream).GetConstructor([typeof(LuaFileOpenMode), typeof(Stream)]), + Is.Not.Null + ); + Assert.That(typeof(ConsoleStandardIO).GetConstructor(Type.EmptyTypes), Is.Not.Null); + } + + [Test] + public async Task TemporaryFilesPreserveRawBytesAndPositions() + { + using var state = CreateState(); + await state.DoStringAsync( + """ + local bytes = string.char(239, 187, 191, 0, 128, 255) + local f = assert(io.tmpfile()) + f:write(bytes) + assert(f:seek("end") == #bytes) + f:seek("set", 0) + assert(f:read(1):byte() == 239) + assert(f:seek() == 1) + f:seek("set", 0) + assert(f:read("*a") == bytes) + f:close() + """ + ); + } + + [TestCase("local f,e=loadfile(path); return f~=nil,e")] + [TestCase("return pcall(io.input,path)")] + [TestCase("return xpcall(function() dofile(path) end,function(e) return e end)")] + public async Task FileErrorsContainTheOriginalUtf8Path(string script) + { + using var state = CreateState(); + var path = LuaByteString.FromString(Path.Combine(directory, "missing_\u1EBF", "file.lua")); + state.Environment["path"] = path; + var result = await state.DoStringAsync(script, LuaByteString.FromString("caller_\u1EBF")); + + Assert.That(result[0], Is.EqualTo(new LuaValue(false))); + var message = result[1].Read(); + Assert.That(message.All(c => c <= 255), Is.True); + Assert.That(message, Does.Contain(path)); + } + + [Test] + public async Task LuaErrorsAndSyntaxDiagnosticsAreNotReencoded() + { + using var state = CreateState(); + var text = LuaByteString.FromString("Lua error \u1EBF"); + state.Environment["message"] = text; + state.Environment["chunk"] = LuaByteString.FromString("chunk_\u1EBF"); + var result = await state.DoStringAsync( + """ + local _,p = pcall(error,message,0) + local _,x = xpcall(function() error(message,0) end,function(e) return e end) + local _,s = load("return )",chunk) + local payload = {} + local ok,value = xpcall(function() error(payload) end,function(e) return e end) + assert(not ok and value == payload) + return p,x,s + """ + ); + Assert.That(result[0].Read(), Is.EqualTo(text)); + Assert.That(result[1].Read(), Is.EqualTo(text)); + Assert.That( + result[2].Read(), + Does.Contain(state.Environment["chunk"].Read()) + ); + } + + [TestCase("read('*a')")] + [TestCase("write('x')")] + [TestCase("flush()")] + public async Task FileHandleErrorsAreUtf8Bytes(string operation) + { + using var state = CreateState(); + state.Environment["file"] = LuaValue.FromUserData(new FileHandle(new FailingStream())); + var result = await state.DoStringAsync($"return file:{operation}"); + Assert.That(result[0], Is.EqualTo(LuaValue.Nil)); + Assert.That( + result[1].Read(), + Is.EqualTo(LuaByteString.FromString(FailingStream.Error)) + ); + } + + [TestCase(1)] + [TestCase(8193)] + public async Task BinaryPacketsRoundTripAcrossBufferBoundaries(int chunkSize) + { + var bytes = Enumerable.Range(0, 20000).Select(i => (byte)i).ToArray(); + using var memory = new MemoryStream(); + using var stream = new LuaStream(LuaFileOpenMode.WriteUpdate, memory, true); + await stream.WriteAsync(LuaByteString.FromBytes(bytes).AsMemory(), default); + Assert.That(memory.ToArray(), Is.EqualTo(bytes)); + stream.Seek(SeekOrigin.Begin, 0); + var result = new StringBuilder(); + string? part; + while ((part = await stream.ReadAsync(chunkSize, default)) != null) + result.Append(part); + Assert.That(LuaByteString.ToBytes(result.ToString()), Is.EqualTo(bytes)); + Assert.That(stream.Seek(SeekOrigin.Current, 0), Is.EqualTo(bytes.Length)); + } + + [Test] + public async Task RawLinesPreserveCarriageReturnsIncludingAtBufferEdges() + { + var first = new string('a', 1023) + "\r"; + var bytes = LuaByteString.ToBytes(first + "\nnext\rpart\nlast\r"); + using var stream = new LuaStream(LuaFileOpenMode.Read, new MemoryStream(bytes), true); + Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo(first)); + Assert.That(await stream.ReadLineAsync(true, default), Is.EqualTo("next\rpart\n")); + Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo("last\r")); + Assert.That(await stream.ReadLineAsync(false, default), Is.Null); + } + + [TestCase(false, false)] + [TestCase(true, false)] + [TestCase(true, true)] + public async Task HostModuleLoadersReceiveDotNetNamesAndReturnTextOrBytes( + bool byteStrings, + bool bytes + ) + { + var loader = new MemoryModuleLoader(bytes); + using var state = LuaState.Create( + byteStrings ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); + state.ModuleLoader = loader; + state.OpenStandardLibraries(); + state.Environment["name"] = byteStrings + ? LuaByteString.FromString(MemoryModuleLoader.Name) + : MemoryModuleLoader.Name; + var result = await state.DoStringAsync("local a=require(name); return a.text,a.size"); + Assert.That( + result[0].Read(), + Is.EqualTo(byteStrings ? LuaByteString.FromString("\u1EBF") : "\u1EBF") + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(byteStrings ? 3 : 1))); + } + + sealed class MemoryModuleLoader(bool bytes) : ILuaModuleLoader + { + public const string Name = "memory_\u1EBF"; + + public bool Exists(string moduleName) => moduleName == Name; + + public async ValueTask LoadAsync( + string moduleName, + CancellationToken ct = default + ) + { + Assert.That(moduleName, Is.EqualTo(Name)); + await Task.Yield(); + const string source = "return {text='\u1EBF',size=#'\u1EBF'}"; + return bytes + ? new LuaModule(moduleName, Encoding.UTF8.GetBytes(source)) + : new LuaModule(moduleName, source); + } + } + + sealed class FailingStream : ILuaStream + { + public const string Error = "I/O failure \u1EBF"; + public bool IsOpen => true; + public LuaFileOpenMode Mode => LuaFileOpenMode.ReadUpdate; + + public ValueTask ReadAllAsync(CancellationToken ct) => throw new IOException(Error); + + public ValueTask WriteAsync(ReadOnlyMemory value, CancellationToken ct) => + throw new IOException(Error); + + public ValueTask FlushAsync(CancellationToken ct) => throw new IOException(Error); + + public void Dispose() { } + } +} From ab4be5d3a45c601af578d9901b3f89a7891a8b91 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:26:24 +0900 Subject: [PATCH 05/11] Run official Lua tests in byte-string mode --- tests/Lua.Tests/Helpers/TestStandardIO.cs | 8 +++-- tests/Lua.Tests/LuaTests.cs | 37 +++++++++++++++-------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tests/Lua.Tests/Helpers/TestStandardIO.cs b/tests/Lua.Tests/Helpers/TestStandardIO.cs index 89a135d2..e447283d 100644 --- a/tests/Lua.Tests/Helpers/TestStandardIO.cs +++ b/tests/Lua.Tests/Helpers/TestStandardIO.cs @@ -2,9 +2,9 @@ namespace Lua.Tests.Helpers; -public class TestStandardIO : ILuaStandardIO +public class TestStandardIO(bool byteStrings = false) : ILuaStandardIO { - readonly ConsoleStandardIO consoleStandardIO = new(); + readonly ConsoleStandardIO consoleStandardIO = new(byteStrings); public ILuaStream Input { @@ -18,7 +18,9 @@ public ILuaStream Input new BufferedOutputStream( (memory) => { - Console.WriteLine(memory.ToString()); + Console.WriteLine( + byteStrings ? LuaByteString.ToString(memory.ToString()) : memory.ToString() + ); } ) ); diff --git a/tests/Lua.Tests/LuaTests.cs b/tests/Lua.Tests/LuaTests.cs index 4332568a..62683ea1 100644 --- a/tests/Lua.Tests/LuaTests.cs +++ b/tests/Lua.Tests/LuaTests.cs @@ -1,13 +1,16 @@ using System.Text; using Lua.IO; +using Lua.Platforms; using Lua.Standard; using Lua.Tests.Helpers; namespace Lua.Tests; -public class LuaTests +[TestFixture(false)] +[TestFixture(true)] +public class LuaTests(bool byteStrings) { - static string PatchFilesLuaSource(string source) + static string PatchFilesLuaSource(string source, bool byteStrings) { const char utf8Bom = '\uFEFF'; var lines = source.Replace("\r\n", "\n").Split('\n'); @@ -19,12 +22,15 @@ static string PatchFilesLuaSource(string source) "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; lines[296] = "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - lines[330] = - $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[331] = - $"testloadfile(\"{utf8Bom}return 239\", 239) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; - lines[332] = - $"testloadfile(\"{utf8Bom}\", nil) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + if (!byteStrings) + { + lines[330] = + $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + lines[331] = + $"testloadfile(\"{utf8Bom}return 239\", 239) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + lines[332] = + $"testloadfile(\"{utf8Bom}\", nil) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; + } for (var i = 341; i <= 346; i++) { lines[i] = @@ -77,11 +83,13 @@ public async Task Test_Lua(string file) { var path = FileHelper.GetAbsolutePath(file); var baseDirectory = Path.GetDirectoryName(path)!; - var state = LuaState.Create(); + using var state = LuaState.Create( + byteStrings ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + ); state.Platform = state.Platform with { - StandardIO = new TestStandardIO(), - FileSystem = new FileSystem(baseDirectory), + StandardIO = new TestStandardIO(byteStrings), + FileSystem = new FileSystem(baseDirectory, byteStrings), }; state.OpenStandardLibraries(); if (file == "tests-lua/errors.lua") @@ -91,10 +99,13 @@ public async Task Test_Lua(string file) if (file == "tests-lua/files.lua") { // files.lua contains raw 8-bit source literals. Decode its bytes 1:1 - // and patch only the byte-sensitive assertions in memory so the + // and apply compatibility patches in memory so the // original Lua test file stays untouched on disk. var sourceBytes = await File.ReadAllBytesAsync(path); - var source = PatchFilesLuaSource(Encoding.Latin1.GetString(sourceBytes)); + var source = PatchFilesLuaSource( + Encoding.Latin1.GetString(sourceBytes), + byteStrings + ); var closure = state.Load(source, "@" + Path.GetFileName(file)); await state.ExecuteAsync(closure); } From 44c9bf1fe0188f321b9bf87e622e7712222cdaac Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:49:52 +0900 Subject: [PATCH 06/11] Define string modes and normalize host text and path APIs --- README.md | 15 +- sandbox/Benchmark/ByteStringIoBenchmark.cs | 9 +- .../Runtime/LuaThreadAssetExtensions.cs | 4 +- src/Lua/Exceptions.cs | 2 +- src/Lua/IO/ConsoleStandardIO.cs | 16 +- src/Lua/IO/FileSystem.cs | 36 ++-- src/Lua/IO/ILuaFileLoader.cs | 1 + src/Lua/IO/ILuaFileSystem.cs | 4 + src/Lua/IO/ILuaStream.cs | 8 +- src/Lua/IO/LuaStream.cs | 16 +- src/Lua/Internal/Utf8Reader.cs | 8 +- src/Lua/LuaByteString.cs | 2 +- src/Lua/LuaState.cs | 44 ++++- src/Lua/LuaStateExtensions.cs | 34 ++-- src/Lua/LuaStringMode.cs | 11 ++ src/Lua/Platforms/LuaPlatform.cs | 21 ++- src/Lua/Standard/BasicLibrary.cs | 24 ++- src/Lua/Standard/IOLibrary.cs | 4 +- src/Lua/Standard/Internal/IOHelper.cs | 2 +- src/Lua/Standard/Internal/MatchState.cs | 2 +- src/Lua/Standard/ModuleLibrary.cs | 16 +- src/Lua/Standard/OperatingSystemLibrary.cs | 26 ++- src/Lua/Standard/StringLibrary.cs | 12 +- tests/Lua.Tests/ByteStringIntegrationTests.cs | 164 ++++++++++++++++-- tests/Lua.Tests/ByteStringTests.cs | 18 +- tests/Lua.Tests/Helpers/TestStandardIO.cs | 8 +- tests/Lua.Tests/LuaTests.cs | 24 +-- 27 files changed, 365 insertions(+), 166 deletions(-) create mode 100644 src/Lua/LuaStringMode.cs diff --git a/README.md b/README.md index 6b242984..8cc396c9 100644 --- a/README.md +++ b/README.md @@ -686,7 +686,7 @@ All string library functions handle strings as UTF-16. #### Byte strings (opt-in) -If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), enable `LuaPlatform.ByteStrings`. `LuaPlatform.DefaultByteStrings` is the default platform with byte strings and byte-oriented file and standard I/O: +If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), use `LuaStringMode.Bytes`. `LuaPlatform.StringMode` defaults to `LuaStringMode.Utf16`. `LuaPlatform.DefaultByteStrings` provides byte strings and byte-oriented file and standard I/O: ```cs using Lua; @@ -704,19 +704,18 @@ var upper = LuaByteString.ToString(results[1].Read()); // back to a .N In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16. -Choose one representation for a state and configure it before executing scripts. Setting `ByteStrings = true` on an otherwise unchanged platform does not switch its streams to raw bytes. When configuring a file root, keep that root as a normal .NET path: +Choose one representation for a state and configure it before executing scripts. Setting `StringMode = LuaStringMode.Bytes` on an otherwise unchanged platform does not switch its streams to raw bytes. Use matching modes for the platform and its streams. When configuring a file root, keep that root as a normal .NET path: ```cs var platform = LuaPlatform.DefaultByteStrings with { - FileSystem = new Lua.IO.FileSystem(baseDirectory: scriptsDirectory, byteStrings: true) + FileSystem = new Lua.IO.FileSystem(scriptsDirectory, LuaStringMode.Bytes) }; using var byteState = LuaState.Create(platform); byteState.OpenStandardLibraries(); -// A LuaModule carries normal .NET text. LoadModule converts its source and name. -var module = new LuaModule("greeting.lua", "return 'こんにちは'"); -var values = await byteState.ExecuteAsync(byteState.LoadModule(module)); +// C# source and filename APIs accept normal .NET text. +var values = await byteState.DoStringAsync("return 'こんにちは'"); var greeting = LuaByteString.ToString(values[0].Read()); // Raw packets can contain NUL, invalid UTF-8 and every value from 0 to 255. @@ -725,9 +724,9 @@ var packet = await byteState.DoStringAsync("return packet:reverse()"); byte[] reversed = LuaByteString.ToBytes(packet[0].Read()); ``` -Use `FromString`/`ToString` for text and `FromBytes`/`ToBytes` for binary data. This also applies to non-ASCII table keys and the arguments/results of C# functions and generated bindings; interop does not convert strings automatically. `DoStringAsync` and `Load` with character source accept Lua strings, so encode non-ASCII source with `FromString`, or use `LoadModule` for .NET text. Byte source passed to `Load` and source read from files are preserved directly. +Use `FromString`/`ToString` for string values and `FromBytes`/`ToBytes` for binary data. This also applies to non-ASCII table keys and the arguments/results of C# functions and generated bindings; value interop does not convert strings automatically. C# source APIs (`DoStringAsync`, `Load` with character source or a `LuaModule`) accept normal .NET text and encode it as UTF-8 in byte mode. Chunk names and paths passed to `LoadFileAsync`/`DoFileAsync` are also .NET text. Byte source passed to `Load`/`ExecuteAsync` and source read from files are preserved directly; their explicit chunk names are .NET text. Lua's `load`, `loadfile`, `dofile` and `require` continue to use Lua byte strings without re-encoding their source. -Names passed to `ILuaModuleLoader` are normal .NET strings. Text modules are treated as .NET text and encoded to UTF-8 in byte mode; byte modules remain bytes. Unity text assets use the same conversion. Custom file systems and standard-I/O providers must implement the selected stream representation themselves; use `ILuaStream.CreateFromStream(stream, mode, byteStrings: true)` for raw streams. Filenames passed directly to the byte-mode `FileSystem`, `DoFileAsync` or `LoadFileAsync` are Lua byte strings, while `FileSystem.BaseDirectory` is a .NET path. +Paths and names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader` are normal .NET strings in both modes. Text modules are treated as .NET text and encoded to UTF-8 in byte mode; byte modules remain bytes. Unity text assets use the same conversion. Custom file systems and standard-I/O providers must implement the selected stream representation themselves; use `ILuaStream.CreateFromStream(stream, mode, LuaStringMode.Bytes)` for raw streams. Files, temporary files and redirected standard I/O preserve bytes, including a BOM read as data. Raw line reads stop at LF and retain CR; `*L` also retains the LF. Invalid UTF-8 is replaced when decoding with `ToString`, so keep binary data in byte strings or byte arrays. Passing unconverted .NET characters above U+00FF to raw output writes `?` for those characters. diff --git a/sandbox/Benchmark/ByteStringIoBenchmark.cs b/sandbox/Benchmark/ByteStringIoBenchmark.cs index adbcd3ee..d961231e 100644 --- a/sandbox/Benchmark/ByteStringIoBenchmark.cs +++ b/sandbox/Benchmark/ByteStringIoBenchmark.cs @@ -1,11 +1,12 @@ using BenchmarkDotNet.Attributes; +using Lua; using Lua.IO; [Config(typeof(BenchmarkConfig))] public class ByteStringIoBenchmark { - [Params(false, true)] - public bool ByteStrings { get; set; } + [Params(LuaStringMode.Utf16, LuaStringMode.Bytes)] + public LuaStringMode StringMode { get; set; } [Params(1, 4096)] public int Size { get; set; } @@ -22,8 +23,8 @@ public void Setup() text = new string('a', Size); input = new MemoryStream(Enumerable.Repeat((byte)'a', Size).ToArray()); output = new MemoryStream(Size); - reader = new LuaStream(LuaFileOpenMode.Read, input, ByteStrings); - writer = new LuaStream(LuaFileOpenMode.Write, output, ByteStrings); + reader = new LuaStream(LuaFileOpenMode.Read, input, StringMode); + writer = new LuaStream(LuaFileOpenMode.Write, output, StringMode); } [GlobalCleanup] diff --git a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs index 6e007250..81856e66 100644 --- a/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs +++ b/src/Lua.Unity/Assets/Lua.Unity/Runtime/LuaThreadAssetExtensions.cs @@ -21,7 +21,7 @@ public static ValueTask ExecuteAsync( } var module = luaAssetBase.GetModule(name); - var closure = state.LoadModule(module); + var closure = state.Load(module); return state.ExecuteAsync(closure, buffer, cancellationToken); } @@ -38,7 +38,7 @@ public static ValueTask ExecuteAsync( } var module = luaAssetBase.GetModule(name); - return state.ExecuteAsync(state.LoadModule(module), cancellationToken); + return state.ExecuteAsync(state.Load(module), cancellationToken); } } } diff --git a/src/Lua/Exceptions.cs b/src/Lua/Exceptions.cs index cf53a76f..82555a12 100644 --- a/src/Lua/Exceptions.cs +++ b/src/Lua/Exceptions.cs @@ -494,7 +494,7 @@ internal void Forget() internal string MinimalMessage(LuaPlatform platform) { var message = InnerException is { } inner - ? platform.ByteStrings + ? platform.StringMode == LuaStringMode.Bytes ? platform.GetErrorMessage(inner) : inner.ToString() : ErrorObject.ToString(); diff --git a/src/Lua/IO/ConsoleStandardIO.cs b/src/Lua/IO/ConsoleStandardIO.cs index 43e3da3b..0834dc6b 100644 --- a/src/Lua/IO/ConsoleStandardIO.cs +++ b/src/Lua/IO/ConsoleStandardIO.cs @@ -5,14 +5,14 @@ namespace Lua.IO; /// /// Default implementation of ILuaStandardIO using Console /// -/// -/// When true, standard input and output are read and written as raw bytes (one byte per char) -/// instead of UTF-8 text. Use together with . +/// +/// Selects UTF-8 text I/O for , or raw byte I/O for . +/// Use the same mode as . /// -public sealed class ConsoleStandardIO(bool byteStrings) : ILuaStandardIO +public sealed class ConsoleStandardIO(LuaStringMode stringMode) : ILuaStandardIO { public ConsoleStandardIO() - : this(false) { } + : this(LuaStringMode.Utf16) { } ILuaStream? standardInput; ILuaStream? standardOutput; @@ -23,7 +23,7 @@ public ConsoleStandardIO() ILuaStream.CreateFromStream( ConsoleHelper.OpenStandardInput(), LuaFileOpenMode.Read, - byteStrings + stringMode ) ); @@ -32,7 +32,7 @@ public ConsoleStandardIO() ILuaStream.CreateFromStream( ConsoleHelper.OpenStandardOutput(), LuaFileOpenMode.Write, - byteStrings + stringMode ) ); @@ -41,7 +41,7 @@ public ConsoleStandardIO() ILuaStream.CreateFromStream( ConsoleHelper.OpenStandardError(), LuaFileOpenMode.Write, - byteStrings + stringMode ) ); } diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index dbc638f1..8052e6b2 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -1,15 +1,14 @@ namespace Lua.IO; /// Directory that relative paths are resolved against. -/// -/// When true, files are read and written as raw bytes (one byte per char) instead of UTF-8 text. -/// Use together with . +/// +/// Selects UTF-8 text I/O for , or raw byte I/O for . +/// Use the same mode as . /// -public sealed class FileSystem(string? baseDirectory = null, bool byteStrings = false) - : ILuaFileSystem +public sealed class FileSystem(string? baseDirectory, LuaStringMode stringMode) : ILuaFileSystem { - public FileSystem(string? baseDirectory) - : this(baseDirectory, false) { } + public FileSystem(string? baseDirectory = null) + : this(baseDirectory, LuaStringMode.Utf16) { } public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory(); @@ -41,15 +40,9 @@ public string GetFullPath(string path) return Path.Combine(baseDirectory, path); } - // Paths come from Lua as byte strings in byte mode: decode the UTF-8 bytes into the real file name. - string ResolvePath(string path) - { - return GetFullPath(byteStrings ? LuaByteString.ToString(path) : path); - } - public bool IsReadable(string path) { - return File.Exists(ResolvePath(path)); + return File.Exists(GetFullPath(path)); } public ValueTask Open( @@ -60,7 +53,7 @@ CancellationToken cancellationToken { var (mode, access) = GetFileMode(openMode); Stream stream; - path = ResolvePath(path); + path = GetFullPath(path); if (openMode == LuaFileOpenMode.AppendUpdate) { stream = File.Open( @@ -75,7 +68,7 @@ CancellationToken cancellationToken stream = File.Open(path, mode, access, FileShare.ReadWrite | FileShare.Delete); } - ILuaStream wrapper = new LuaStream(openMode, stream, byteStrings); + ILuaStream wrapper = new LuaStream(openMode, stream, stringMode); if (openMode == LuaFileOpenMode.AppendUpdate) { @@ -87,8 +80,8 @@ CancellationToken cancellationToken public ValueTask Rename(string oldName, string newName, CancellationToken cancellationToken) { - oldName = ResolvePath(oldName); - newName = ResolvePath(newName); + oldName = GetFullPath(oldName); + newName = GetFullPath(newName); // Check the source first: a failed rename must leave the destination untouched. var isFile = File.Exists(oldName); @@ -127,7 +120,7 @@ public ValueTask Rename(string oldName, string newName, CancellationToken cancel public ValueTask Remove(string path, CancellationToken cancellationToken) { - path = ResolvePath(path); + path = GetFullPath(path); if (!File.Exists(path)) { throw new FileNotFoundException("No such file or directory", path); @@ -143,8 +136,7 @@ public ValueTask Remove(string path, CancellationToken cancellationToken) public string GetTempFileName() { - var name = Path.GetTempFileName(); - return byteStrings ? LuaByteString.FromString(name) : name; + return Path.GetTempFileName(); } public ValueTask OpenTempFileStream(CancellationToken cancellationToken) @@ -153,7 +145,7 @@ public ValueTask OpenTempFileStream(CancellationToken cancellationTo new LuaStream( LuaFileOpenMode.WriteUpdate, File.Open(Path.GetTempFileName(), FileMode.Open, FileAccess.ReadWrite), - byteStrings + stringMode ) ); } diff --git a/src/Lua/IO/ILuaFileLoader.cs b/src/Lua/IO/ILuaFileLoader.cs index f5abb58d..14c7ba6e 100644 --- a/src/Lua/IO/ILuaFileLoader.cs +++ b/src/Lua/IO/ILuaFileLoader.cs @@ -1,5 +1,6 @@ namespace Lua.IO; +/// Loads files by .NET path; stream contents use the state's Lua string representation. public interface ILuaFileLoader { bool Exists(string path); diff --git a/src/Lua/IO/ILuaFileSystem.cs b/src/Lua/IO/ILuaFileSystem.cs index 16f5e9ff..dfd7eec9 100644 --- a/src/Lua/IO/ILuaFileSystem.cs +++ b/src/Lua/IO/ILuaFileSystem.cs @@ -1,5 +1,9 @@ namespace Lua.IO; +/// +/// Host file operations. Paths and temporary filenames are .NET text in every string mode. +/// Stream contents use the Lua representation configured for the state. +/// public interface ILuaFileSystem { bool IsReadable(string path); diff --git a/src/Lua/IO/ILuaStream.cs b/src/Lua/IO/ILuaStream.cs index 63341090..7a02b989 100644 --- a/src/Lua/IO/ILuaStream.cs +++ b/src/Lua/IO/ILuaStream.cs @@ -74,9 +74,13 @@ static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode) return new LuaStream(openMode, stream); } - static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode, bool byteStrings) + static ILuaStream CreateFromStream( + Stream stream, + LuaFileOpenMode openMode, + LuaStringMode stringMode + ) { - return new LuaStream(openMode, stream, byteStrings); + return new LuaStream(openMode, stream, stringMode); } static ILuaStream CreateFromString(string content) diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index 1907bd71..7bdbfabc 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -4,12 +4,12 @@ namespace Lua.IO; -public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings) +public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, LuaStringMode stringMode) : ILuaStream, ILuaByteStream { public LuaStream(LuaFileOpenMode mode, Stream innerStream) - : this(mode, innerStream, false) { } + : this(mode, innerStream, LuaStringMode.Utf16) { } Utf8Reader? reader; ulong flushSize = ulong.MaxValue; @@ -24,14 +24,14 @@ public LuaStream(LuaFileOpenMode mode, Stream innerStream) public ValueTask ReadLineAsync(bool keepEol, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(byteStrings); + reader ??= new(stringMode); return new(reader.ReadLine(innerStream, keepEol)); } public ValueTask ReadAllAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(byteStrings); + reader ??= new(stringMode); var text = reader.ReadToEnd(innerStream); return new(text); } @@ -62,7 +62,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadAsync(int count, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(byteStrings); + reader ??= new(stringMode); if (count == 0) { return new(reader.IsEndOfStream(innerStream) ? null : string.Empty); @@ -74,7 +74,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadNumberAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(byteStrings); + reader ??= new(stringMode); // Use the Utf8Reader's ReadNumber method which handles positioning correctly var numberStr = reader.ReadNumber(innerStream); @@ -99,14 +99,14 @@ public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cance using PooledArray byteBuffer = new(4096); var bytes = byteBuffer.AsSpan(); - var encoder = byteStrings ? null : Encoding.UTF8.GetEncoder(); + var encoder = stringMode == LuaStringMode.Bytes ? null : Encoding.UTF8.GetEncoder(); var remainingChars = buffer.Span; var totalBytes = 0; while (!remainingChars.IsEmpty) { int charsUsed, bytesUsed; - if (byteStrings) + if (stringMode == LuaStringMode.Bytes) { charsUsed = Math.Min(remainingChars.Length, bytes.Length); bytesUsed = LuaByteEncoding.Instance.GetBytes(remainingChars[..charsUsed], bytes); diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index 83045718..ca3a78da 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -22,13 +22,13 @@ sealed class Utf8Reader const int ThreadStaticBufferSize = 1024; - /// - /// When true, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, + /// + /// In byte mode, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, /// like the reference C implementation. Otherwise the stream is decoded as UTF-8. /// - public Utf8Reader(bool byteStrings = false) + public Utf8Reader(LuaStringMode stringMode = LuaStringMode.Utf16) { - this.byteStrings = byteStrings; + byteStrings = stringMode == LuaStringMode.Bytes; encoding = byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8; if (scratchBufferUsed) diff --git a/src/Lua/LuaByteString.cs b/src/Lua/LuaByteString.cs index 5f238aae..a522afc7 100644 --- a/src/Lua/LuaByteString.cs +++ b/src/Lua/LuaByteString.cs @@ -5,7 +5,7 @@ namespace Lua; /// /// Conversions between .NET strings and Lua byte strings (see -/// ), where every holds one byte. +/// ), where every holds one byte. /// public static class LuaByteString { diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index d43c6dc9..38bbb651 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -594,7 +594,25 @@ public async ValueTask RunAsync( } } - public unsafe LuaClosure Load( + /// + /// Loads .NET source text and a .NET chunk name. In byte-string mode, text is + /// encoded as UTF-8. Use the byte overload for source that is already encoded. + /// + public LuaClosure Load(ReadOnlySpan chunk, string chunkName, LuaTable? environment = null) + { + var name = Platform.ToLuaString(chunkName); + if (Platform.StringMode != LuaStringMode.Bytes) + { + return LoadCore(chunk, name, environment); + } + + using PooledArray bytes = new(Encoding.UTF8.GetByteCount(chunk)); + var count = Encoding.UTF8.GetBytes(chunk, bytes.AsSpan()); + return LoadCore(bytes.AsSpan()[..count], name, "t", environment); + } + + // Source and chunk name already use this state's Lua string representation. + internal unsafe LuaClosure LoadCore( ReadOnlySpan chunk, string chunkName, LuaTable? environment = null @@ -609,12 +627,32 @@ public unsafe LuaClosure Load( return new(this, prototype, environment); } + /// + /// Loads encoded source bytes with a .NET chunk name. Byte-string mode preserves + /// the source bytes, except for an initial UTF-8 BOM. + /// public LuaClosure Load( ReadOnlySpan chunk, string? chunkName = null, string mode = "bt", LuaTable? environment = null ) + { + return LoadCore( + chunk, + chunkName is null ? null : Platform.ToLuaString(chunkName), + mode, + environment + ); + } + + // Source is encoded bytes; an explicit chunk name already uses the Lua string representation. + internal LuaClosure LoadCore( + ReadOnlySpan chunk, + string? chunkName = null, + string mode = "bt", + LuaTable? environment = null + ) { static bool AllowsMode(string mode, char chunkMode) { @@ -640,7 +678,7 @@ static bool AllowsMode(string mode, char chunkMode) } Encoding encoding; - if (Platform.ByteStrings) + if (Platform.StringMode == LuaStringMode.Bytes) { // Byte strings: like luaL_loadfile, skip a UTF-8 BOM and keep every other byte as one char. chunk = BomUtility.SkipUtf8Bom(chunk); @@ -659,7 +697,7 @@ static bool AllowsMode(string mode, char chunkMode) encoding.GetChars(chunk, chars); chunkName ??= chars.ToString(); - return Load(chars, chunkName, environment); + return LoadCore(chars, chunkName, environment); } finally { diff --git a/src/Lua/LuaStateExtensions.cs b/src/Lua/LuaStateExtensions.cs index 186b5a08..93e827b5 100644 --- a/src/Lua/LuaStateExtensions.cs +++ b/src/Lua/LuaStateExtensions.cs @@ -1,6 +1,5 @@ using System.Buffers; using System.Runtime.CompilerServices; -using System.Text; using Lua.Internal; using Lua.Internal.CompilerServices; using Lua.IO; @@ -16,25 +15,14 @@ public static class LuaStateExtensions /// Loads a host module. Its name and text source are .NET text; byte source is /// preserved. In byte-string mode, text source is encoded as UTF-8 before parsing. /// - public static LuaClosure LoadModule(this LuaState state, LuaModule module) + public static LuaClosure Load(this LuaState state, LuaModule module) { - var name = state.Platform.ToLuaString(module.Name); - if (module.Type == LuaModuleType.Bytes) - { - return state.Load(module.ReadBytes(), name); - } - - if (!state.Platform.ByteStrings) - { - return state.Load(module.ReadText(), name); - } - - var source = module.ReadText(); - using PooledArray bytes = new(Encoding.UTF8.GetByteCount(source)); - var count = Encoding.UTF8.GetBytes(source, bytes.AsSpan()); - return state.Load(bytes.AsSpan()[..count], name); + return module.Type == LuaModuleType.Bytes + ? state.Load(module.ReadBytes(), module.Name) + : state.Load(module.ReadText(), module.Name); } + /// Loads a file whose name is .NET text, including in byte-string mode. public static async ValueTask LoadFileAsync( this LuaState state, string fileName, @@ -43,7 +31,7 @@ public static async ValueTask LoadFileAsync( CancellationToken cancellationToken ) { - var name = "@" + fileName; + var name = state.Platform.ToLuaString("@" + fileName); using var stream = await state.GlobalState.Platform.FileSystem.Open( fileName, LuaFileOpenMode.Read, @@ -61,7 +49,7 @@ CancellationToken cancellationToken if (firstByte < 0) { - closure = state.Load(ReadOnlySpan.Empty, name, mode, environment); + closure = state.LoadCore(ReadOnlySpan.Empty, name, mode, environment); } else { @@ -69,7 +57,7 @@ CancellationToken cancellationToken source.GetSpan(1)[0] = (byte)firstByte; source.Advance(1); await byteStream.ReadBytesAsync(source, cancellationToken); - closure = state.Load(source.WrittenSpan, name, mode, environment); + closure = state.LoadCore(source.WrittenSpan, name, mode, environment); } } else if (!mode.Contains('t')) @@ -79,12 +67,13 @@ CancellationToken cancellationToken else { var source = await stream.ReadAllAsync(cancellationToken); - closure = state.Load(source, name, environment); + closure = state.LoadCore(source, name, environment); } return closure; } + /// Executes .NET source text with an optional .NET chunk name. public static ValueTask DoStringAsync( this LuaState state, string source, @@ -97,6 +86,7 @@ public static ValueTask DoStringAsync( return ExecuteAsync(state, closure, results, cancellationToken); } + /// Executes .NET source text with an optional .NET chunk name. public static ValueTask DoStringAsync( this LuaState state, string source, @@ -131,6 +121,7 @@ public static ValueTask ExecuteAsync( return ExecuteAsync(state, closure, cancellationToken); } + /// Executes a file whose path is .NET text, including in byte-string mode. [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] public static async ValueTask DoFileAsync( this LuaState state, @@ -146,6 +137,7 @@ public static async ValueTask DoFileAsync( return results.Count; } + /// Executes a file whose path is .NET text, including in byte-string mode. [AsyncMethodBuilder(typeof(LightAsyncValueTaskMethodBuilder<>))] public static async ValueTask DoFileAsync( this LuaState state, diff --git a/src/Lua/LuaStringMode.cs b/src/Lua/LuaStringMode.cs new file mode 100644 index 00000000..35a8da0a --- /dev/null +++ b/src/Lua/LuaStringMode.cs @@ -0,0 +1,11 @@ +namespace Lua; + +/// The string representation used by a Lua state and its streams. +public enum LuaStringMode +{ + /// Strings contain UTF-16 code units. This is the default. + Utf16, + + /// Each char holds one byte, including arbitrary binary data. + Bytes, +} diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index b0aaa2a0..491b2f01 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -17,21 +17,26 @@ TimeProvider TimeProvider ) { /// - /// When true, Lua strings are byte strings: every holds one byte (0x00-0xFF), + /// In mode, every holds one byte (0x00-0xFF), /// so #s, string.len, string.sub and string.byte count bytes exactly /// like the reference C implementation (UTF-8 text such as "ế" has length 3 instead of 1). /// Character classes (%a, %w, ...), string.upper and string.lower follow the /// C locale (ASCII only) and string.char rejects values above 255. /// The and of the platform must read and write - /// bytes as well, which does. The default is false (UTF-16 strings). + /// bytes as well, which does. The default is . /// Use to convert between .NET strings and byte strings. /// - public bool ByteStrings { get; init; } + public LuaStringMode StringMode { get; init; } // Converts a .NET string (file name, environment value, error text) into a Lua string for this platform. internal string ToLuaString(string value) { - return ByteStrings ? LuaByteString.FromString(value) : value; + return StringMode == LuaStringMode.Bytes ? LuaByteString.FromString(value) : value; + } + + internal string ToHostString(string value) + { + return StringMode == LuaStringMode.Bytes ? LuaByteString.ToString(value) : value; } internal string GetErrorMessage(Exception exception) @@ -56,17 +61,17 @@ internal string GetErrorMessage(Exception exception) ); /// - /// Same as , but with enabled: files and standard I/O + /// Same as , but with set to : files and standard I/O /// are read and written as raw bytes. /// public static LuaPlatform DefaultByteStrings { get; } = new( - FileSystem: new FileSystem(byteStrings: true), + FileSystem: new FileSystem(baseDirectory: null, stringMode: LuaStringMode.Bytes), OsEnvironment: new SystemOsEnvironment(), - StandardIO: new ConsoleStandardIO(byteStrings: true), + StandardIO: new ConsoleStandardIO(LuaStringMode.Bytes), TimeProvider: TimeProvider.System ) { - ByteStrings = true, + StringMode = LuaStringMode.Bytes, }; } diff --git a/src/Lua/Standard/BasicLibrary.cs b/src/Lua/Standard/BasicLibrary.cs index 5323305e..4c2a8514 100644 --- a/src/Lua/Standard/BasicLibrary.cs +++ b/src/Lua/Standard/BasicLibrary.cs @@ -131,7 +131,12 @@ CancellationToken cancellationToken { var arg0 = context.GetArgument(0); context.State.Stack.PopUntil(context.ReturnFrameBase); - var closure = await context.State.LoadFileAsync(arg0, "bt", null, cancellationToken); + var closure = await context.State.LoadFileAsync( + context.State.Platform.ToHostString(arg0), + "bt", + null, + cancellationToken + ); return await context.State.RunAsync(closure, cancellationToken); } @@ -222,7 +227,12 @@ CancellationToken cancellationToken // do not use LuaState.DoFileAsync as it uses the newExecutionContext try { - var closure = await context.State.LoadFileAsync(arg0, mode, null, cancellationToken); + var closure = await context.State.LoadFileAsync( + context.State.Platform.ToHostString(arg0), + mode, + null, + cancellationToken + ); if (hasEnvironment) { closure.SetEnvironment(environment); @@ -260,7 +270,7 @@ CancellationToken cancellationToken throw new Exception("attempt to load a text chunk (mode is 'b')"); } - var closure = context.State.Load(str, name ?? str, null); + var closure = context.State.LoadCore(str, name ?? str, null); if (hasEnvironment) { closure.SetEnvironment(environment); @@ -768,9 +778,11 @@ CancellationToken cancellationToken if (ex is LuaRuntimeException luaEx) { - var error = luaEx.InnerException is { } inner - ? state.Platform.GetErrorMessage(inner) - : luaEx.ErrorObject; + var error = + state.Platform.StringMode == LuaStringMode.Bytes + && luaEx.InnerException is { } inner + ? state.Platform.GetErrorMessage(inner) + : luaEx.ErrorObject; luaEx.Forget(); state.Push(error); } diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index 04ec0ecd..a67d8832 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -95,7 +95,7 @@ CancellationToken cancellationToken else { var stream = await context.GlobalState.Platform.FileSystem.Open( - arg.ToString(), + context.State.Platform.ToHostString(arg.ToString()), LuaFileOpenMode.Read, cancellationToken ); @@ -263,7 +263,7 @@ CancellationToken cancellationToken } var stream = await context.GlobalState.Platform.FileSystem.Open( - arg.ToString(), + context.State.Platform.ToHostString(arg.ToString()), LuaFileOpenMode.Write, cancellationToken ); diff --git a/src/Lua/Standard/Internal/IOHelper.cs b/src/Lua/Standard/Internal/IOHelper.cs index 82d00227..39a44b01 100644 --- a/src/Lua/Standard/Internal/IOHelper.cs +++ b/src/Lua/Standard/Internal/IOHelper.cs @@ -23,7 +23,7 @@ CancellationToken cancellationToken try { var stream = await state.GlobalState.Platform.FileSystem.Open( - fileName, + state.Platform.ToHostString(fileName), fileMode, cancellationToken ); diff --git a/src/Lua/Standard/Internal/MatchState.cs b/src/Lua/Standard/Internal/MatchState.cs index aa34a7ca..3aed4afb 100644 --- a/src/Lua/Standard/Internal/MatchState.cs +++ b/src/Lua/Standard/Internal/MatchState.cs @@ -19,7 +19,7 @@ internal struct Capture } // Byte strings use the C locale: classes only match ASCII, bytes 0x80-0xFF never belong to a class. - readonly bool byteStrings = state.Platform.ByteStrings; + readonly bool byteStrings = state.Platform.StringMode == LuaStringMode.Bytes; public readonly LuaState State = state; public readonly string Source = source; diff --git a/src/Lua/Standard/ModuleLibrary.cs b/src/Lua/Standard/ModuleLibrary.cs index c1225fbf..788b1dbe 100644 --- a/src/Lua/Standard/ModuleLibrary.cs +++ b/src/Lua/Standard/ModuleLibrary.cs @@ -31,14 +31,11 @@ CancellationToken cancellationToken { LuaFunction loader; var moduleLoader = context.GlobalState.ModuleLoader; - var hostName = - moduleLoader != null && context.State.Platform.ByteStrings - ? LuaByteString.ToString(arg0) - : arg0; + var hostName = moduleLoader is null ? arg0 : context.State.Platform.ToHostString(arg0); if (moduleLoader != null && moduleLoader.Exists(hostName)) { var module = await moduleLoader.LoadAsync(hostName, cancellationToken); - loader = context.State.LoadModule(module); + loader = context.State.Load(module); } else { @@ -109,7 +106,7 @@ string dirSeparator { path = pathSpan[..nextIndex].ToString(); var fileName = path.Replace("?", name); - if (state.GlobalState.Platform.FileSystem.IsReadable(fileName)) + if (state.Platform.FileSystem.IsReadable(state.Platform.ToHostString(fileName))) { return fileName; } @@ -201,7 +198,12 @@ CancellationToken cancellationToken } return context.Return( - await context.State.LoadFileAsync(fileName, "bt", null, cancellationToken) + await context.State.LoadFileAsync( + context.State.Platform.ToHostString(fileName), + "bt", + null, + cancellationToken + ) ); } } diff --git a/src/Lua/Standard/OperatingSystemLibrary.cs b/src/Lua/Standard/OperatingSystemLibrary.cs index 3d1870ab..de4deafa 100644 --- a/src/Lua/Standard/OperatingSystemLibrary.cs +++ b/src/Lua/Standard/OperatingSystemLibrary.cs @@ -156,17 +156,9 @@ CancellationToken cancellationToken ) { var variable = context.GetArgument(0); - var byteStrings = context.GlobalState.Platform.ByteStrings; - var value = context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable( - byteStrings ? LuaByteString.ToString(variable) : variable - ); - return new( - context.Return( - value is not null && byteStrings ? LuaByteString.FromString(value) - : value is not null ? value - : LuaValue.Nil - ) - ); + var platform = context.State.Platform; + var value = platform.OsEnvironment.GetEnvironmentVariable(platform.ToHostString(variable)); + return new(context.Return(value is null ? LuaValue.Nil : platform.ToLuaString(value))); } public async ValueTask Remove( @@ -177,7 +169,10 @@ CancellationToken cancellationToken var fileName = context.GetArgument(0); try { - await context.GlobalState.Platform.FileSystem.Remove(fileName, cancellationToken); + await context.GlobalState.Platform.FileSystem.Remove( + context.State.Platform.ToHostString(fileName), + cancellationToken + ); return context.Return(true); } catch (IOException ex) @@ -200,8 +195,8 @@ CancellationToken cancellationToken try { await context.GlobalState.Platform.FileSystem.Rename( - oldName, - newName, + context.State.Platform.ToHostString(oldName), + context.State.Platform.ToHostString(newName), cancellationToken ); return context.Return(true); @@ -259,6 +254,7 @@ public ValueTask TmpName( CancellationToken cancellationToken ) { - return new(context.Return(context.GlobalState.Platform.FileSystem.GetTempFileName())); + var platform = context.State.Platform; + return new(context.Return(platform.ToLuaString(platform.FileSystem.GetTempFileName()))); } } diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index afa2cebb..f034972b 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -69,7 +69,7 @@ CancellationToken cancellationToken return new(context.Return("")); } - var byteStrings = context.State.Platform.ByteStrings; + var byteStrings = context.State.Platform.StringMode == LuaStringMode.Bytes; ValueStringBuilder builder = new(context.ArgumentCount); for (var i = 0; i < context.ArgumentCount; i++) { @@ -419,7 +419,7 @@ context with break; case 'c': formattedValue = ( - context.State.Platform.ByteStrings + context.State.Platform.StringMode == LuaStringMode.Bytes ? (char)((int)x & 0xFF) : (char)(int)x ).ToString(); @@ -872,7 +872,9 @@ CancellationToken cancellationToken var s = context.GetArgument(0); return new( context.Return( - context.State.Platform.ByteStrings ? StringHelper.ToLowerAscii(s) : s.ToLower() + context.State.Platform.StringMode == LuaStringMode.Bytes + ? StringHelper.ToLowerAscii(s) + : s.ToLower() ) ); } @@ -1070,7 +1072,9 @@ CancellationToken cancellationToken var s = context.GetArgument(0); return new( context.Return( - context.State.Platform.ByteStrings ? StringHelper.ToUpperAscii(s) : s.ToUpper() + context.State.Platform.StringMode == LuaStringMode.Bytes + ? StringHelper.ToUpperAscii(s) + : s.ToUpper() ) ); } diff --git a/tests/Lua.Tests/ByteStringIntegrationTests.cs b/tests/Lua.Tests/ByteStringIntegrationTests.cs index f9959d1c..c0281ede 100644 --- a/tests/Lua.Tests/ByteStringIntegrationTests.cs +++ b/tests/Lua.Tests/ByteStringIntegrationTests.cs @@ -16,12 +16,16 @@ public sealed class ByteStringIntegrationTests : IDisposable public void Dispose() => Directory.Delete(directory, true); - LuaState CreateState() + LuaState CreateState(LuaStringMode stringMode = LuaStringMode.Bytes) { var state = LuaState.Create( - LuaPlatform.DefaultByteStrings with + ( + stringMode == LuaStringMode.Bytes + ? LuaPlatform.DefaultByteStrings + : LuaPlatform.Default + ) with { - FileSystem = new FileSystem(directory, true), + FileSystem = new FileSystem(directory, stringMode), } ); state.OpenStandardLibraries(); @@ -32,6 +36,9 @@ LuaPlatform.DefaultByteStrings with public void ExistingConstructorSignaturesRemainAvailable() { Assert.That(typeof(FileSystem).GetConstructor([typeof(string)]), Is.Not.Null); + var baseDirectory = typeof(FileSystem).GetConstructor([typeof(string)])!.GetParameters()[0]; + Assert.That(baseDirectory.IsOptional, Is.True); + Assert.That(baseDirectory.DefaultValue, Is.Null); Assert.That( typeof(LuaStream).GetConstructor([typeof(LuaFileOpenMode), typeof(Stream)]), Is.Not.Null @@ -39,6 +46,30 @@ public void ExistingConstructorSignaturesRemainAvailable() Assert.That(typeof(ConsoleStandardIO).GetConstructor(Type.EmptyTypes), Is.Not.Null); } + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task XPCallHostErrorsPreserveDefaultBehaviorAndEncodeByteMode( + LuaStringMode stringMode + ) + { + using var state = CreateState(stringMode); + const string message = "host error ế"; + state.Environment["fail"] = new LuaFunction( + "fail", + (_, _) => throw new IOException(message) + ); + var result = await state.DoStringAsync( + "return xpcall(function() fail() end,function(e) return e end)" + ); + Assert.That(result[0], Is.EqualTo(new LuaValue(false))); + Assert.That( + result[1], + Is.EqualTo( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(message) : LuaValue.Nil + ) + ); + } + [Test] public async Task TemporaryFilesPreserveRawBytesAndPositions() { @@ -67,7 +98,7 @@ public async Task FileErrorsContainTheOriginalUtf8Path(string script) using var state = CreateState(); var path = LuaByteString.FromString(Path.Combine(directory, "missing_\u1EBF", "file.lua")); state.Environment["path"] = path; - var result = await state.DoStringAsync(script, LuaByteString.FromString("caller_\u1EBF")); + var result = await state.DoStringAsync(script, "caller_\u1EBF"); Assert.That(result[0], Is.EqualTo(new LuaValue(false))); var message = result[1].Read(); @@ -90,6 +121,9 @@ public async Task LuaErrorsAndSyntaxDiagnosticsAreNotReencoded() local payload = {} local ok,value = xpcall(function() error(payload) end,function(e) return e end) assert(not ok and value == payload) + local raw = string.char(255,128) + local f = assert(load("return '"..raw.."'",raw)) + assert(f() == raw and debug.getinfo(f).source == raw) return p,x,s """ ); @@ -122,7 +156,7 @@ public async Task BinaryPacketsRoundTripAcrossBufferBoundaries(int chunkSize) { var bytes = Enumerable.Range(0, 20000).Select(i => (byte)i).ToArray(); using var memory = new MemoryStream(); - using var stream = new LuaStream(LuaFileOpenMode.WriteUpdate, memory, true); + using var stream = new LuaStream(LuaFileOpenMode.WriteUpdate, memory, LuaStringMode.Bytes); await stream.WriteAsync(LuaByteString.FromBytes(bytes).AsMemory(), default); Assert.That(memory.ToArray(), Is.EqualTo(bytes)); stream.Seek(SeekOrigin.Begin, 0); @@ -139,36 +173,134 @@ public async Task RawLinesPreserveCarriageReturnsIncludingAtBufferEdges() { var first = new string('a', 1023) + "\r"; var bytes = LuaByteString.ToBytes(first + "\nnext\rpart\nlast\r"); - using var stream = new LuaStream(LuaFileOpenMode.Read, new MemoryStream(bytes), true); + using var stream = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(bytes), + LuaStringMode.Bytes + ); Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo(first)); Assert.That(await stream.ReadLineAsync(true, default), Is.EqualTo("next\rpart\n")); Assert.That(await stream.ReadLineAsync(false, default), Is.EqualTo("last\r")); Assert.That(await stream.ReadLineAsync(false, default), Is.Null); } - [TestCase(false, false)] - [TestCase(true, false)] - [TestCase(true, true)] + [TestCase(LuaStringMode.Utf16, false)] + [TestCase(LuaStringMode.Bytes, false)] + [TestCase(LuaStringMode.Bytes, true)] public async Task HostModuleLoadersReceiveDotNetNamesAndReturnTextOrBytes( - bool byteStrings, + LuaStringMode stringMode, bool bytes ) { var loader = new MemoryModuleLoader(bytes); using var state = LuaState.Create( - byteStrings ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default ); state.ModuleLoader = loader; state.OpenStandardLibraries(); - state.Environment["name"] = byteStrings - ? LuaByteString.FromString(MemoryModuleLoader.Name) - : MemoryModuleLoader.Name; + state.Environment["name"] = + stringMode == LuaStringMode.Bytes + ? LuaByteString.FromString(MemoryModuleLoader.Name) + : MemoryModuleLoader.Name; var result = await state.DoStringAsync("local a=require(name); return a.text,a.size"); Assert.That( result[0].Read(), - Is.EqualTo(byteStrings ? LuaByteString.FromString("\u1EBF") : "\u1EBF") + Is.EqualTo( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString("\u1EBF") : "\u1EBF" + ) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(stringMode == LuaStringMode.Bytes ? 3 : 1))); + } + + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task HostSourceEntryPointsAcceptDotNetTextAndChunkNames(LuaStringMode stringMode) + { + using var state = CreateState(stringMode); + const string text = "こんにちはé😀"; + const string source = "return 'こんにちはé😀', #'こんにちはé😀', debug.getinfo(1).source"; + const string name = "@日本_é_😀.lua"; + LuaValue[] expected = + [ + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(text) : text, + stringMode == LuaStringMode.Bytes ? Encoding.UTF8.GetByteCount(text) : text.Length, + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(name) : name, + ]; + + Assert.That(await state.ExecuteAsync(state.Load(source, name)), Is.EqualTo(expected)); + var buffer = new LuaValue[expected.Length]; + Assert.That(await state.DoStringAsync(source, buffer, name), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + + var bytes = Encoding.UTF8.GetBytes(source); + Assert.That(await state.ExecuteAsync(bytes, name), Is.EqualTo(expected)); + Assert.That(await state.ExecuteAsync(bytes, buffer, name), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + } + + [TestCase(LuaStringMode.Utf16)] + [TestCase(LuaStringMode.Bytes)] + public async Task HostFileEntryPointsAcceptDotNetPathsWhileLuaUsesLuaStrings( + LuaStringMode stringMode + ) + { + using var state = CreateState(stringMode); + const string fileName = "日本_é_😀.lua"; + const string text = "こんにちは"; + var path = Path.Combine(directory, fileName); + await File.WriteAllTextAsync(path, "return 'こんにちは', #'こんにちは'", Encoding.UTF8); + Assert.That(state.Platform.FileSystem.IsReadable(path), Is.True); + LuaValue[] expected = + [ + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(text) : text, + stringMode == LuaStringMode.Bytes ? Encoding.UTF8.GetByteCount(text) : text.Length, + ]; + + Assert.That(await state.DoFileAsync(path), Is.EqualTo(expected)); + var buffer = new LuaValue[expected.Length]; + Assert.That(await state.DoFileAsync(fileName, buffer), Is.EqualTo(expected.Length)); + Assert.That(buffer, Is.EqualTo(expected)); + var closure = await state.LoadFileAsync(path, "t", null, default); + Assert.That(await state.ExecuteAsync(closure), Is.EqualTo(expected)); + + state.Environment["path"] = + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(fileName) : fileName; + var module = Path.GetFileNameWithoutExtension(fileName); + state.Environment["module"] = + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(module) : module; + state.Environment["text"] = expected[0]; + await state.DoStringAsync( + """ + assert(assert(loadfile(path))() == text) + assert(dofile(path) == text) + assert(require(module) == text) + """ ); - Assert.That(result[1], Is.EqualTo(new LuaValue(byteStrings ? 3 : 1))); + + state.Platform = state.Platform with + { + FileSystem = CompositeLoaderFileSystem.Create( + state.Platform.FileSystem, + new MemoryFileLoader(stringMode) + ), + }; + await state.DoStringAsync("assert(require('virtual_日本') == text)"); + } + + sealed class MemoryFileLoader(LuaStringMode stringMode) : ILuaFileLoader + { + public bool Exists(string path) => path == "virtual_日本.lua"; + + public ValueTask LoadAsync(string path, CancellationToken cancellationToken) + { + Assert.That(path, Is.EqualTo("virtual_日本.lua")); + const string source = "return 'こんにちは'"; + return new( + ILuaStream.CreateFromString( + stringMode == LuaStringMode.Bytes ? LuaByteString.FromString(source) : source + ) + ); + } } sealed class MemoryModuleLoader(bool bytes) : ILuaModuleLoader diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs index 2ceb825e..d5de8595 100644 --- a/tests/Lua.Tests/ByteStringTests.cs +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -39,7 +39,7 @@ LuaState CreateByteState() var state = LuaState.Create( LuaPlatform.DefaultByteStrings with { - FileSystem = new FileSystem(testDirectory, byteStrings: true), + FileSystem = new FileSystem(testDirectory, LuaStringMode.Bytes), } ); state.OpenStandardLibraries(); @@ -55,8 +55,8 @@ async Task RunAsync(string body) [Test] public void DefaultPlatform_KeepsUtf16Strings() { - Assert.That(LuaPlatform.Default.ByteStrings, Is.False); - Assert.That(LuaPlatform.DefaultByteStrings.ByteStrings, Is.True); + Assert.That(LuaPlatform.Default.StringMode, Is.EqualTo(LuaStringMode.Utf16)); + Assert.That(LuaPlatform.DefaultByteStrings.StringMode, Is.EqualTo(LuaStringMode.Bytes)); } [Test] @@ -241,10 +241,10 @@ public Task Comparison_IsByteWise() => ); [Test] - public async Task LoadString_KeepsSourceBytes() + public async Task DoString_EncodesDotNetSourceAsUtf8() { using var state = CreateByteState(); - var source = LuaByteString.FromString("return 'Ti\u1EBFn', #'Ti\u1EBFn', '\u1EBF'"); + var source = "return 'Ti\u1EBFn', #'Ti\u1EBFn', '\u1EBF'"; var result = await state.DoStringAsync(source); Assert.That(result[0].Read(), Is.EqualTo(LuaByteString.FromString("Ti\u1EBFn"))); @@ -401,7 +401,7 @@ public async Task Require_FindsModuleWithNonAsciiName() using var state = CreateByteState(); var result = await state.DoStringAsync( $$""" - package.path = "{{LuaByteString.FromString(testDirectory.Replace("\\", "/"))}}/?.lua" + package.path = "{{testDirectory.Replace("\\", "/")}}/?.lua" return require("m\xE1\xBA\xBFd") """ ); @@ -498,7 +498,7 @@ public async Task LuaStream_ByteMode_RoundTripsEveryByte() { var all = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray(); using var memory = new MemoryStream(); - using (var writer = new LuaStream(LuaFileOpenMode.Write, memory, byteStrings: true)) + using (var writer = new LuaStream(LuaFileOpenMode.Write, memory, LuaStringMode.Bytes)) { await writer.WriteAsync( LuaByteString.FromBytes(all).AsMemory(), @@ -511,7 +511,7 @@ await writer.WriteAsync( using var reader = new LuaStream( LuaFileOpenMode.Read, new MemoryStream(all), - byteStrings: true + LuaStringMode.Bytes ); var text = await reader.ReadAllAsync(CancellationToken.None); Assert.That(LuaByteString.ToBytes(text), Is.EqualTo(all)); @@ -534,7 +534,7 @@ public async Task LuaStream_ByteMode_ReadCountCountsBytes() using var reader = new LuaStream( LuaFileOpenMode.Read, new MemoryStream(new byte[] { 0xE1, 0xBA, 0xBF, 0x61 }), - byteStrings: true + LuaStringMode.Bytes ); var first = await reader.ReadAsync(2, CancellationToken.None); diff --git a/tests/Lua.Tests/Helpers/TestStandardIO.cs b/tests/Lua.Tests/Helpers/TestStandardIO.cs index e447283d..ad867851 100644 --- a/tests/Lua.Tests/Helpers/TestStandardIO.cs +++ b/tests/Lua.Tests/Helpers/TestStandardIO.cs @@ -2,9 +2,9 @@ namespace Lua.Tests.Helpers; -public class TestStandardIO(bool byteStrings = false) : ILuaStandardIO +public class TestStandardIO(LuaStringMode stringMode = LuaStringMode.Utf16) : ILuaStandardIO { - readonly ConsoleStandardIO consoleStandardIO = new(byteStrings); + readonly ConsoleStandardIO consoleStandardIO = new(stringMode); public ILuaStream Input { @@ -19,7 +19,9 @@ public ILuaStream Input (memory) => { Console.WriteLine( - byteStrings ? LuaByteString.ToString(memory.ToString()) : memory.ToString() + stringMode == LuaStringMode.Bytes + ? LuaByteString.ToString(memory.ToString()) + : memory.ToString() ); } ) diff --git a/tests/Lua.Tests/LuaTests.cs b/tests/Lua.Tests/LuaTests.cs index 62683ea1..08483a8c 100644 --- a/tests/Lua.Tests/LuaTests.cs +++ b/tests/Lua.Tests/LuaTests.cs @@ -6,11 +6,11 @@ namespace Lua.Tests; -[TestFixture(false)] -[TestFixture(true)] -public class LuaTests(bool byteStrings) +[TestFixture(LuaStringMode.Utf16)] +[TestFixture(LuaStringMode.Bytes)] +public class LuaTests(LuaStringMode stringMode) { - static string PatchFilesLuaSource(string source, bool byteStrings) + static string PatchFilesLuaSource(string source, LuaStringMode stringMode) { const char utf8Bom = '\uFEFF'; var lines = source.Replace("\r\n", "\n").Split('\n'); @@ -22,7 +22,7 @@ static string PatchFilesLuaSource(string source, bool byteStrings) "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; lines[296] = "do local __f = assert(io.open(file)); local __src = assert(__f:read('*a')); __f:close(); assert(load(__src))() end -- Lua-CSharp test harness uses string load because load(reader) is unsupported."; - if (!byteStrings) + if (stringMode != LuaStringMode.Bytes) { lines[330] = $"testloadfile(\"{utf8Bom}# some comment\\nreturn 234\", 234) -- Lua-CSharp test harness uses a real BOM code point because Lua strings are UTF-16 text, not raw bytes."; @@ -84,12 +84,12 @@ public async Task Test_Lua(string file) var path = FileHelper.GetAbsolutePath(file); var baseDirectory = Path.GetDirectoryName(path)!; using var state = LuaState.Create( - byteStrings ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default + stringMode == LuaStringMode.Bytes ? LuaPlatform.DefaultByteStrings : LuaPlatform.Default ); state.Platform = state.Platform with { - StandardIO = new TestStandardIO(byteStrings), - FileSystem = new FileSystem(baseDirectory, byteStrings), + StandardIO = new TestStandardIO(stringMode), + FileSystem = new FileSystem(baseDirectory, stringMode), }; state.OpenStandardLibraries(); if (file == "tests-lua/errors.lua") @@ -104,9 +104,13 @@ public async Task Test_Lua(string file) var sourceBytes = await File.ReadAllBytesAsync(path); var source = PatchFilesLuaSource( Encoding.Latin1.GetString(sourceBytes), - byteStrings + stringMode ); - var closure = state.Load(source, "@" + Path.GetFileName(file)); + var name = "@" + Path.GetFileName(file); + var closure = + stringMode == LuaStringMode.Bytes + ? state.Load(LuaByteString.ToBytes(source), name) + : state.Load(source, name); await state.ExecuteAsync(closure); } else From c9b658124003aa4deef979fd0e917abb4dba7265 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:29:20 +0900 Subject: [PATCH 07/11] Encode UTF-8 byte strings directly into chars --- .../Benchmark/Utf8CharConversionBenchmark.cs | 37 +++++++++ src/Lua/Internal/Utf8CharEncoder.cs | 76 +++++++++++++++++++ src/Lua/LuaByteString.cs | 12 ++- src/Lua/LuaState.cs | 21 ++++- tests/Lua.Tests/ByteStringIntegrationTests.cs | 7 ++ tests/Lua.Tests/ByteStringTests.cs | 23 ++++++ 6 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 sandbox/Benchmark/Utf8CharConversionBenchmark.cs create mode 100644 src/Lua/Internal/Utf8CharEncoder.cs diff --git a/sandbox/Benchmark/Utf8CharConversionBenchmark.cs b/sandbox/Benchmark/Utf8CharConversionBenchmark.cs new file mode 100644 index 00000000..41f3ecbb --- /dev/null +++ b/sandbox/Benchmark/Utf8CharConversionBenchmark.cs @@ -0,0 +1,37 @@ +using System.Text; +using BenchmarkDotNet.Attributes; +using Lua; + +[MemoryDiagnoser] +[ShortRunJob] +public class Utf8CharConversionBenchmark +{ + [Params("Ascii", "SparseUnicode", "Japanese", "Mixed", "InvalidUtf16")] + public string Corpus { get; set; } = "Ascii"; + + [Params(4096)] + public int Length { get; set; } + + string source = default!; + + [GlobalSetup] + public void Setup() + { + var unit = Corpus switch + { + "Ascii" => "hello world 1234 ", + "SparseUnicode" => new string('a', Length - 1) + "ế", + "Japanese" => "こんにちは世界", + "Mixed" => "aế😀漢字 ", + "InvalidUtf16" => "a\uD800b\uDC00c", + _ => throw new ArgumentOutOfRangeException(nameof(Corpus)), + }; + source = string.Concat(Enumerable.Repeat(unit, Length / unit.Length + 1))[..Length]; + } + + [Benchmark(Baseline = true)] + public string ViaBytes() => LuaByteString.FromBytes(Encoding.UTF8.GetBytes(source)); + + [Benchmark] + public string DirectChars() => LuaByteString.FromString(source); +} diff --git a/src/Lua/Internal/Utf8CharEncoder.cs b/src/Lua/Internal/Utf8CharEncoder.cs new file mode 100644 index 00000000..0a464a1a --- /dev/null +++ b/src/Lua/Internal/Utf8CharEncoder.cs @@ -0,0 +1,76 @@ +using System.Runtime.InteropServices; + +namespace Lua.Internal; + +// Encodes UTF-16 text as UTF-8, storing each output byte in one char. +// The destination must have at least Encoding.UTF8.GetByteCount(source) chars. +static class Utf8CharEncoder +{ + public static int Encode(ReadOnlySpan source, Span destination) + { + var written = 0; + for (var i = 0; i < source.Length; i++) + { + uint value = source[i]; + if (value <= 0x7F) + { + // Copy four ASCII code units together, including within mixed text. + if (source.Length - i >= 4) + { + var block = MemoryMarshal.Read( + MemoryMarshal.AsBytes(source.Slice(i, 4)) + ); + if ((block & 0xFF80FF80FF80FF80UL) == 0) + { + BitConverter.TryWriteBytes( + MemoryMarshal.AsBytes(destination.Slice(written, 4)), + block + ); + written += 4; + i += 3; + continue; + } + } + destination[written++] = (char)value; + } + else if (value <= 0x7FF) + { + destination[written++] = (char)(0xC0 | (value >> 6)); + destination[written++] = (char)(0x80 | (value & 0x3F)); + } + else + { + if (char.IsSurrogate((char)value)) + { + if ( + char.IsHighSurrogate((char)value) + && i + 1 < source.Length + && char.IsLowSurrogate(source[i + 1]) + ) + { + value = (uint)char.ConvertToUtf32(source[i], source[i + 1]); + i++; + } + else + { + // Match Encoding.UTF8's replacement of each unpaired surrogate. + value = 0xFFFD; + } + } + + if (value > 0xFFFF) + { + destination[written++] = (char)(0xF0 | (value >> 18)); + destination[written++] = (char)(0x80 | ((value >> 12) & 0x3F)); + } + else + { + destination[written++] = (char)(0xE0 | (value >> 12)); + } + destination[written++] = (char)(0x80 | ((value >> 6) & 0x3F)); + destination[written++] = (char)(0x80 | (value & 0x3F)); + } + } + return written; + } +} diff --git a/src/Lua/LuaByteString.cs b/src/Lua/LuaByteString.cs index a522afc7..332a0d33 100644 --- a/src/Lua/LuaByteString.cs +++ b/src/Lua/LuaByteString.cs @@ -1,4 +1,5 @@ using System.Text; +using Lua.Internal; using Lua.IO; namespace Lua; @@ -30,7 +31,16 @@ public static byte[] ToBytes(string byteString) /// public static string FromString(string value) { - return FromBytes(Encoding.UTF8.GetBytes(value)); + var count = Encoding.UTF8.GetByteCount(value); + if (count == value.Length) + { + return value; // ASCII already has the byte-string representation. + } + return string.Create( + count, + value, + static (destination, source) => Utf8CharEncoder.Encode(source, destination) + ); } /// diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 38bbb651..3b841d48 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -606,9 +606,24 @@ public LuaClosure Load(ReadOnlySpan chunk, string chunkName, LuaTable? env return LoadCore(chunk, name, environment); } - using PooledArray bytes = new(Encoding.UTF8.GetByteCount(chunk)); - var count = Encoding.UTF8.GetBytes(chunk, bytes.AsSpan()); - return LoadCore(bytes.AsSpan()[..count], name, "t", environment); + var byteCount = Encoding.UTF8.GetByteCount(chunk); + if (byteCount > 4 && chunk[0] == '\e') + { + throw new Exception("attempt to load a binary chunk (mode is 't')"); + } + // Match the byte-source loader's handling of an initial UTF-8 BOM. + if (!chunk.IsEmpty && chunk[0] == '\uFEFF') + { + chunk = chunk[1..]; + byteCount -= 3; + } + if (byteCount == chunk.Length) + { + return LoadCore(chunk, name, environment); + } + using PooledArray chars = new(byteCount); + var count = Utf8CharEncoder.Encode(chunk, chars.AsSpan()); + return LoadCore(chars.AsSpan()[..count], name, environment); } // Source and chunk name already use this state's Lua string representation. diff --git a/tests/Lua.Tests/ByteStringIntegrationTests.cs b/tests/Lua.Tests/ByteStringIntegrationTests.cs index c0281ede..3d456cd9 100644 --- a/tests/Lua.Tests/ByteStringIntegrationTests.cs +++ b/tests/Lua.Tests/ByteStringIntegrationTests.cs @@ -228,6 +228,13 @@ public async Task HostSourceEntryPointsAcceptDotNetTextAndChunkNames(LuaStringMo ]; Assert.That(await state.ExecuteAsync(state.Load(source, name)), Is.EqualTo(expected)); + if (stringMode == LuaStringMode.Bytes) + { + Assert.That( + await state.ExecuteAsync(state.Load("\uFEFF" + source, name)), + Is.EqualTo(expected) + ); + } var buffer = new LuaValue[expected.Length]; Assert.That(await state.DoStringAsync(source, buffer, name), Is.EqualTo(expected.Length)); Assert.That(buffer, Is.EqualTo(expected)); diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs index d5de8595..8c689470 100644 --- a/tests/Lua.Tests/ByteStringTests.cs +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -544,6 +544,29 @@ public async Task LuaStream_ByteMode_ReadCountCountsBytes() Assert.That(LuaByteString.ToBytes(second!), Is.EqualTo(new byte[] { 0xBF, 0x61 })); } + [Test] + public void Utf8EncodingMatchesDotNetForSurrogateEdgeCases() + { + var source = new StringBuilder(); + for (var value = 0; value <= char.MaxValue; value++) + source.Append((char)value); + for (var value = 0x10000; value <= 0x10FFFF; value += 997) + source.Append(char.ConvertFromUtf32(value)); + source.Append("\uD800\uDC00\uDBFF\uDFFF\uD800\uD800\uDC00\uDC00\uD800x\uDC00"); + var text = source.ToString(); + var actual = LuaByteString.FromString(text); + Assert.That(actual.All(c => c <= 255), Is.True); + Assert.That(LuaByteString.ToBytes(actual), Is.EqualTo(Encoding.UTF8.GetBytes(text))); + for (var length = 0; length < 65; length++) + { + text = new string('a', length) + "ế😀\uD800x" + new string('z', length); + Assert.That( + LuaByteString.ToBytes(LuaByteString.FromString(text)), + Is.EqualTo(Encoding.UTF8.GetBytes(text)) + ); + } + } + [Test] public void LuaByteString_ConvertsBetweenDotNetStringsAndBytes() { From 91bad26fdb1f73f20b5528711dc395765e581499 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:32:41 +0900 Subject: [PATCH 08/11] Document UTF-16 and byte string behavior --- README.md | 4 +-- docs/string-modes.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 docs/string-modes.md diff --git a/README.md b/README.md index 8cc396c9..02955c31 100644 --- a/README.md +++ b/README.md @@ -673,7 +673,7 @@ Lua-CSharp is designed with integration into .NET in mind, so there are several ### Character Encoding -The character encoding used in Lua-CSharp is UTF-16. Since standard Lua assumes a single-byte character encoding, string behavior differs significantly. +By default, Lua-CSharp uses UTF-16. Since standard Lua assumes a single-byte character encoding, string behavior differs significantly. See [String modes](docs/string-modes.md) for UTF-16 and byte-string behavior and C# conversions. For example, in regular Lua, the following code outputs `15`, but in Lua-CSharp, it outputs `5`. @@ -682,7 +682,7 @@ local l = string.len("あいうえお") print(l) ``` -All string library functions handle strings as UTF-16. +In the default mode, string library functions handle strings as UTF-16. #### Byte strings (opt-in) diff --git a/docs/string-modes.md b/docs/string-modes.md new file mode 100644 index 00000000..8993ac1a --- /dev/null +++ b/docs/string-modes.md @@ -0,0 +1,66 @@ +# String modes + +`LuaStringMode.Utf16` is the default. Lua strings contain .NET UTF-16 code units. +`LuaStringMode.Bytes` stores each byte in one C# `char` (0–255), including arbitrary binary data. + +Choose the mode before running scripts: + +```csharp +using Lua; +using Lua.Platforms; +using Lua.Standard; + +using var textState = LuaState.Create(); // Utf16 +using var byteState = LuaState.Create(LuaPlatform.DefaultByteStrings); +byteState.OpenStandardLibraries(); +``` + +## Lua behavior + +The examples below assume UTF-8 source text. + +| Operation | `Utf16` | `Bytes` | +| --- | --- | --- | +| `#"ế"` | 1 | 3 | +| `#"😀"` | 2 | 4 | +| `string.len`, `string.sub`, `string.byte` | UTF-16 code units | Bytes | +| Case conversion and pattern classes | .NET character rules | ASCII C-locale rules | +| Built-in file and standard I/O | UTF-8 text decoded to/from UTF-16 | Raw bytes | + +Neither mode counts user-perceived characters. In byte mode, `string.sub` can split a UTF-8 sequence and `string.char` accepts only 0–255. + +## C# string values + +UTF-16 values can use .NET strings directly. In byte mode, convert text and binary data explicitly: + +```csharp +byteState.Environment["greeting"] = LuaByteString.FromString("こんにちは"); +var values = await byteState.DoStringAsync("return greeting"); +string greeting = LuaByteString.ToString(values[0].Read()); + +byteState.Environment["packet"] = LuaByteString.FromBytes(new byte[] { 0xFF, 0, 0x80 }); +var packets = await byteState.DoStringAsync("return packet"); +byte[] packet = LuaByteString.ToBytes(packets[0].Read()); +``` + +`FromString` and `ToString` encode/decode UTF-8. Invalid sequences are replaced with U+FFFD when decoding. `FromBytes` and `ToBytes` preserve binary data; chars above U+00FF become `?` when converted to bytes. + +Value conversion is also explicit for non-ASCII table keys, C# function arguments/results, and generated bindings. + +## Source, paths and streams + +- C# `DoStringAsync`, character-source `Load`, and text `LuaModule` sources accept normal .NET text. Byte mode encodes that source as UTF-8. +- In byte mode, byte-source `Load`, byte modules and source files preserve source bytes, except for an initial UTF-8 BOM. Lua's `load` consumes strings in the state's representation without encoding them again. +- C# chunk names and paths, including names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader`, are normal .NET strings in both modes. +- Byte-mode I/O preserves a BOM read as data. Line reads stop at LF and retain CR; `*L` also retains LF. Raw output replaces chars above U+00FF with `?`. + +`LuaPlatform.DefaultByteStrings` configures matching byte-mode file and standard I/O. Changing `StringMode` alone does not reconfigure streams. Custom providers must use the same representation as the state; wrap raw streams with `ILuaStream.CreateFromStream(stream, openMode, LuaStringMode.Bytes)`. + +To set a file root, keep the path as .NET text: + +```csharp +var platform = LuaPlatform.DefaultByteStrings with +{ + FileSystem = new Lua.IO.FileSystem(scriptsDirectory, LuaStringMode.Bytes) +}; +``` From 349cbad5c9b6509ec0fe533d1eb7058c37d819c9 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:06:56 +0900 Subject: [PATCH 09/11] Organize string mode guidance and practical caveats --- docs/string-modes.md | 80 +++++++++++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/docs/string-modes.md b/docs/string-modes.md index 8993ac1a..7d9befc8 100644 --- a/docs/string-modes.md +++ b/docs/string-modes.md @@ -1,9 +1,8 @@ # String modes -`LuaStringMode.Utf16` is the default. Lua strings contain .NET UTF-16 code units. -`LuaStringMode.Bytes` stores each byte in one C# `char` (0–255), including arbitrary binary data. +## Choose a mode -Choose the mode before running scripts: +`LuaStringMode.Utf16` is the default and uses ordinary .NET UTF-16 strings. Use `LuaStringMode.Bytes` for Lua byte semantics or binary data; each C# `char` then holds one byte (0–255). ```csharp using Lua; @@ -15,9 +14,11 @@ using var byteState = LuaState.Create(LuaPlatform.DefaultByteStrings); byteState.OpenStandardLibraries(); ``` -## Lua behavior +Keep the mode fixed for a state's lifetime. Changing its platform does not convert existing strings, table keys or compiled chunks. Values shared between states with different modes need explicit conversion. -The examples below assume UTF-8 source text. +## Lua string operations + +For UTF-8 source text: | Operation | `Utf16` | `Bytes` | | --- | --- | --- | @@ -25,16 +26,28 @@ The examples below assume UTF-8 source text. | `#"😀"` | 2 | 4 | | `string.len`, `string.sub`, `string.byte` | UTF-16 code units | Bytes | | Case conversion and pattern classes | .NET character rules | ASCII C-locale rules | -| Built-in file and standard I/O | UTF-8 text decoded to/from UTF-16 | Raw bytes | -Neither mode counts user-perceived characters. In byte mode, `string.sub` can split a UTF-8 sequence and `string.char` accepts only 0–255. +Lengths and slices do not count user-perceived characters. UTF-16 slicing can split a surrogate pair; byte slicing can split a UTF-8 sequence. Byte positions from Lua cannot be reused as indices into decoded .NET text. Use .NET text-element APIs on decoded text for user-perceived-character editing. + +In byte mode, `string.char` accepts only 0–255. Non-ASCII letters remain unchanged by `string.upper`/`string.lower` and do not match `%a`. Decode text before applying Unicode case conversion or classification in C#. + +Neither mode normalizes Unicode. For example, C# `"é"` and `"e\u0301"` remain different strings and table keys. + +## Passing values between C# and Lua + +UTF-16 values can use .NET strings directly. In byte mode, convert values according to their contents: -## C# string values +| Conversion | API | +| --- | --- | +| .NET text → Lua byte string | `LuaByteString.FromString(text)` | +| UTF-8 or binary bytes → Lua byte string | `LuaByteString.FromBytes(bytes)` | +| Lua byte string → .NET text | `LuaByteString.ToString(value)` | +| Lua byte string → raw bytes | `LuaByteString.ToBytes(value)` | -UTF-16 values can use .NET strings directly. In byte mode, convert text and binary data explicitly: +C# UTF-8 literals (`u8`, C# 11+) can supply bytes directly. `FromString("é")` and `FromBytes("é"u8)` produce the same bytes, `C3 A9`. ```csharp -byteState.Environment["greeting"] = LuaByteString.FromString("こんにちは"); +byteState.Environment["greeting"] = LuaByteString.FromBytes("こんにちは"u8); var values = await byteState.DoStringAsync("return greeting"); string greeting = LuaByteString.ToString(values[0].Read()); @@ -43,24 +56,53 @@ var packets = await byteState.DoStringAsync("return packet"); byte[] packet = LuaByteString.ToBytes(packets[0].Read()); ``` -`FromString` and `ToString` encode/decode UTF-8. Invalid sequences are replaced with U+FFFD when decoding. `FromBytes` and `ToBytes` preserve binary data; chars above U+00FF become `?` when converted to bytes. +### Convert text once + +Both representations use C# `string`; the type does not identify the representation. A string read from a byte-mode state is already a byte string. Calling `FromString` on it again changes non-ASCII data. + +Value conversion is explicit for table keys, C# function arguments/results and generated bindings too. ASCII-only testing can hide missing conversions: directly assigning .NET `"é"` in byte mode gives one byte, `E9`, rather than the UTF-8 bytes `C3 A9`. + +### Preserve binary data + +Use `FromBytes`/`ToBytes` for NUL bytes, invalid UTF-8 and other binary data. `FromString` replaces unpaired UTF-16 surrogates; `ToString` replaces invalid UTF-8 sequences with U+FFFD. These replacements lose the original data. -Value conversion is also explicit for non-ASCII table keys, C# function arguments/results, and generated bindings. +Valid byte strings contain only chars up to U+00FF. `ToBytes` and raw output replace higher chars with `?`. -## Source, paths and streams +## Loading source and naming files -- C# `DoStringAsync`, character-source `Load`, and text `LuaModule` sources accept normal .NET text. Byte mode encodes that source as UTF-8. -- In byte mode, byte-source `Load`, byte modules and source files preserve source bytes, except for an initial UTF-8 BOM. Lua's `load` consumes strings in the state's representation without encoding them again. -- C# chunk names and paths, including names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader`, are normal .NET strings in both modes. -- Byte-mode I/O preserves a BOM read as data. Line reads stop at LF and retain CR; `*L` also retains LF. Raw output replaces chars above U+00FF with `?`. +Source-loading APIs handle conversion for you: + +```csharp +// .NET text: encoded as UTF-8 in byte mode. +var textChunk = byteState.Load("return 'é'", "example"); + +// Already encoded UTF-8: pass bytes directly. +var utf8Chunk = byteState.Load("return 'é'"u8, "example"); +``` + +- `DoStringAsync`, character-source `Load` and text `LuaModule` sources accept normal .NET text. Applying `FromString` first would encode the source twice in byte mode. +- In byte mode, byte-source `Load`, byte modules and source files preserve source bytes, except for an initial UTF-8 BOM. Lua's `load` uses strings in the state's representation without encoding them again. +- C# chunk names and paths are normal .NET strings in both modes, including names passed to `ILuaFileSystem`, `ILuaFileLoader` and `ILuaModuleLoader`. + +## Configuring file and standard I/O `LuaPlatform.DefaultByteStrings` configures matching byte-mode file and standard I/O. Changing `StringMode` alone does not reconfigure streams. Custom providers must use the same representation as the state; wrap raw streams with `ILuaStream.CreateFromStream(stream, openMode, LuaStringMode.Bytes)`. -To set a file root, keep the path as .NET text: +A file root remains a normal .NET path: ```csharp var platform = LuaPlatform.DefaultByteStrings with { - FileSystem = new Lua.IO.FileSystem(scriptsDirectory, LuaStringMode.Bytes) + FileSystem = new Lua.IO.FileSystem("scripts", LuaStringMode.Bytes) }; ``` + +| I/O behavior | `Utf16` | `Bytes` | +| --- | --- | --- | +| File and standard stream contents | UTF-8 text decoded to/from UTF-16 | Raw bytes | +| `file:read(n)` | Up to `n` UTF-16 code units | Up to `n` bytes | +| `*l` reading `hello\r\n` | `hello` | `hello\r` | +| `*L` reading `hello\r\n` | `hello\r\n` | `hello\r\n` | +| Initial UTF-8 BOM read as data | Skipped | Preserved | + +The `b` flag in `io.open(path, "rb")` does not select byte-string mode. Built-in streams use their configured `LuaStringMode` for both `"r"` and `"rb"`. Byte-mode line reads stop at LF; remove a trailing CR explicitly when the application expects a line without either newline character. From 289abe1e273c51e2189a173af992424e5a601391 Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:15:48 +0900 Subject: [PATCH 10/11] Clarify Lua byte-string semantics and compatibility limits --- docs/string-modes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/string-modes.md b/docs/string-modes.md index 7d9befc8..732ab361 100644 --- a/docs/string-modes.md +++ b/docs/string-modes.md @@ -11,6 +11,7 @@ using Lua.Standard; using var textState = LuaState.Create(); // Utf16 using var byteState = LuaState.Create(LuaPlatform.DefaultByteStrings); +textState.OpenStandardLibraries(); byteState.OpenStandardLibraries(); ``` @@ -18,6 +19,8 @@ Keep the mode fixed for a state's lifetime. Changing its platform does not conve ## Lua string operations +Byte mode follows Lua 5.2's byte-string model: strings can contain arbitrary bytes, and lengths, substring positions and pattern matching operate on bytes. Character classes and case conversion follow ASCII C-locale rules. + For UTF-8 source text: | Operation | `Utf16` | `Bytes` | @@ -27,12 +30,23 @@ For UTF-8 source text: | `string.len`, `string.sub`, `string.byte` | UTF-16 code units | Bytes | | Case conversion and pattern classes | .NET character rules | ASCII C-locale rules | +For example, this script returns different results in each mode: + +```lua +local s = "é" +return #s, string.byte(s, 1), s:match("^%a+$") ~= nil +-- Utf16: 1, 233, true +-- Bytes: 2, 195, false +``` + Lengths and slices do not count user-perceived characters. UTF-16 slicing can split a surrogate pair; byte slicing can split a UTF-8 sequence. Byte positions from Lua cannot be reused as indices into decoded .NET text. Use .NET text-element APIs on decoded text for user-perceived-character editing. In byte mode, `string.char` accepts only 0–255. Non-ASCII letters remain unchanged by `string.upper`/`string.lower` and do not match `%a`. Decode text before applying Unicode case conversion or classification in C#. Neither mode normalizes Unicode. For example, C# `"é"` and `"e\u0301"` remain different strings and table keys. +This describes string semantics. Existing Lua-CSharp limitations still apply; for example, `string.dump` is unsupported. C# values still use `string` and require the conversions below, with matching stream configuration. + ## Passing values between C# and Lua UTF-16 values can use .NET strings directly. In byte mode, convert values according to their contents: From 7ffb715b73734748d19340f08cfb555da7021c2c Mon Sep 17 00:00:00 2001 From: akeit0 <90429982+Akeit0@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:17:39 +0900 Subject: [PATCH 11/11] Decode byte strings for Unity log output --- docs/string-modes.md | 2 ++ .../Lua.Unity/Runtime/UnityStandardIO.cs | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/string-modes.md b/docs/string-modes.md index 732ab361..a3714678 100644 --- a/docs/string-modes.md +++ b/docs/string-modes.md @@ -102,6 +102,8 @@ var utf8Chunk = byteState.Load("return 'é'"u8, "example"); `LuaPlatform.DefaultByteStrings` configures matching byte-mode file and standard I/O. Changing `StringMode` alone does not reconfigure streams. Custom providers must use the same representation as the state; wrap raw streams with `ILuaStream.CreateFromStream(stream, openMode, LuaStringMode.Bytes)`. +For Unity, use `new Lua.Unity.UnityStandardIO(LuaStringMode.Bytes)` with a byte-mode state. It decodes UTF-8 for display through `Debug.Log`/`Debug.LogError`; use raw streams to preserve binary output. A supplied input stream must also use byte strings. + A file root remains a normal .NET path: ```csharp diff --git a/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs b/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs index 0c3a4b98..2c921dce 100644 --- a/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs +++ b/src/Lua.Unity/Assets/Lua.Unity/Runtime/UnityStandardIO.cs @@ -9,20 +9,18 @@ namespace Lua.Unity public class UnityStandardIO : ILuaStandardIO { public UnityStandardIO(ILuaStream input = null) + : this(LuaStringMode.Utf16, input) { } + + public UnityStandardIO(LuaStringMode stringMode, ILuaStream input = null) { - if (input != null) - { - Input = input; - } - else - { - Input = new DummyInputStream(); - } + Input = input ?? new DummyInputStream(); + Output = new DebugLogStream(false, stringMode); + Error = new DebugLogStream(true, stringMode); } public ILuaStream Input { get; } - public ILuaStream Output { get; } = new DebugLogStream(false); - public ILuaStream Error { get; } = new DebugLogStream(true); + public ILuaStream Output { get; } + public ILuaStream Error { get; } } internal class DummyInputStream : ILuaStream @@ -36,8 +34,12 @@ public void Dispose() { } public class DebugLogStream : ILuaStream { public DebugLogStream(bool isError = false) + : this(isError, LuaStringMode.Utf16) { } + + public DebugLogStream(bool isError, LuaStringMode stringMode) { IsError = isError; + this.stringMode = stringMode; } public bool IsError { get; } = false; @@ -45,6 +47,7 @@ public DebugLogStream(bool isError = false) public LuaFileOpenMode Mode => LuaFileOpenMode.Write; private readonly StringBuilder stringBuilder = new(); + private readonly LuaStringMode stringMode; ValueTask ILuaStream.WriteAsync( ReadOnlyMemory content, @@ -60,6 +63,8 @@ ValueTask ILuaStream.FlushAsync(CancellationToken cancellationToken) if (stringBuilder.Length > 0) { var message = stringBuilder.ToString(); + if (stringMode == LuaStringMode.Bytes) + message = LuaByteString.ToString(message); if (IsError) UnityEngine.Debug.LogError(message); else