Add opt-in byte strings (LuaPlatform.ByteStrings)

README.md

@@ -684,6 +684,26 @@ print(l)
 
 All string library functions handle strings as UTF-16.
 
+#### Byte strings (opt-in)
+
+If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), enable `LuaPlatform.ByteStrings`. `LuaPlatform.DefaultByteStrings` is the default platform with byte strings and byte-oriented file and standard I/O:
+
+```cs
+using Lua;
+using Lua.Platforms;
+using Lua.Standard;
+
+var state = LuaState.Create(LuaPlatform.DefaultByteStrings);
+state.OpenStandardLibraries();
+
+// Every char of a Lua string holds one byte, so convert .NET strings at the boundary.
+state.Environment["name"] = LuaByteString.FromString("Tiến Lên"); // UTF-8 bytes, #name == 11
+var results = await state.DoStringAsync("return #name, name:upper()");
+var upper = LuaByteString.ToString(results[1].Read<string>());   // back to a .NET string
+```
+
+In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16.
+
 ### Garbage Collection
 
 Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua.

src/Lua/CodeAnalysis/Compilation/BomUtility.cs

@@ -12,6 +12,11 @@ static class BomUtility
 
     static ReadOnlySpan<byte> BomUtf32Little => [0xFF, 0xFE, 0x00, 0x00];
 
+    public static ReadOnlySpan<byte> SkipUtf8Bom(ReadOnlySpan<byte> text)
+    {
+        return text.StartsWith(BomUtf8) ? text.Slice(BomUtf8.Length) : text;
+    }
+
     /// <summary>
     ///  Removes the BOM from the beginning of the text and returns the encoding.
     ///  Supported encodings are UTF-8, UTF-16 (little and big endian), and UTF-32 (little endian).

src/Lua/IO/ConsoleStandardIO.cs

@@ -5,24 +5,40 @@ namespace Lua.IO;
 /// <summary>
 /// Default implementation of ILuaStandardIO using Console
 /// </summary>
-public sealed class ConsoleStandardIO : ILuaStandardIO
+/// <param name="byteStrings">
+/// When true, standard input and output are read and written as raw bytes (one byte per char)
+/// instead of UTF-8 text. Use together with <see cref="Lua.Platforms.LuaPlatform.ByteStrings"/>.
+/// </param>
+public sealed class ConsoleStandardIO(bool byteStrings = false) : ILuaStandardIO
 {
     ILuaStream? standardInput;
     ILuaStream? standardOutput;
     ILuaStream? standardError;
 
     public ILuaStream Input =>
         standardInput ??= new StandardIOStream(
-            ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardInput(), LuaFileOpenMode.Read)
+            ILuaStream.CreateFromStream(
+                ConsoleHelper.OpenStandardInput(),
+                LuaFileOpenMode.Read,
+                byteStrings
+            )
         );
 
     public ILuaStream Output =>
         standardOutput ??= new StandardIOStream(
-            ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardOutput(), LuaFileOpenMode.Write)
+            ILuaStream.CreateFromStream(
+                ConsoleHelper.OpenStandardOutput(),
+                LuaFileOpenMode.Write,
+                byteStrings
+            )
         );
 
     public ILuaStream Error =>
         standardError ??= new StandardIOStream(
-            ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardError(), LuaFileOpenMode.Write)
+            ILuaStream.CreateFromStream(
+                ConsoleHelper.OpenStandardError(),
+                LuaFileOpenMode.Write,
+                byteStrings
+            )
         );
 }

src/Lua/IO/FileSystem.cs

@@ -1,6 +1,12 @@
 namespace Lua.IO;
 
-public sealed class FileSystem(string? baseDirectory = null) : ILuaFileSystem
+/// <param name="baseDirectory">Directory that relative paths are resolved against.</param>
+/// <param name="byteStrings">
+/// When true, files are read and written as raw bytes (one byte per char) instead of UTF-8 text.
+/// Use together with <see cref="Lua.Platforms.LuaPlatform.ByteStrings"/>.
+/// </param>
+public sealed class FileSystem(string? baseDirectory = null, bool byteStrings = false)
+    : ILuaFileSystem
 {
     public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory();
 
@@ -32,9 +38,15 @@ public string GetFullPath(string path)
         return Path.Combine(baseDirectory, path);
     }
 
+    // Paths come from Lua as byte strings in byte mode: decode the UTF-8 bytes into the real file name.
+    string ResolvePath(string path)
+    {
+        return GetFullPath(byteStrings ? LuaByteString.ToString(path) : path);
+    }
+
     public bool IsReadable(string path)
     {
-        return File.Exists(GetFullPath(path));
+        return File.Exists(ResolvePath(path));
     }
 
     public ValueTask<ILuaStream> Open(
@@ -45,7 +57,7 @@ CancellationToken cancellationToken
     {
         var (mode, access) = GetFileMode(openMode);
         Stream stream;
-        path = GetFullPath(path);
+        path = ResolvePath(path);
         if (openMode == LuaFileOpenMode.AppendUpdate)
         {
             stream = File.Open(
@@ -60,7 +72,7 @@ CancellationToken cancellationToken
             stream = File.Open(path, mode, access, FileShare.ReadWrite | FileShare.Delete);
         }
 
-        ILuaStream wrapper = new LuaStream(openMode, stream);
+        ILuaStream wrapper = new LuaStream(openMode, stream, byteStrings);
 
         if (openMode == LuaFileOpenMode.AppendUpdate)
         {
@@ -72,8 +84,8 @@ CancellationToken cancellationToken
 
     public ValueTask Rename(string oldName, string newName, CancellationToken cancellationToken)
     {
-        oldName = GetFullPath(oldName);
-        newName = GetFullPath(newName);
+        oldName = ResolvePath(oldName);
+        newName = ResolvePath(newName);
         if (oldName == newName)
         {
             return default;
@@ -91,7 +103,7 @@ public ValueTask Rename(string oldName, string newName, CancellationToken cancel
 
     public ValueTask Remove(string path, CancellationToken cancellationToken)
     {
-        path = GetFullPath(path);
+        path = ResolvePath(path);
         if (!File.Exists(path))
         {
             throw new FileNotFoundException("No such file or directory", path);
@@ -107,7 +119,8 @@ public ValueTask Remove(string path, CancellationToken cancellationToken)
 
     public string GetTempFileName()
     {
-        return Path.GetTempFileName();
+        var name = Path.GetTempFileName();
+        return byteStrings ? LuaByteString.FromString(name) : name;
     }
 
     public ValueTask<ILuaStream> OpenTempFileStream(CancellationToken cancellationToken)

src/Lua/IO/ILuaStream.cs

@@ -74,6 +74,11 @@ static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode)
         return new LuaStream(openMode, stream);
     }
 
+    static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode, bool byteStrings)
+    {
+        return new LuaStream(openMode, stream, byteStrings);
+    }
+
     static ILuaStream CreateFromString(string content)
     {
         return new StringStream(content);

src/Lua/IO/LuaByteEncoding.cs

@@ -0,0 +1,88 @@
+using System.Text;
+
+namespace Lua.IO;
+
+/// <summary>
+/// Maps one byte to one <see cref="char"/> (0x00-0xFF) and back, so that a Lua string can carry
+/// arbitrary bytes (for example UTF-8 text) with the same <c>#s</c>, <c>string.sub</c> and
+/// <c>string.byte</c> results as the reference C implementation.
+/// Characters above 0xFF cannot be represented and are written as '?'.
+/// </summary>
+sealed class LuaByteEncoding : Encoding
+{
+    public static LuaByteEncoding Instance { get; } = new();
+
+    public override int GetByteCount(char[] chars, int index, int count) => count;
+
+    public override int GetByteCount(string s) => s.Length;
+
+    public override int GetByteCount(ReadOnlySpan<char> chars) => chars.Length;
+
+    public override int GetBytes(
+        char[] chars,
+        int charIndex,
+        int charCount,
+        byte[] bytes,
+        int byteIndex
+    ) => GetBytes(chars.AsSpan(charIndex, charCount), bytes.AsSpan(byteIndex));
+
+    public override int GetBytes(ReadOnlySpan<char> chars, Span<byte> bytes)
+    {
+        if (bytes.Length < chars.Length)
+        {
+            throw new ArgumentException("Destination is too small.", nameof(bytes));
+        }
+
+        for (var i = 0; i < chars.Length; i++)
+        {
+            var c = chars[i];
+            bytes[i] = c <= 0xFF ? (byte)c : (byte)'?';
+        }
+
+        return chars.Length;
+    }
+
+    public override int GetCharCount(byte[] bytes, int index, int count) => count;
+
+    public override int GetCharCount(ReadOnlySpan<byte> bytes) => bytes.Length;
+
+    public override int GetChars(
+        byte[] bytes,
+        int byteIndex,
+        int byteCount,
+        char[] chars,
+        int charIndex
+    ) => GetChars(bytes.AsSpan(byteIndex, byteCount), chars.AsSpan(charIndex));
+
+    public override int GetChars(ReadOnlySpan<byte> bytes, Span<char> chars)
+    {
+        if (chars.Length < bytes.Length)
+        {
+            throw new ArgumentException("Destination is too small.", nameof(chars));
+        }
+
+        for (var i = 0; i < bytes.Length; i++)
+        {
+            chars[i] = (char)bytes[i];
+        }
+
+        return bytes.Length;
+    }
+
+    public override string GetString(byte[] bytes, int index, int count) =>
+        string.Create(
+            count,
+            (bytes, index),
+            static (span, state) =>
+            {
+                for (var i = 0; i < span.Length; i++)
+                {
+                    span[i] = (char)state.bytes[state.index + i];
+                }
+            }
+        );
+
+    public override int GetMaxByteCount(int charCount) => charCount;
+
+    public override int GetMaxCharCount(int byteCount) => byteCount;
+}

src/Lua/IO/LuaStream.cs

@@ -4,7 +4,9 @@
 
 namespace Lua.IO;
 
-public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaStream, ILuaByteStream
+public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings = false)
+    : ILuaStream,
+        ILuaByteStream
 {
     Utf8Reader? reader;
     ulong flushSize = ulong.MaxValue;
@@ -19,14 +21,14 @@ public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaSt
     public ValueTask<string?> ReadLineAsync(bool keepEol, CancellationToken cancellationToken)
     {
         mode.ThrowIfNotReadable();
-        reader ??= new();
+        reader ??= new(byteStrings);
         return new(reader.ReadLine(innerStream, keepEol));
     }
 
     public ValueTask<string> ReadAllAsync(CancellationToken cancellationToken)
     {
         mode.ThrowIfNotReadable();
-        reader ??= new();
+        reader ??= new(byteStrings);
         var text = reader.ReadToEnd(innerStream);
         return new(text);
     }
@@ -57,7 +59,7 @@ public ValueTask<int> ReadByteAsync(CancellationToken cancellationToken)
     public ValueTask<string?> ReadAsync(int count, CancellationToken cancellationToken)
     {
         mode.ThrowIfNotReadable();
-        reader ??= new();
+        reader ??= new(byteStrings);
         if (count == 0)
         {
             return new(reader.IsEndOfStream(innerStream) ? null : string.Empty);
@@ -69,7 +71,7 @@ public ValueTask<int> ReadByteAsync(CancellationToken cancellationToken)
     public ValueTask<double?> ReadNumberAsync(CancellationToken cancellationToken)
     {
         mode.ThrowIfNotReadable();
-        reader ??= new();
+        reader ??= new(byteStrings);
 
         // Use the Utf8Reader's ReadNumber method which handles positioning correctly
         var numberStr = reader.ReadNumber(innerStream);
@@ -93,7 +95,7 @@ public ValueTask WriteAsync(ReadOnlyMemory<char> buffer, CancellationToken cance
         }
 
         using PooledArray<byte> byteBuffer = new(4096);
-        var encoder = Encoding.UTF8.GetEncoder();
+        var encoder = (byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8).GetEncoder();
         var remainingChars = buffer.Span;
         var totalBytes = 0;
         while (!remainingChars.IsEmpty)

src/Lua/Internal/StringHelper.cs

@@ -379,6 +379,66 @@ public static Regex ToRegex(ReadOnlySpan<char> pattern)
         return new(builder.ToString());
     }
 
+    /// <summary>
+    /// toupper in the C locale: only 'a'-'z' change, every other char (including bytes 0x80-0xFF) is kept.
+    /// </summary>
+    public static string ToUpperAscii(string s)
+    {
+        var i = 0;
+        while (i < s.Length && !(s[i] >= 'a' && s[i] <= 'z'))
+        {
+            i++;
+        }
+
+        if (i == s.Length)
+        {
+            return s;
+        }
+
+        return string.Create(
+            s.Length,
+            s,
+            static (span, source) =>
+            {
+                for (var k = 0; k < span.Length; k++)
+                {
+                    var c = source[k];
+                    span[k] = c >= 'a' && c <= 'z' ? (char)(c - 32) : c;
+                }
+            }
+        );
+    }
+
+    /// <summary>
+    /// tolower in the C locale: only 'A'-'Z' change, every other char (including bytes 0x80-0xFF) is kept.
+    /// </summary>
+    public static string ToLowerAscii(string s)
+    {
+        var i = 0;
+        while (i < s.Length && !(s[i] >= 'A' && s[i] <= 'Z'))
+        {
+            i++;
+        }
+
+        if (i == s.Length)
+        {
+            return s;
+        }
+
+        return string.Create(
+            s.Length,
+            s,
+            static (span, source) =>
+            {
+                for (var k = 0; k < span.Length; k++)
+                {
+                    var c = source[k];
+                    span[k] = c >= 'A' && c <= 'Z' ? (char)(c + 32) : c;
+                }
+            }
+        );
+    }
+
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     public static bool IsNumber(char c)
     {