diff --git a/README.md b/README.md index 5909f922..9526ddf0 100644 --- a/README.md +++ b/README.md @@ -684,6 +684,26 @@ print(l) All string library functions handle strings as UTF-16. +#### Byte strings (opt-in) + +If you need byte-exact strings like the reference C implementation (for example `#"ế"` is `3`, `string.sub` cuts inside a UTF-8 sequence, and files are read and written as raw bytes), enable `LuaPlatform.ByteStrings`. `LuaPlatform.DefaultByteStrings` is the default platform with byte strings and byte-oriented file and standard I/O: + +```cs +using Lua; +using Lua.Platforms; +using Lua.Standard; + +var state = LuaState.Create(LuaPlatform.DefaultByteStrings); +state.OpenStandardLibraries(); + +// Every char of a Lua string holds one byte, so convert .NET strings at the boundary. +state.Environment["name"] = LuaByteString.FromString("Tiến Lên"); // UTF-8 bytes, #name == 11 +var results = await state.DoStringAsync("return #name, name:upper()"); +var upper = LuaByteString.ToString(results[1].Read()); // back to a .NET string +``` + +In this mode `string.upper`, `string.lower` and the pattern classes (`%a`, `%w`, `%l`, ...) follow the C locale (ASCII only), `string.char` rejects values above 255, and UTF-8 source files are loaded byte by byte (a UTF-8 BOM is skipped). The default stays UTF-16. + ### Garbage Collection Since Lua-CSharp is implemented in C#, it relies on .NET's garbage collector. As a result, memory management behavior differs from regular Lua. diff --git a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs index 074c1caa..6cd18cfd 100644 --- a/src/Lua/CodeAnalysis/Compilation/BomUtility.cs +++ b/src/Lua/CodeAnalysis/Compilation/BomUtility.cs @@ -12,6 +12,11 @@ static class BomUtility static ReadOnlySpan BomUtf32Little => [0xFF, 0xFE, 0x00, 0x00]; + public static ReadOnlySpan SkipUtf8Bom(ReadOnlySpan text) + { + return text.StartsWith(BomUtf8) ? text.Slice(BomUtf8.Length) : text; + } + /// /// Removes the BOM from the beginning of the text and returns the encoding. /// Supported encodings are UTF-8, UTF-16 (little and big endian), and UTF-32 (little endian). diff --git a/src/Lua/IO/ConsoleStandardIO.cs b/src/Lua/IO/ConsoleStandardIO.cs index 8d532ecb..91cff485 100644 --- a/src/Lua/IO/ConsoleStandardIO.cs +++ b/src/Lua/IO/ConsoleStandardIO.cs @@ -5,7 +5,11 @@ namespace Lua.IO; /// /// Default implementation of ILuaStandardIO using Console /// -public sealed class ConsoleStandardIO : ILuaStandardIO +/// +/// When true, standard input and output are read and written as raw bytes (one byte per char) +/// instead of UTF-8 text. Use together with . +/// +public sealed class ConsoleStandardIO(bool byteStrings = false) : ILuaStandardIO { ILuaStream? standardInput; ILuaStream? standardOutput; @@ -13,16 +17,28 @@ public sealed class ConsoleStandardIO : ILuaStandardIO public ILuaStream Input => standardInput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardInput(), LuaFileOpenMode.Read) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardInput(), + LuaFileOpenMode.Read, + byteStrings + ) ); public ILuaStream Output => standardOutput ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardOutput(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardOutput(), + LuaFileOpenMode.Write, + byteStrings + ) ); public ILuaStream Error => standardError ??= new StandardIOStream( - ILuaStream.CreateFromStream(ConsoleHelper.OpenStandardError(), LuaFileOpenMode.Write) + ILuaStream.CreateFromStream( + ConsoleHelper.OpenStandardError(), + LuaFileOpenMode.Write, + byteStrings + ) ); } diff --git a/src/Lua/IO/FileSystem.cs b/src/Lua/IO/FileSystem.cs index 161d3c8b..1539d95b 100644 --- a/src/Lua/IO/FileSystem.cs +++ b/src/Lua/IO/FileSystem.cs @@ -1,6 +1,12 @@ namespace Lua.IO; -public sealed class FileSystem(string? baseDirectory = null) : ILuaFileSystem +/// Directory that relative paths are resolved against. +/// +/// When true, files are read and written as raw bytes (one byte per char) instead of UTF-8 text. +/// Use together with . +/// +public sealed class FileSystem(string? baseDirectory = null, bool byteStrings = false) + : ILuaFileSystem { public string BaseDirectory => baseDirectory ?? Directory.GetCurrentDirectory(); @@ -32,9 +38,15 @@ public string GetFullPath(string path) return Path.Combine(baseDirectory, path); } + // Paths come from Lua as byte strings in byte mode: decode the UTF-8 bytes into the real file name. + string ResolvePath(string path) + { + return GetFullPath(byteStrings ? LuaByteString.ToString(path) : path); + } + public bool IsReadable(string path) { - return File.Exists(GetFullPath(path)); + return File.Exists(ResolvePath(path)); } public ValueTask Open( @@ -45,7 +57,7 @@ CancellationToken cancellationToken { var (mode, access) = GetFileMode(openMode); Stream stream; - path = GetFullPath(path); + path = ResolvePath(path); if (openMode == LuaFileOpenMode.AppendUpdate) { stream = File.Open( @@ -60,7 +72,7 @@ CancellationToken cancellationToken stream = File.Open(path, mode, access, FileShare.ReadWrite | FileShare.Delete); } - ILuaStream wrapper = new LuaStream(openMode, stream); + ILuaStream wrapper = new LuaStream(openMode, stream, byteStrings); if (openMode == LuaFileOpenMode.AppendUpdate) { @@ -72,8 +84,8 @@ CancellationToken cancellationToken public ValueTask Rename(string oldName, string newName, CancellationToken cancellationToken) { - oldName = GetFullPath(oldName); - newName = GetFullPath(newName); + oldName = ResolvePath(oldName); + newName = ResolvePath(newName); if (oldName == newName) { return default; @@ -91,7 +103,7 @@ public ValueTask Rename(string oldName, string newName, CancellationToken cancel public ValueTask Remove(string path, CancellationToken cancellationToken) { - path = GetFullPath(path); + path = ResolvePath(path); if (!File.Exists(path)) { throw new FileNotFoundException("No such file or directory", path); @@ -107,7 +119,8 @@ public ValueTask Remove(string path, CancellationToken cancellationToken) public string GetTempFileName() { - return Path.GetTempFileName(); + var name = Path.GetTempFileName(); + return byteStrings ? LuaByteString.FromString(name) : name; } public ValueTask OpenTempFileStream(CancellationToken cancellationToken) diff --git a/src/Lua/IO/ILuaStream.cs b/src/Lua/IO/ILuaStream.cs index ed6ab400..63341090 100644 --- a/src/Lua/IO/ILuaStream.cs +++ b/src/Lua/IO/ILuaStream.cs @@ -74,6 +74,11 @@ static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode) return new LuaStream(openMode, stream); } + static ILuaStream CreateFromStream(Stream stream, LuaFileOpenMode openMode, bool byteStrings) + { + return new LuaStream(openMode, stream, byteStrings); + } + static ILuaStream CreateFromString(string content) { return new StringStream(content); diff --git a/src/Lua/IO/LuaByteEncoding.cs b/src/Lua/IO/LuaByteEncoding.cs new file mode 100644 index 00000000..bdc03d83 --- /dev/null +++ b/src/Lua/IO/LuaByteEncoding.cs @@ -0,0 +1,88 @@ +using System.Text; + +namespace Lua.IO; + +/// +/// Maps one byte to one (0x00-0xFF) and back, so that a Lua string can carry +/// arbitrary bytes (for example UTF-8 text) with the same #s, string.sub and +/// string.byte results as the reference C implementation. +/// Characters above 0xFF cannot be represented and are written as '?'. +/// +sealed class LuaByteEncoding : Encoding +{ + public static LuaByteEncoding Instance { get; } = new(); + + public override int GetByteCount(char[] chars, int index, int count) => count; + + public override int GetByteCount(string s) => s.Length; + + public override int GetByteCount(ReadOnlySpan chars) => chars.Length; + + public override int GetBytes( + char[] chars, + int charIndex, + int charCount, + byte[] bytes, + int byteIndex + ) => GetBytes(chars.AsSpan(charIndex, charCount), bytes.AsSpan(byteIndex)); + + public override int GetBytes(ReadOnlySpan chars, Span bytes) + { + if (bytes.Length < chars.Length) + { + throw new ArgumentException("Destination is too small.", nameof(bytes)); + } + + for (var i = 0; i < chars.Length; i++) + { + var c = chars[i]; + bytes[i] = c <= 0xFF ? (byte)c : (byte)'?'; + } + + return chars.Length; + } + + public override int GetCharCount(byte[] bytes, int index, int count) => count; + + public override int GetCharCount(ReadOnlySpan bytes) => bytes.Length; + + public override int GetChars( + byte[] bytes, + int byteIndex, + int byteCount, + char[] chars, + int charIndex + ) => GetChars(bytes.AsSpan(byteIndex, byteCount), chars.AsSpan(charIndex)); + + public override int GetChars(ReadOnlySpan bytes, Span chars) + { + if (chars.Length < bytes.Length) + { + throw new ArgumentException("Destination is too small.", nameof(chars)); + } + + for (var i = 0; i < bytes.Length; i++) + { + chars[i] = (char)bytes[i]; + } + + return bytes.Length; + } + + public override string GetString(byte[] bytes, int index, int count) => + string.Create( + count, + (bytes, index), + static (span, state) => + { + for (var i = 0; i < span.Length; i++) + { + span[i] = (char)state.bytes[state.index + i]; + } + } + ); + + public override int GetMaxByteCount(int charCount) => charCount; + + public override int GetMaxCharCount(int byteCount) => byteCount; +} diff --git a/src/Lua/IO/LuaStream.cs b/src/Lua/IO/LuaStream.cs index a7c77369..80f82a4d 100644 --- a/src/Lua/IO/LuaStream.cs +++ b/src/Lua/IO/LuaStream.cs @@ -4,7 +4,9 @@ namespace Lua.IO; -public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaStream, ILuaByteStream +public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream, bool byteStrings = false) + : ILuaStream, + ILuaByteStream { Utf8Reader? reader; ulong flushSize = ulong.MaxValue; @@ -19,14 +21,14 @@ public sealed class LuaStream(LuaFileOpenMode mode, Stream innerStream) : ILuaSt public ValueTask ReadLineAsync(bool keepEol, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); return new(reader.ReadLine(innerStream, keepEol)); } public ValueTask ReadAllAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); var text = reader.ReadToEnd(innerStream); return new(text); } @@ -57,7 +59,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadAsync(int count, CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); if (count == 0) { return new(reader.IsEndOfStream(innerStream) ? null : string.Empty); @@ -69,7 +71,7 @@ public ValueTask ReadByteAsync(CancellationToken cancellationToken) public ValueTask ReadNumberAsync(CancellationToken cancellationToken) { mode.ThrowIfNotReadable(); - reader ??= new(); + reader ??= new(byteStrings); // Use the Utf8Reader's ReadNumber method which handles positioning correctly var numberStr = reader.ReadNumber(innerStream); @@ -93,7 +95,7 @@ public ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cance } using PooledArray byteBuffer = new(4096); - var encoder = Encoding.UTF8.GetEncoder(); + var encoder = (byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8).GetEncoder(); var remainingChars = buffer.Span; var totalBytes = 0; while (!remainingChars.IsEmpty) diff --git a/src/Lua/Internal/StringHelper.cs b/src/Lua/Internal/StringHelper.cs index 25fe5deb..bf543bf9 100644 --- a/src/Lua/Internal/StringHelper.cs +++ b/src/Lua/Internal/StringHelper.cs @@ -379,6 +379,66 @@ public static Regex ToRegex(ReadOnlySpan pattern) return new(builder.ToString()); } + /// + /// toupper in the C locale: only 'a'-'z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToUpperAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'a' && s[i] <= 'z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'a' && c <= 'z' ? (char)(c - 32) : c; + } + } + ); + } + + /// + /// tolower in the C locale: only 'A'-'Z' change, every other char (including bytes 0x80-0xFF) is kept. + /// + public static string ToLowerAscii(string s) + { + var i = 0; + while (i < s.Length && !(s[i] >= 'A' && s[i] <= 'Z')) + { + i++; + } + + if (i == s.Length) + { + return s; + } + + return string.Create( + s.Length, + s, + static (span, source) => + { + for (var k = 0; k < span.Length; k++) + { + var c = source[k]; + span[k] = c >= 'A' && c <= 'Z' ? (char)(c + 32) : c; + } + } + ); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsNumber(char c) { diff --git a/src/Lua/Internal/Utf8Reader.cs b/src/Lua/Internal/Utf8Reader.cs index 6c23e0e2..e6a4cc26 100644 --- a/src/Lua/Internal/Utf8Reader.cs +++ b/src/Lua/Internal/Utf8Reader.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Text; using Lua.CodeAnalysis.Compilation; +using Lua.IO; namespace Lua.Internal; @@ -13,14 +14,23 @@ sealed class Utf8Reader internal static bool scratchBufferUsed; readonly byte[] buffer; + readonly Encoding encoding; + readonly bool byteStrings; int bufPos, bufLen; Decoder? decoder; const int ThreadStaticBufferSize = 1024; - public Utf8Reader() + /// + /// When true, every byte is decoded to one char (0x00-0xFF) and a UTF-8 BOM is not skipped, + /// like the reference C implementation. Otherwise the stream is decoded as UTF-8. + /// + public Utf8Reader(bool byteStrings = false) { + this.byteStrings = byteStrings; + encoding = byteStrings ? LuaByteEncoding.Instance : Encoding.UTF8; + if (scratchBufferUsed) { buffer = new byte[ThreadStaticBufferSize]; @@ -46,7 +56,7 @@ public Utf8Reader() if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -92,7 +102,7 @@ ref lineLen } } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } else { @@ -107,7 +117,7 @@ ref lineLen return null; } - return Encoding.UTF8.GetString(resultBuffer, 0, lineLen); + return encoding.GetString(resultBuffer, 0, lineLen); } finally { @@ -126,7 +136,7 @@ public string ReadToEnd(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -143,7 +153,7 @@ public string ReadToEnd(Stream stream) return ""; } - return Encoding.UTF8.GetString(resultBuffer, 0, len); + return encoding.GetString(resultBuffer, 0, len); } finally { @@ -209,7 +219,7 @@ public byte ReadByte(Stream stream) if (bufPos >= bufLen) { bufLen = stream.Read(buffer, 0, buffer.Length); - bufPos = buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; + bufPos = !byteStrings && buffer.AsSpan().StartsWith(BomUtility.BomUtf8) ? 3 : 0; if (bufLen <= bufPos) { break; // EOF @@ -218,7 +228,7 @@ public byte ReadByte(Stream stream) ReadOnlySpan byteSpan = new(buffer, bufPos, bufLen - bufPos); Span charSpan = new(resultBuffer, len, charCount - len); - decoder ??= Encoding.UTF8.GetDecoder(); + decoder ??= encoding.GetDecoder(); decoder.Convert( byteSpan, charSpan, @@ -269,6 +279,8 @@ static void AppendToBuffer(ref byte[] buffer, ReadOnlySpan segment, ref in length += segment.Length; } + static bool IsAsciiSpace(char c) => c == ' ' || (c >= '\t' && c <= '\r'); + public void Clear() { bufPos = 0; @@ -296,7 +308,7 @@ public void Clear() } var c = (char)b; - if (!char.IsWhiteSpace(c)) + if (byteStrings ? !IsAsciiSpace(c) : !char.IsWhiteSpace(c)) { break; } diff --git a/src/Lua/LuaByteString.cs b/src/Lua/LuaByteString.cs new file mode 100644 index 00000000..5f238aae --- /dev/null +++ b/src/Lua/LuaByteString.cs @@ -0,0 +1,44 @@ +using System.Text; +using Lua.IO; + +namespace Lua; + +/// +/// Conversions between .NET strings and Lua byte strings (see +/// ), where every holds one byte. +/// +public static class LuaByteString +{ + /// + /// Converts raw bytes (for example UTF-8 text) to a byte string. + /// + public static string FromBytes(ReadOnlySpan bytes) + { + return LuaByteEncoding.Instance.GetString(bytes); + } + + /// + /// Converts a byte string back to raw bytes. Chars above 0xFF are written as '?'. + /// + public static byte[] ToBytes(string byteString) + { + return LuaByteEncoding.Instance.GetBytes(byteString); + } + + /// + /// Converts a .NET string to a byte string holding its UTF-8 bytes. + /// + public static string FromString(string value) + { + return FromBytes(Encoding.UTF8.GetBytes(value)); + } + + /// + /// Decodes the UTF-8 bytes held by a byte string into a .NET string. + /// Invalid UTF-8 is replaced with U+FFFD. + /// + public static string ToString(string byteString) + { + return Encoding.UTF8.GetString(ToBytes(byteString)); + } +} diff --git a/src/Lua/LuaState.cs b/src/Lua/LuaState.cs index 290ba387..d43c6dc9 100644 --- a/src/Lua/LuaState.cs +++ b/src/Lua/LuaState.cs @@ -1,8 +1,10 @@ using System.Buffers; using System.Runtime.CompilerServices; +using System.Text; using Lua.CodeAnalysis.Compilation; using Lua.Internal; using Lua.Internal.CompilerServices; +using Lua.IO; using Lua.Platforms; using Lua.Runtime; @@ -637,7 +639,17 @@ static bool AllowsMode(string mode, char chunkMode) throw new Exception("attempt to load a text chunk (mode is 'b')"); } - chunk = BomUtility.GetEncodingFromBytes(chunk, out var encoding); + Encoding encoding; + if (Platform.ByteStrings) + { + // Byte strings: like luaL_loadfile, skip a UTF-8 BOM and keep every other byte as one char. + chunk = BomUtility.SkipUtf8Bom(chunk); + encoding = LuaByteEncoding.Instance; + } + else + { + chunk = BomUtility.GetEncodingFromBytes(chunk, out encoding); + } var charCount = encoding.GetCharCount(chunk); var pooled = ArrayPool.Shared.Rent(charCount); diff --git a/src/Lua/Platforms/LuaPlatform.cs b/src/Lua/Platforms/LuaPlatform.cs index 009082cb..70befa6c 100644 --- a/src/Lua/Platforms/LuaPlatform.cs +++ b/src/Lua/Platforms/LuaPlatform.cs @@ -16,6 +16,24 @@ public record LuaPlatform( TimeProvider TimeProvider ) { + /// + /// When true, Lua strings are byte strings: every holds one byte (0x00-0xFF), + /// so #s, string.len, string.sub and string.byte count bytes exactly + /// like the reference C implementation (UTF-8 text such as "ế" has length 3 instead of 1). + /// Character classes (%a, %w, ...), string.upper and string.lower follow the + /// C locale (ASCII only) and string.char rejects values above 255. + /// The and of the platform must read and write + /// bytes as well, which does. The default is false (UTF-16 strings). + /// Use to convert between .NET strings and byte strings. + /// + public bool ByteStrings { get; init; } + + // Converts a .NET string (file name, environment value, error text) into a Lua string for this platform. + internal string ToLuaString(string value) + { + return ByteStrings ? LuaByteString.FromString(value) : value; + } + /// /// Standard console platform implementation. /// Uses real file system, console I/O, and system operations. @@ -27,4 +45,19 @@ TimeProvider TimeProvider StandardIO: new ConsoleStandardIO(), TimeProvider: TimeProvider.System ); + + /// + /// Same as , but with enabled: files and standard I/O + /// are read and written as raw bytes. + /// + public static LuaPlatform DefaultByteStrings { get; } = + new( + FileSystem: new FileSystem(byteStrings: true), + OsEnvironment: new SystemOsEnvironment(), + StandardIO: new ConsoleStandardIO(byteStrings: true), + TimeProvider: TimeProvider.System + ) + { + ByteStrings = true, + }; } diff --git a/src/Lua/Standard/IOLibrary.cs b/src/Lua/Standard/IOLibrary.cs index c8ed4f3e..04ec0ecd 100644 --- a/src/Lua/Standard/IOLibrary.cs +++ b/src/Lua/Standard/IOLibrary.cs @@ -44,7 +44,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -62,7 +66,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -213,7 +221,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } diff --git a/src/Lua/Standard/Internal/MatchState.cs b/src/Lua/Standard/Internal/MatchState.cs index 3dc20f82..a56e3bf7 100644 --- a/src/Lua/Standard/Internal/MatchState.cs +++ b/src/Lua/Standard/Internal/MatchState.cs @@ -17,6 +17,9 @@ internal struct Capture public bool IsPosition => Len == CapPosition; } + // Byte strings use the C locale: classes only match ASCII, bytes 0x80-0xFF never belong to a class. + readonly bool byteStrings = state.Platform.ByteStrings; + public readonly LuaState State = state; public readonly string Source = source; public readonly string Pattern = pattern; @@ -444,8 +447,13 @@ int ClassEnd(ReadOnlySpan pattern, int pIdx) } } - static bool MatchClass(char c, char cl) + bool MatchClass(char c, char cl) { + if (byteStrings) + { + return MatchClassAscii(c, cl); + } + bool res; switch (char.ToLower(cl)) { @@ -495,12 +503,62 @@ static bool MatchClass(char c, char cl) return char.IsLower(cl) ? res : !res; } + static bool MatchClassAscii(char c, char cl) + { + bool res; + switch (char.ToLowerInvariant(cl)) + { + case 'a': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); + break; + case 'c': + res = c < ' ' || c == 127; + break; + case 'd': + res = c >= '0' && c <= '9'; + break; + case 'g': + res = c > ' ' && c < 127; + break; + case 'l': + res = c >= 'a' && c <= 'z'; + break; + case 'p': + res = + c + is (>= '!' and <= '/') + or (>= ':' and <= '@') + or (>= '[' and <= '`') + or (>= '{' and <= '~'); + break; + case 's': + res = c == ' ' || (c >= '\t' && c <= '\r'); + break; + case 'u': + res = c >= 'A' && c <= 'Z'; + break; + case 'w': + res = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + break; + case 'x': + res = IsHexDigit(c); + break; + case 'z': + res = c == '\0'; + break; + default: + return cl == c; + } + + return cl is >= 'a' and <= 'z' ? res : !res; + } + static bool IsHexDigit(char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } - static bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) + bool MatchBracketClass(char c, ReadOnlySpan pattern, int pIdx, int ec) { var sig = true; if (pIdx + 1 < pattern.Length && pattern[pIdx + 1] == '^') diff --git a/src/Lua/Standard/OperatingSystemLibrary.cs b/src/Lua/Standard/OperatingSystemLibrary.cs index 318c4542..3d1870ab 100644 --- a/src/Lua/Standard/OperatingSystemLibrary.cs +++ b/src/Lua/Standard/OperatingSystemLibrary.cs @@ -156,10 +156,15 @@ CancellationToken cancellationToken ) { var variable = context.GetArgument(0); + var byteStrings = context.GlobalState.Platform.ByteStrings; + var value = context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable( + byteStrings ? LuaByteString.ToString(variable) : variable + ); return new( context.Return( - context.GlobalState.Platform.OsEnvironment.GetEnvironmentVariable(variable) - ?? LuaValue.Nil + value is not null && byteStrings ? LuaByteString.FromString(value) + : value is not null ? value + : LuaValue.Nil ) ); } @@ -177,7 +182,11 @@ CancellationToken cancellationToken } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } @@ -199,7 +208,11 @@ await context.GlobalState.Platform.FileSystem.Rename( } catch (IOException ex) { - return context.Return(LuaValue.Nil, ex.Message, ex.HResult); + return context.Return( + LuaValue.Nil, + context.GlobalState.Platform.ToLuaString(ex.Message), + ex.HResult + ); } } diff --git a/src/Lua/Standard/StringLibrary.cs b/src/Lua/Standard/StringLibrary.cs index 53bf4aa6..099f9c63 100644 --- a/src/Lua/Standard/StringLibrary.cs +++ b/src/Lua/Standard/StringLibrary.cs @@ -69,11 +69,17 @@ CancellationToken cancellationToken return new(context.Return("")); } + var byteStrings = context.State.Platform.ByteStrings; ValueStringBuilder builder = new(context.ArgumentCount); for (var i = 0; i < context.ArgumentCount; i++) { var arg = context.GetArgument(i); LuaRuntimeException.ThrowBadArgumentIfNumberIsNotInteger(context.State, i + 1, arg); + if (byteStrings && (arg < 0 || arg > 255)) + { + LuaRuntimeException.BadArgument(context.State, i + 1, "value out of range"); + } + builder.Append((char)arg); } @@ -412,7 +418,11 @@ context with } break; case 'c': - formattedValue = ((char)(int)x).ToString(); + formattedValue = ( + context.State.Platform.ByteStrings + ? (char)((int)x & 0xFF) + : (char)(int)x + ).ToString(); break; case 'x': { @@ -827,7 +837,11 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToLower())); + return new( + context.Return( + context.State.Platform.ByteStrings ? StringHelper.ToLowerAscii(s) : s.ToLower() + ) + ); } public ValueTask Match( @@ -1021,6 +1035,10 @@ CancellationToken cancellationToken ) { var s = context.GetArgument(0); - return new(context.Return(s.ToUpper())); + return new( + context.Return( + context.State.Platform.ByteStrings ? StringHelper.ToUpperAscii(s) : s.ToUpper() + ) + ); } } diff --git a/tests/Lua.Tests/ByteStringTests.cs b/tests/Lua.Tests/ByteStringTests.cs new file mode 100644 index 00000000..2ceb825e --- /dev/null +++ b/tests/Lua.Tests/ByteStringTests.cs @@ -0,0 +1,564 @@ +using System.Text; +using Lua.IO; +using Lua.Platforms; +using Lua.Standard; + +namespace Lua.Tests; + +// Expected values in this file were produced by the reference Lua 5.2.4 interpreter. +public sealed class ByteStringTests : IDisposable +{ + // VI is "Tien Len" with accents (Vietnamese) as UTF-8 bytes; the sources use \x escapes so this file stays ASCII. + const string Prelude = """ + local VI = "Ti\xE1\xBA\xBFn L\xC3\xAAn" + local E = "\xE1\xBA\xBF" + local function hex(s) return (s:gsub(".", function(c) return string.format("%02x", c:byte()) end)) end + + """; + + readonly string testDirectory = Path.Combine( + Path.GetTempPath(), + $"LuaByteStringTests_{Guid.NewGuid()}" + ); + + public ByteStringTests() + { + Directory.CreateDirectory(testDirectory); + } + + public void Dispose() + { + if (Directory.Exists(testDirectory)) + { + Directory.Delete(testDirectory, true); + } + } + + LuaState CreateByteState() + { + var state = LuaState.Create( + LuaPlatform.DefaultByteStrings with + { + FileSystem = new FileSystem(testDirectory, byteStrings: true), + } + ); + state.OpenStandardLibraries(); + return state; + } + + async Task RunAsync(string body) + { + using var state = CreateByteState(); + await state.DoStringAsync(Prelude + body); + } + + [Test] + public void DefaultPlatform_KeepsUtf16Strings() + { + Assert.That(LuaPlatform.Default.ByteStrings, Is.False); + Assert.That(LuaPlatform.DefaultByteStrings.ByteStrings, Is.True); + } + + [Test] + public async Task DefaultMode_StringLenCountsUtf16Chars() + { + var state = LuaState.Create(); + state.OpenStandardLibraries(); + var result = await state.DoStringAsync( + "return string.len('\u3042\u3044\u3046\u3048\u304A'), #'\u1EBF', ('\u1EBF'):find('%a')" + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(5))); + Assert.That(result[1], Is.EqualTo(new LuaValue(1))); + Assert.That(result[2], Is.EqualTo(new LuaValue(1))); + } + + [Test] + public Task Length_Byte_Char_Sub() => + RunAsync( + """ + assert(#VI == 11 and VI:len() == 11) + assert(#E == 3) + assert(#"\xE3\x81\x82\xE3\x81\x84\xE3\x81\x86\xE3\x81\x88\xE3\x81\x8A" == 15) + assert(#"\xF0\x9F\x98\x80" == 4) + assert(select("#", E:byte(1, -1)) == 3) + local a, b, c = E:byte(1, -1) + assert(a == 225 and b == 186 and c == 191) + assert(string.char(225, 186, 191) == E) + assert(hex(string.char(255, 0, 128)) == "ff0080") + assert(hex(VI:sub(3, 4)) == "e1ba") + assert(hex(VI:sub(-3)) == "c3aa6e") + assert(hex(E:sub(1, 1)) == "e1" and hex(E:sub(2)) == "babf") + assert(hex(E:reverse()) == "bfbae1") + assert(hex(E:rep(2, "|")) == "e1babf7ce1babf") + assert(hex(E .. "-" .. E) == "e1babf2de1babf") + """ + ); + + [Test] + public Task StringChar_RejectsValuesAboveByteRange() => + RunAsync( + """ + assert(not pcall(string.char, 256)) + assert(not pcall(string.char, -1)) + assert(not pcall(string.char, 65 + 256)) + local ok, err = pcall(string.char, 256) + assert(err:find("value out of range", 1, true)) + assert(hex(string.format("%c", 256 + 65)) == "41") + assert(hex(string.format("%c%c%c", 225, 186, 191)) == "e1babf") + """ + ); + + [Test] + public Task UpperLower_OnlyChangeAscii() => + RunAsync( + """ + assert(hex(("a\xC3\xA0b\xE1\xBA\xBF"):upper()) == "41c3a042e1babf") + assert(hex(("A\xC3\x80B\xE1\xBA\xBE"):lower()) == "61c38062e1babe") + local all = {} + for c = 0, 255 do all[#all + 1] = string.char(c) end + all = table.concat(all) + local up, low = all:upper(), all:lower() + assert(#up == 256 and #low == 256) + for c = 0, 255 do + local ch = string.char(c) + local wantUp = (c >= 97 and c <= 122) and c - 32 or c + local wantLow = (c >= 65 and c <= 90) and c + 32 or c + assert(up:byte(c + 1) == wantUp, "upper " .. c) + assert(low:byte(c + 1) == wantLow, "lower " .. c) + end + """ + ); + + // C locale ctype for every byte, written independently of the implementation. + static bool CType(char cls, int c) + { + var upper = c >= 'A' && c <= 'Z'; + var lower = c >= 'a' && c <= 'z'; + var digit = c >= '0' && c <= '9'; + return cls switch + { + 'a' => upper || lower, + 'c' => c < 32 || c == 127, + 'd' => digit, + 'g' => c > 32 && c < 127, + 'l' => lower, + 'p' => c > 32 && c < 127 && !(upper || lower || digit), + 's' => c == 32 || (c >= 9 && c <= 13), + 'u' => upper, + 'w' => upper || lower || digit, + 'x' => digit || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'), + _ => throw new ArgumentOutOfRangeException(nameof(cls)), + }; + } + + [TestCase('a')] + [TestCase('c')] + [TestCase('d')] + [TestCase('g')] + [TestCase('l')] + [TestCase('p')] + [TestCase('s')] + [TestCase('u')] + [TestCase('w')] + [TestCase('x')] + public async Task CharacterClasses_FollowCLocaleForEveryByte(char cls) + { + var expected = new StringBuilder(); + var expectedComplement = new StringBuilder(); + for (var c = 0; c < 256; c++) + { + expected.Append(CType(cls, c) ? '1' : '0'); + expectedComplement.Append(CType(cls, c) ? '0' : '1'); + } + + var upperCls = char.ToUpperInvariant(cls); + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + local function bits(pattern) + local t = {} + for c = 0, 255 do t[#t + 1] = string.char(c):find(pattern) and "1" or "0" end + return table.concat(t) + end + return bits("%{{cls}}"), bits("%{{upperCls}}"), bits("[%{{cls}}]"), bits("[^%{{cls}}]") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[1].Read(), Is.EqualTo(expectedComplement.ToString())); + Assert.That(result[2].Read(), Is.EqualTo(expected.ToString())); + Assert.That(result[3].Read(), Is.EqualTo(expectedComplement.ToString())); + } + + [Test] + public Task Patterns_WorkOnBytes() => + RunAsync( + """ + local s, n = VI:gsub("%a+", "X") + assert(hex(s) == hex("X\xE1\xBA\xBFX X\xC3\xAAX") and n == 4) + assert(select(2, VI:gsub("%a+", "X")) == 4) + assert(E:find("%a") == nil) + assert(VI:find(E, 1, true) == 3) + assert(VI:find("L") == 8) + assert(VI:find("n", 7) == 11) + local c1, c2 = VI:match("^(.)(.)") + assert(hex(c1) == "54" and hex(c2) == "69") + assert(VI:match("()L") == 8) + assert(hex((E:gsub(".", "<%0>"))) == "3ce13e3cba3e3cbf3e") + local words = {} + for w in VI:gmatch("%S+") do words[#words + 1] = hex(w) end + assert(table.concat(words, ",") == "5469e1babf6e,4cc3aa6e") + local bytes = {} + for ch in E:gmatch(".") do bytes[#bytes + 1] = hex(ch) end + assert(table.concat(bytes, ",") == "e1,ba,bf") + assert(VI:gsub("%f[%a]%a+", "<%0>") == "\xE1\xBA\xBF \xC3\xAA") + assert(("(\xE1\xBA\xBF (x)) y"):match("%b()") == "(\xE1\xBA\xBF (x))") + """ + ); + + [Test] + public Task Format_PadsByBytes() => + RunAsync( + """ + assert(hex(string.format("%5s|%-5s|", E, E)) == "2020e1babf7c" .. "e1babf20207c") + assert(hex(string.format("%.2s", E)) == "e1ba") + assert(#string.format("%s", VI) == 11) + """ + ); + + [Test] + public Task Comparison_IsByteWise() => + RunAsync( + """ + assert("\x80" > "\x7F" and "\xFF" > "\x01" and "\x7F" < "\x80") + local w = { "\xC3\xA9", "e", "z", "\xE1\xBA\xBF", "a", "\xC3\x80", "Z", "\xC3\xAA" } + table.sort(w) + local h = {} + for i, x in ipairs(w) do h[i] = hex(x) end + assert(table.concat(h, ",") == "5a,61,65,7a,c380,c3a9,c3aa,e1babf") + """ + ); + + [Test] + public async Task LoadString_KeepsSourceBytes() + { + using var state = CreateByteState(); + var source = LuaByteString.FromString("return 'Ti\u1EBFn', #'Ti\u1EBFn', '\u1EBF'"); + var result = await state.DoStringAsync(source); + + Assert.That(result[0].Read(), Is.EqualTo(LuaByteString.FromString("Ti\u1EBFn"))); + Assert.That(result[1], Is.EqualTo(new LuaValue(6))); + Assert.That( + LuaByteString.ToBytes(result[2].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + } + + [Test] + public async Task LoadBytes_SkipsOnlyUtf8Bom() + { + using var state = CreateByteState(); + var withBom = new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray(); + var result = await state.ExecuteAsync(state.Load(withBom, "bom", "t")); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task LoadFile_ReadsUtf8SourceAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "chunk.lua"), + new byte[] { 0xEF, 0xBB, 0xBF } + .Concat(Encoding.UTF8.GetBytes("return '\u1EBF', #'\u1EBF'")) + .ToArray() + ); + + using var state = CreateByteState(); + var closure = await state.LoadFileAsync("chunk.lua", "bt", null, CancellationToken.None); + var result = await state.ExecuteAsync(closure); + + Assert.That( + LuaByteString.ToBytes(result[0].Read()), + Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF }) + ); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + } + + [Test] + public async Task Require_LoadsUtf8ModuleAsBytes() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "viet.lua"), + Encoding.UTF8.GetBytes("return { s = 'Ti\u1EBFn L\u00EAn' }") + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{testDirectory.Replace("\\", "/")}}/?.lua" + local m = require("viet") + return #m.s + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(11))); + } + + [Test] + public async Task Io_WritesAndReadsRawBytes() + { + var data = new byte[] + { + 0x54, + 0x69, + 0xE1, + 0xBA, + 0xBF, + 0x6E, + 0x0A, + 0xE1, + 0xBA, + 0xBF, + 0xFF, + 0x00, + 0x80, + 0x7A, + 0x0D, + 0x0A, + 0x65, + 0x6E, + 0x64, + }; + using var state = CreateByteState(); + state.Environment["data"] = LuaByteString.FromBytes(data); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("raw.bin", "wb")) + f:write(data) + f:close() + f = assert(io.open("raw.bin", "rb")) + local size = f:seek("end") + f:seek("set", 0) + local all = f:read("*a") + f:close() + f = assert(io.open("raw.bin", "rb")) + local l1 = f:read("*l") + local n2 = f:read(2) + f:close() + return size, all == data, #l1, #n2 + """ + ); + + Assert.That(File.ReadAllBytes(Path.Combine(testDirectory, "raw.bin")), Is.EqualTo(data)); + Assert.That(result[0], Is.EqualTo(new LuaValue(data.Length))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(6))); + Assert.That(result[3], Is.EqualTo(new LuaValue(2))); + } + + [Test] + public async Task FileNames_AreDecodedFromUtf8Bytes() + { + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("t\xE1\xBA\xBFn.bin", "wb")) + f:write("x") + f:close() + local g = assert(io.open("r\xC3\xAA.bin", "wb")) + g:close() + local renamed = os.rename("r\xC3\xAA.bin", "r2\xC3\xAA.bin") + local gone = io.open("r\xC3\xAA.bin", "rb") == nil + local removed = os.remove("r2\xC3\xAA.bin") + local ok, msg = io.open("missing_\xE1\xBA\xBF.bin", "rb") + return renamed, gone, removed, ok == nil, type(msg) + """ + ); + + Assert.That(File.Exists(Path.Combine(testDirectory, "t\u1EBFn.bin")), Is.True); + Assert.That(File.Exists(Path.Combine(testDirectory, "r\u00EA.bin")), Is.False); + Assert.That(File.Exists(Path.Combine(testDirectory, "r2\u00EA.bin")), Is.False); + Assert.That(result[0], Is.EqualTo(new LuaValue(true))); + Assert.That(result[1], Is.EqualTo(new LuaValue(true))); + Assert.That(result[2], Is.EqualTo(new LuaValue(true))); + Assert.That(result[3], Is.EqualTo(new LuaValue(true))); + Assert.That(result[4].Read(), Is.EqualTo("string")); + } + + [Test] + public async Task Require_FindsModuleWithNonAsciiName() + { + await File.WriteAllTextAsync(Path.Combine(testDirectory, "m\u1EBFd.lua"), "return 'ok'"); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + $$""" + package.path = "{{LuaByteString.FromString(testDirectory.Replace("\\", "/"))}}/?.lua" + return require("m\xE1\xBA\xBFd") + """ + ); + + Assert.That(result[0].Read(), Is.EqualTo("ok")); + } + + [Test] + public async Task OsGetenv_ReturnsUtf8Bytes() + { + const string name = "LUA_CSHARP_BYTE_STRINGS_TEST"; + Environment.SetEnvironmentVariable(name, "Ti\u1EBFn"); + try + { + using var state = CreateByteState(); + var result = await state.DoStringAsync($"local v = os.getenv('{name}'); return #v, v"); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(LuaByteString.ToString(result[1].Read()), Is.EqualTo("Ti\u1EBFn")); + } + finally + { + Environment.SetEnvironmentVariable(name, null); + } + } + + [Test] + public async Task Io_DoesNotSkipBomOnRead() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "bom.bin"), + new byte[] { 0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63 } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("bom.bin", "rb")) + local all = f:read("*a") + f:close() + f = assert(io.open("bom.bin", "rb")) + local n3 = f:read(3) + local rest = f:read("*l") + f:close() + return #all, #n3, rest + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(6))); + Assert.That(result[1], Is.EqualTo(new LuaValue(3))); + Assert.That(result[2].Read(), Is.EqualTo("abc")); + } + + [Test] + public async Task Io_NumberReadingTreatsHighBytesAsNotWhitespace() + { + await File.WriteAllBytesAsync( + Path.Combine(testDirectory, "num.bin"), + new byte[] + { + 0x20, + 0x20, + 0x31, + 0x32, + 0x20, + 0x30, + 0x78, + 0x31, + 0x46, + 0x20, + 0xA0, + 0x37, + 0x0A, + } + ); + + using var state = CreateByteState(); + var result = await state.DoStringAsync( + """ + local f = assert(io.open("num.bin", "rb")) + local a, b, c = f:read("*n", "*n", "*n") + f:close() + return a, b, c + """ + ); + + Assert.That(result[0], Is.EqualTo(new LuaValue(12))); + Assert.That(result[1], Is.EqualTo(new LuaValue(31))); + Assert.That(result[2], Is.EqualTo(LuaValue.Nil)); + } + + [Test] + public async Task LuaStream_ByteMode_RoundTripsEveryByte() + { + var all = Enumerable.Range(0, 256).Select(i => (byte)i).ToArray(); + using var memory = new MemoryStream(); + using (var writer = new LuaStream(LuaFileOpenMode.Write, memory, byteStrings: true)) + { + await writer.WriteAsync( + LuaByteString.FromBytes(all).AsMemory(), + CancellationToken.None + ); + await writer.FlushAsync(CancellationToken.None); + Assert.That(memory.ToArray(), Is.EqualTo(all)); + } + + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(all), + byteStrings: true + ); + var text = await reader.ReadAllAsync(CancellationToken.None); + Assert.That(LuaByteString.ToBytes(text), Is.EqualTo(all)); + } + + [Test] + public async Task LuaStream_DefaultMode_StillEncodesUtf8() + { + using var memory = new MemoryStream(); + var writer = new LuaStream(LuaFileOpenMode.Write, memory); + await writer.WriteAsync("\u1EBF".AsMemory(), CancellationToken.None); + await writer.FlushAsync(CancellationToken.None); + + Assert.That(memory.ToArray(), Is.EqualTo(new byte[] { 0xE1, 0xBA, 0xBF })); + } + + [Test] + public async Task LuaStream_ByteMode_ReadCountCountsBytes() + { + using var reader = new LuaStream( + LuaFileOpenMode.Read, + new MemoryStream(new byte[] { 0xE1, 0xBA, 0xBF, 0x61 }), + byteStrings: true + ); + + var first = await reader.ReadAsync(2, CancellationToken.None); + var second = await reader.ReadAsync(2, CancellationToken.None); + + Assert.That(LuaByteString.ToBytes(first!), Is.EqualTo(new byte[] { 0xE1, 0xBA })); + Assert.That(LuaByteString.ToBytes(second!), Is.EqualTo(new byte[] { 0xBF, 0x61 })); + } + + [Test] + public void LuaByteString_ConvertsBetweenDotNetStringsAndBytes() + { + const string text = "Ti\u1EBFn L\u00EAn \U0001F600"; + var byteString = LuaByteString.FromString(text); + + Assert.That(byteString.Length, Is.EqualTo(Encoding.UTF8.GetByteCount(text))); + Assert.That(byteString.All(c => c <= 0xFF), Is.True); + Assert.That(LuaByteString.ToString(byteString), Is.EqualTo(text)); + Assert.That(LuaByteString.ToBytes(byteString), Is.EqualTo(Encoding.UTF8.GetBytes(text))); + Assert.That( + LuaByteString.FromBytes(new byte[] { 0xFF, 0x00, 0x80 }), + Is.EqualTo("\u00FF\0\u0080") + ); + Assert.That(LuaByteString.ToString("\u00FF"), Is.EqualTo("\uFFFD")); + Assert.That(LuaByteString.ToBytes("\u1EBF"), Is.EqualTo(new byte[] { (byte)'?' })); + } +}