From 1c4fdae630a41041869c27af84c4734b6c57533a Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 29 Oct 2026 18:00:16 +0000 Subject: [PATCH 1/2] feat(v18): accept the .wasm bytes or a compiled module in loadModule() Co-authored-by: Gldywn <14254051+Gldywn@users.noreply.github.com> --- LOADING_WASM.md | 12 +++- templates/full/index.ts | 43 ++++++++++++- versions/18/README.md | 12 ++++ versions/18/package.json | 2 +- versions/18/src/index.ts | 43 ++++++++++++- versions/18/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ 6 files changed, 202 insertions(+), 4 deletions(-) create mode 100644 versions/18/test/load-module.test.js diff --git a/LOADING_WASM.md b/LOADING_WASM.md index 52462af0..0b3680b2 100644 --- a/LOADING_WASM.md +++ b/LOADING_WASM.md @@ -19,4 +19,14 @@ Bundlers don’t emit `.wasm` files by default — they stay inside `node_module * **Next.js + Webpack:** Add a `.wasm` asset rule or use the copy method. * **Turbopack:** Only the copy method works for now. -* **Dev mode:** Optionally use a watcher script to auto-copy on rebuilds. \ No newline at end of file +* **Dev mode:** Optionally use a watcher script to auto-copy on rebuilds. + +#### 📦 Hosts that can't fetch the file + +If the runtime can't fetch or read the `.wasm` file at all (e.g. a custom protocol without fetch support), load the bytes yourself and pass them in: + +```ts +import { loadModule } from 'libpg-query'; + +await loadModule({ wasm: bytes }); // ArrayBuffer, typed array, or a compiled WebAssembly.Module +``` diff --git a/templates/full/index.ts b/templates/full/index.ts index 761a633d..879403dd 100644 --- a/templates/full/index.ts +++ b/templates/full/index.ts @@ -157,6 +157,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -172,11 +175,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: WasmModule = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -263,7 +275,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule(): Promise { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/18/README.md b/versions/18/README.md index be93bf2f..ebf01ee9 100644 --- a/versions/18/README.md +++ b/versions/18/README.md @@ -185,6 +185,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/18/package.json b/versions/18/package.json index a1537ea3..b5a33adc 100644 --- a/versions/18/package.json +++ b/versions/18/package.json @@ -31,7 +31,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js" + "test": "node --test test/parsing.test.js test/pg18.test.js test/fingerprint.test.js test/normalize.test.js test/plpgsql.test.js test/scan.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/18/src/index.ts b/versions/18/src/index.ts index 12303117..4d7bc0be 100644 --- a/versions/18/src/index.ts +++ b/versions/18/src/index.ts @@ -164,6 +164,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -179,11 +182,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: WasmModule = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -270,7 +282,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule(): Promise { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/18/test/load-module.test.js b/versions/18/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/18/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +}); From b226c8236c75352269eaa578ccfd0e035ece1fc8 Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Thu, 29 Oct 2026 18:14:42 +0000 Subject: [PATCH 2/2] feat(v13-v17): accept the .wasm bytes or a compiled module in loadModule() Co-authored-by: Gldywn <14254051+Gldywn@users.noreply.github.com> --- templates/index.ts | 43 ++++++++++++- versions/13/README.md | 12 ++++ versions/13/package.json | 2 +- versions/13/src/index.ts | 43 ++++++++++++- versions/13/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ versions/14/README.md | 12 ++++ versions/14/package.json | 2 +- versions/14/src/index.ts | 43 ++++++++++++- versions/14/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ versions/15/README.md | 12 ++++ versions/15/package.json | 2 +- versions/15/src/index.ts | 43 ++++++++++++- versions/15/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ versions/16/README.md | 12 ++++ versions/16/package.json | 2 +- versions/16/src/index.ts | 43 ++++++++++++- versions/16/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ versions/17/README.md | 12 ++++ versions/17/package.json | 2 +- versions/17/src/index.ts | 43 ++++++++++++- versions/17/test/load-module.test.js | 94 ++++++++++++++++++++++++++++ 21 files changed, 787 insertions(+), 11 deletions(-) create mode 100644 versions/13/test/load-module.test.js create mode 100644 versions/14/test/load-module.test.js create mode 100644 versions/15/test/load-module.test.js create mode 100644 versions/16/test/load-module.test.js create mode 100644 versions/17/test/load-module.test.js diff --git a/templates/index.ts b/templates/index.ts index 4b3120c5..4e4c16ce 100644 --- a/templates/index.ts +++ b/templates/index.ts @@ -142,6 +142,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -157,11 +160,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -248,7 +260,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/13/README.md b/versions/13/README.md index 18715b24..fb7e67d0 100644 --- a/versions/13/README.md +++ b/versions/13/README.md @@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/13/package.json b/versions/13/package.json index 9b39742d..a9a9ce7a 100644 --- a/versions/13/package.json +++ b/versions/13/package.json @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/13/src/index.ts b/versions/13/src/index.ts index d717d1e4..c1a6f807 100644 --- a/versions/13/src/index.ts +++ b/versions/13/src/index.ts @@ -149,6 +149,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -255,7 +267,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/13/test/load-module.test.js b/versions/13/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/13/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +}); diff --git a/versions/14/README.md b/versions/14/README.md index 18715b24..fb7e67d0 100644 --- a/versions/14/README.md +++ b/versions/14/README.md @@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/14/package.json b/versions/14/package.json index 082b9bbd..56f336a1 100644 --- a/versions/14/package.json +++ b/versions/14/package.json @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/14/src/index.ts b/versions/14/src/index.ts index d717d1e4..c1a6f807 100644 --- a/versions/14/src/index.ts +++ b/versions/14/src/index.ts @@ -149,6 +149,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -255,7 +267,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/14/test/load-module.test.js b/versions/14/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/14/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +}); diff --git a/versions/15/README.md b/versions/15/README.md index 18715b24..fb7e67d0 100644 --- a/versions/15/README.md +++ b/versions/15/README.md @@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/15/package.json b/versions/15/package.json index 09244a46..9f8c7cad 100644 --- a/versions/15/package.json +++ b/versions/15/package.json @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/15/src/index.ts b/versions/15/src/index.ts index d717d1e4..c1a6f807 100644 --- a/versions/15/src/index.ts +++ b/versions/15/src/index.ts @@ -149,6 +149,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -255,7 +267,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/15/test/load-module.test.js b/versions/15/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/15/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +}); diff --git a/versions/16/README.md b/versions/16/README.md index 18715b24..fb7e67d0 100644 --- a/versions/16/README.md +++ b/versions/16/README.md @@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/16/package.json b/versions/16/package.json index 863e3024..a3b50db1 100644 --- a/versions/16/package.json +++ b/versions/16/package.json @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/16/src/index.ts b/versions/16/src/index.ts index d717d1e4..c1a6f807 100644 --- a/versions/16/src/index.ts +++ b/versions/16/src/index.ts @@ -149,6 +149,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -255,7 +267,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/16/test/load-module.test.js b/versions/16/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/16/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +}); diff --git a/versions/17/README.md b/versions/17/README.md index a1346826..9558feef 100644 --- a/versions/17/README.md +++ b/versions/17/README.md @@ -134,6 +134,18 @@ const result = parseSync('SELECT * FROM users'); Note: We recommend using async methods as they handle initialization automatically. Use sync methods only when necessary, and always call `loadModule()` first. +### `loadModule({ wasm }): Promise` + +For hosts where the wrapper cannot read `libpg-query.wasm` itself (for example a custom protocol without fetch support), pass the file's bytes or a compiled `WebAssembly.Module`. It is compiled once, and every later instance is created from it. + +```typescript +import { loadModule, parseSync } from 'libpg-query'; +import wasm from 'libpg-query/wasm/libpg-query.wasm'; // bytes, via your bundler's binary loader + +await loadModule({ wasm }); +const result = parseSync('SELECT * FROM users'); +``` + ### Type Definitions ```typescript diff --git a/versions/17/package.json b/versions/17/package.json index 5b26d341..d692b40a 100644 --- a/versions/17/package.json +++ b/versions/17/package.json @@ -30,7 +30,7 @@ "wasm:rebuild": "pnpm wasm:make rebuild", "wasm:clean": "pnpm wasm:make clean", "wasm:clean-cache": "pnpm wasm:make clean-cache", - "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js" + "test": "node --test test/parsing.test.js test/errors.test.js test/input-encoding.test.js test/wasm-recovery.test.js test/wasm-stale-stack.test.js test/load-module.test.js" }, "author": "Constructive ", "license": "MIT", diff --git a/versions/17/src/index.ts b/versions/17/src/index.ts index d717d1e4..c1a6f807 100644 --- a/versions/17/src/index.ts +++ b/versions/17/src/index.ts @@ -149,6 +149,9 @@ let replaceError: unknown; const LARGE_INSTANCE_BYTES = 512 * 1024 * 1024; let largeUnfreed = 0; +// Set once the host hands over the .wasm itself; the wrapper then never reads the file. +let wasmProvided = false; + // Reads the .wasm file the way the Emscripten loader does: from disk in Node, else with fetch. async function compileWasm(url: string): Promise { const proc = (globalThis as any).process; @@ -164,11 +167,20 @@ async function compileWasm(url: string): Promise { function instantiate(): Promise { if (loading) return loading; const promise = (async () => { + if (wasmProvided) { + // Same yield as for callers of the default loader, so a capped out-of-memory run is freed first. + await undefined; + startProvidedInstance(); + return; + } let wasmUrl: string | undefined; const module: any = await PgQueryModule({ locateFile: (path: string, dir: string) => (wasmUrl = dir + path), }); + // loadModule({ wasm }) may have finished while the file was loading; keep its instance. + if (wasmProvided) return; compiled ??= await compileWasm(wasmUrl); + if (wasmProvided) return; wasmModule = module; replaceError = undefined; })(); @@ -255,7 +267,36 @@ function ensureLoaded() { throw new Error("WASM module not initialized. Call `loadModule()` first."); } -export async function loadModule() { +type WasmCompiledModule = typeof globalThis extends { WebAssembly: { Module: new (...args: any[]) => infer M } } ? M : never; + +export interface LoadModuleOptions { + /** + * The contents of `libpg-query.wasm`, or the module compiled from them, for + * hosts where the wrapper cannot read the file itself (a custom protocol + * without fetch support, a bundle that inlines assets). It is compiled once; + * every instance, including replacements after a failed call, is created from it. + */ + wasm?: ArrayBuffer | ArrayBufferView | WasmCompiledModule; +} + +function startProvidedInstance(): void { + replaceInstance(); + if (!wasmModule) throw replaceError; +} + +// A rest parameter keeps loadModule.length at 0, so `before(loadModule)` is not +// mistaken for a callback-style hook by test runners. +export async function loadModule(...args: [options?: LoadModuleOptions]): Promise { + const wasm = args[0]?.wasm; + if (wasm !== undefined) { + const module = wasm instanceof WebAssembly.Module + ? wasm + : await WebAssembly.compile(wasm as ArrayBuffer | ArrayBufferView); + compiled = module; + wasmProvided = true; + startProvidedInstance(); + return; + } while (!wasmModule) await instantiate(); } diff --git a/versions/17/test/load-module.test.js b/versions/17/test/load-module.test.js new file mode 100644 index 00000000..f8cd5084 --- /dev/null +++ b/versions/17/test/load-module.test.js @@ -0,0 +1,94 @@ +const { describe, it } = require('node:test'); +const assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const WASM = path.join(__dirname, '../wasm/libpg-query.wasm'); + +// Runs `body` in a fresh process whose copy of the wrapper has no +// libpg-query.wasm next to it, like a host that cannot read its own files. +function withoutWasmFile(body) { + const dir = fs.mkdtempSync(path.join(__dirname, '.no-wasm-')); + try { + for (const file of ['index.cjs', 'libpg-query.js']) { + fs.copyFileSync(path.join(__dirname, '../wasm', file), path.join(dir, file)); + } + const script = ` + const fs = require('node:fs'); + const lib = require(${JSON.stringify(path.join(dir, 'index.cjs'))}); + const bytes = fs.readFileSync(${JSON.stringify(WASM)}); + const DEEP = 'SELECT 1' + '+1'.repeat(100000); + (async () => { ${body} })().then( + (result) => console.log(JSON.stringify(result)), + (error) => { console.log(JSON.stringify({ error: String(error && error.message || error) })); } + ); + `; + const out = execFileSync(process.execPath, ['--max-old-space-size=4096', '-e', script], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + timeout: 120000, + }); + return JSON.parse(out.trim().split('\n').pop()); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +describe('loadModule({ wasm }) when the wrapper cannot read libpg-query.wasm', { timeout: 300000 }, () => { + it('fails to load without it', () => { + const result = withoutWasmFile(`await lib.loadModule(); return { ok: true };`); + assert.ok(result.error, 'expected loadModule() to reject without the .wasm file'); + }); + + it('loads from the bytes and recovers after a failed call without the file', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const first = lib.parseSync('SELECT 1').stmts.length; + let threw = false; + try { lib.parseSync(DEEP); } catch { threw = true; } + const afterSync = lib.parseSync('SELECT 2').stmts.length; + let threwAsync = false; + try { await lib.parse(DEEP); } catch { threwAsync = true; } + const afterAsync = (await lib.parse('SELECT 3')).stmts.length; + return { first, threw, afterSync, threwAsync, afterAsync }; + `); + assert.deepEqual(result, { first: 1, threw: true, afterSync: 1, threwAsync: true, afterAsync: 1 }); + }); + + it('accepts a compiled WebAssembly.Module and a typed array view', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: new WebAssembly.Module(bytes) }); + const fromModule = lib.parseSync('SELECT 1').stmts.length; + await lib.loadModule({ wasm: new Uint8Array(bytes) }); + return { fromModule, fromView: (await lib.parse('SELECT 1; SELECT 2')).stmts.length }; + `); + assert.deepEqual(result, { fromModule: 1, fromView: 2 }); + }); + + it('starts a new instance from the bytes after repeated out-of-memory failures', () => { + const result = withoutWasmFile(` + await lib.loadModule({ wasm: bytes }); + const huge = 'SELECT 1;\\n'.repeat(5000000); + const messages = []; + for (let i = 0; i < 3; i++) { + try { lib.parseSync(huge); } catch (e) { messages.push(e.message); } + } + return { messages, after: (await lib.parse('SELECT 1')).stmts.length }; + `); + assert.equal(result.messages.length, 3); + assert.match(result.messages[2], /^WASM memory limit hit repeatedly in one synchronous run/); + assert.equal(result.after, 1); + }); + + it('rejects input that is not WebAssembly', () => { + const result = withoutWasmFile(`await lib.loadModule({ wasm: new Uint8Array([1, 2, 3]) }); return { ok: true };`); + assert.ok(result.error); + }); +}); + +describe('loadModule() arity', () => { + it('stays 0 so test runners can pass it directly as a hook', () => { + assert.equal(require('../wasm/index.cjs').loadModule.length, 0); + }); +});