From 3c4c5d2a87abb157dd8690e0ae2609d7e8e1b487 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 27 Oct 2026 03:14:15 +0000 Subject: [PATCH 1/7] feat(gridfs): add error codes and writePathPolicy.gridfs Introduce the six GridFS error codes and a gridfs policy field. model-only inherit blocks GridFS writes from both sources, and violations keep INVALID_OPERATION with reason WRITE_PATH_POLICY. Co-authored-by: rocky --- src/capabilities/write-path-policy/index.ts | 21 ++++- src/core/errors/index.ts | 7 ++ src/entry/runtime-exports.ts | 1 + test/unit/errors/errors.test.ts | 1 + test/unit/runtime/write-path-policy.test.ts | 91 +++++++++++++++++++++ types/base.d.ts | 6 ++ types/monsqlize.d.ts | 11 +++ 7 files changed, 137 insertions(+), 1 deletion(-) diff --git a/src/capabilities/write-path-policy/index.ts b/src/capabilities/write-path-policy/index.ts index 0ea5896..c02c786 100644 --- a/src/capabilities/write-path-policy/index.ts +++ b/src/capabilities/write-path-policy/index.ts @@ -5,12 +5,15 @@ import { ErrorCodes, createError } from '../../core/errors'; export type WritePathPolicyMode = 'allow-both' | 'model-only'; export type WritePathPolicyRawMode = 'inherit' | 'allow' | 'block'; export type WritePathPolicyManagementMode = 'inherit' | 'allow' | 'block'; +export type WritePathPolicyGridfsMode = 'inherit' | 'allow' | 'block'; export type WritePathPolicyViolationAction = 'throw' | 'warn'; export interface WritePathPolicyRule { mode?: WritePathPolicyMode; raw?: WritePathPolicyRawMode; management?: WritePathPolicyManagementMode; + /** GridFS writes. `inherit` allows them only in `allow-both`; `model-only` still blocks until `allow`. */ + gridfs?: WritePathPolicyGridfsMode; onViolation?: WritePathPolicyViolationAction; } @@ -19,7 +22,7 @@ export interface WritePathPolicyOptions { namespaces?: Record; } -export type WritePathOperationCategory = 'write' | 'batch' | 'management' | 'raw'; +export type WritePathOperationCategory = 'write' | 'batch' | 'management' | 'raw' | 'gridfs'; export type WritePathSource = 'collection' | 'model' | 'legacy' | 'db' | 'client'; export interface WritePathNamespace { @@ -33,6 +36,7 @@ export interface NormalizedWritePathRule { mode: WritePathPolicyMode; raw: WritePathPolicyRawMode; management: WritePathPolicyManagementMode; + gridfs: WritePathPolicyGridfsMode; onViolation: WritePathPolicyViolationAction; } @@ -51,12 +55,14 @@ const DEFAULT_RULE: NormalizedWritePathRule = Object.freeze({ mode: 'allow-both', raw: 'inherit', management: 'inherit', + gridfs: 'inherit', onViolation: 'throw', }); const ALLOWED_MODES = new Set(['allow-both', 'model-only']); const ALLOWED_RAW = new Set(['inherit', 'allow', 'block']); const ALLOWED_MANAGEMENT = new Set(['inherit', 'allow', 'block']); +const ALLOWED_GRIDFS = new Set(['inherit', 'allow', 'block']); const ALLOWED_VIOLATION_ACTIONS = new Set(['throw', 'warn']); type AggregateInvalidationNormalizer = (input: { @@ -117,6 +123,10 @@ function normalizeRule( if (!ALLOWED_MANAGEMENT.has(management as WritePathPolicyManagementMode)) { invalidConfig(`${path}.management`, 'must be "inherit", "allow", or "block"'); } + const gridfs = ownValue(input, 'gridfs') ?? DEFAULT_RULE.gridfs; + if (!ALLOWED_GRIDFS.has(gridfs as WritePathPolicyGridfsMode)) { + invalidConfig(`${path}.gridfs`, 'must be "inherit", "allow", or "block"'); + } const onViolation = ownValue(input, 'onViolation') ?? DEFAULT_RULE.onViolation; if (!ALLOWED_VIOLATION_ACTIONS.has(onViolation as WritePathPolicyViolationAction)) { invalidConfig(`${path}.onViolation`, 'must be "throw" or "warn"'); @@ -126,6 +136,7 @@ function normalizeRule( mode: mode as WritePathPolicyMode, raw: raw as WritePathPolicyRawMode, management: management as WritePathPolicyManagementMode, + gridfs: gridfs as WritePathPolicyGridfsMode, onViolation: onViolation as WritePathPolicyViolationAction, }; } @@ -231,6 +242,11 @@ function isAllowed(rule: NormalizedWritePathRule, source: WritePathSource, categ } return resolveManagement(rule) === 'allow'; } + if (category === 'gridfs') { + if (rule.gridfs === 'allow') return true; + if (rule.gridfs === 'block') return false; + return rule.mode === 'allow-both'; + } if (rule.mode === 'model-only') { return source === 'model'; } @@ -261,6 +277,7 @@ export function assertWritePathAllowed(config: { namespace: config.namespace, matchedRule: resolved.key, rule: resolved.rule, + reason: 'WRITE_PATH_POLICY' as const, }; const message = buildViolationMessage(config.operation, config.category, config.source); if (resolved.rule.onViolation === 'warn') { @@ -345,6 +362,7 @@ export function assertDbLevelWritePathAllowed(config: { namespace: { db: config.dbName }, matchedRule: blocked.key, rule: blocked.rule, + reason: 'WRITE_PATH_POLICY' as const, }; const message = buildViolationMessage(config.operation, config.category, 'db'); if (blocked.rule.onViolation === 'warn') { @@ -386,6 +404,7 @@ export function assertClientLevelWritePathAllowed(config: { namespace: {}, matchedRule: blocked.key, rule: blocked.rule, + reason: 'WRITE_PATH_POLICY' as const, }; const message = buildViolationMessage(config.operation, 'raw', 'client'); if (blocked.rule.onViolation === 'warn') { diff --git a/src/core/errors/index.ts b/src/core/errors/index.ts index ddcf47c..0cfac1a 100644 --- a/src/core/errors/index.ts +++ b/src/core/errors/index.ts @@ -31,6 +31,13 @@ export const ErrorCodes = { SEARCH_INDEX_UNSUPPORTED: 'SEARCH_INDEX_UNSUPPORTED', SEARCH_INDEX_BUILD_FAILED: 'SEARCH_INDEX_BUILD_FAILED', SEARCH_INDEX_WAIT_TIMEOUT: 'SEARCH_INDEX_WAIT_TIMEOUT', + /** GridFS file, range, and resumable-upload errors (v3.5.0). */ + GRIDFS_FILE_NOT_FOUND: 'GRIDFS_FILE_NOT_FOUND', + GRIDFS_FILE_EXISTS: 'GRIDFS_FILE_EXISTS', + GRIDFS_INTEGRITY_MISMATCH: 'GRIDFS_INTEGRITY_MISMATCH', + GRIDFS_UPLOAD_INCOMPLETE: 'GRIDFS_UPLOAD_INCOMPLETE', + GRIDFS_RESUME_MISMATCH: 'GRIDFS_RESUME_MISMATCH', + GRIDFS_RANGE_INVALID: 'GRIDFS_RANGE_INVALID', /** Relation select is outside the supported inclusion-only top-level contract. */ INVALID_RELATION_SELECT: 'INVALID_RELATION_SELECT', /** A protected delete found declared Model relations that still reference its target. */ diff --git a/src/entry/runtime-exports.ts b/src/entry/runtime-exports.ts index dbf66e6..bfb1b85 100644 --- a/src/entry/runtime-exports.ts +++ b/src/entry/runtime-exports.ts @@ -141,6 +141,7 @@ export type { NormalizedWritePathRule, WritePathNamespace, WritePathOperationCategory, + WritePathPolicyGridfsMode, WritePathPolicyManagementMode, WritePathPolicyMode, WritePathPolicyOptions, diff --git a/test/unit/errors/errors.test.ts b/test/unit/errors/errors.test.ts index 4d10bba..dd982f8 100644 --- a/test/unit/errors/errors.test.ts +++ b/test/unit/errors/errors.test.ts @@ -36,6 +36,7 @@ describe('Error system: ErrorCodes', () => { 'MODEL_NOT_DEFINED', 'NO_POOL_MANAGER', 'POOL_NOT_FOUND', 'INVALID_MODEL_DEFINITION', 'INVALID_SCHEMA_TYPE', 'INVALID_SEARCH_INDEX_DEFINITION', 'SEARCH_INDEX_EXISTS', 'SEARCH_INDEX_NOT_FOUND', 'SEARCH_INDEX_UNSUPPORTED', 'SEARCH_INDEX_BUILD_FAILED', 'SEARCH_INDEX_WAIT_TIMEOUT', + 'GRIDFS_FILE_NOT_FOUND', 'GRIDFS_FILE_EXISTS', 'GRIDFS_INTEGRITY_MISMATCH', 'GRIDFS_UPLOAD_INCOMPLETE', 'GRIDFS_RESUME_MISMATCH', 'GRIDFS_RANGE_INVALID', ]; for (const code of expected) { assert.equal( diff --git a/test/unit/runtime/write-path-policy.test.ts b/test/unit/runtime/write-path-policy.test.ts index 918f150..cac30b0 100644 --- a/test/unit/runtime/write-path-policy.test.ts +++ b/test/unit/runtime/write-path-policy.test.ts @@ -506,3 +506,94 @@ describe('writePathPolicy runtime enforcement', () => { })); }); }); + +describe('writePathPolicy gridfs category', () => { + const namespace = { db: 'policy_db', collection: 'fs.files' }; + + function assertGridfs(policyInput: Parameters[0], source: 'db' | 'model') { + return () => assertWritePathAllowed({ + policy: normalizeWritePathPolicy(policyInput), + namespace, + source, + operation: 'upload', + category: 'gridfs', + }); + } + + it('keeps gridfs inherit and preserves an explicit allow value', () => { + const implicit = normalizeWritePathPolicy({ default: 'model-only' }); + assert.equal(implicit.default.gridfs, 'inherit'); + const explicit = normalizeWritePathPolicy({ default: { mode: 'model-only', gridfs: 'allow' } }); + assert.equal(explicit.default.gridfs, 'allow'); + assert.equal(explicit.default.mode, 'model-only'); + }); + + it('rejects an unknown gridfs enum and ignores unrelated keys', () => { + assert.throws( + () => normalizeWritePathPolicy({ default: { gridfs: 'sometimes' as 'allow' } }), + /gridfs/, + ); + const normalized = normalizeWritePathPolicy({ default: { mode: 'allow-both', extra: true } as { mode: 'allow-both' } }); + assert.equal(normalized.default.gridfs, 'inherit'); + assert.equal(normalized.default.mode, 'allow-both'); + }); + + it('allows gridfs inherit writes only in allow-both, from either source', () => { + assert.doesNotThrow(assertGridfs({ default: 'allow-both' }, 'db')); + assert.doesNotThrow(assertGridfs({ default: 'allow-both' }, 'model')); + assert.throws(assertGridfs({ default: 'model-only' }, 'db'), /writePathPolicy blocked/); + assert.throws(assertGridfs({ default: 'model-only' }, 'model'), /writePathPolicy blocked/); + }); + + it('lets gridfs allow or block override mode, including model source', () => { + assert.doesNotThrow(assertGridfs({ default: { mode: 'model-only', gridfs: 'allow' } }, 'model')); + assert.doesNotThrow(assertGridfs({ default: { mode: 'model-only', gridfs: 'allow' } }, 'db')); + assert.throws(assertGridfs({ default: { mode: 'allow-both', gridfs: 'block' } }, 'db'), /writePathPolicy blocked/); + assert.throws(assertGridfs({ default: { mode: 'allow-both', gridfs: 'block' } }, 'model'), /writePathPolicy blocked/); + }); + + it('matches the bucket files namespace and records category plus reason', () => { + const policy = normalizeWritePathPolicy({ + default: 'allow-both', + namespaces: { 'policy_db.fs.files': { gridfs: 'block' } }, + }); + try { + assertWritePathAllowed({ + policy, + namespace, + source: 'db', + operation: 'upload', + category: 'gridfs', + }); + assert.fail('expected gridfs write to be blocked'); + } catch (error) { + const details = (error as { details?: Array> }).details?.[0]; + assert.equal(details?.category, 'gridfs'); + assert.equal(details?.reason, 'WRITE_PATH_POLICY'); + assert.equal(details?.matchedRule, 'policy_db.fs.files'); + } + }); + + it('warns and allows a blocked gridfs write when onViolation is warn', () => { + const warnings: unknown[] = []; + assert.doesNotThrow(() => assertWritePathAllowed({ + policy: normalizeWritePathPolicy({ default: { mode: 'model-only', onViolation: 'warn' } }), + namespace, + source: 'model', + operation: 'upload', + category: 'gridfs', + logger: { warn: (...args: unknown[]) => { warnings.push(args); } }, + })); + assert.equal(warnings.length, 1); + }); + + it('keeps management inherit distinct from gridfs inherit', () => { + const policy = normalizeWritePathPolicy({ default: 'model-only' }); + assert.throws(() => assertWritePathAllowed({ + policy, namespace, source: 'db', operation: 'drop', category: 'management', + }), /writePathPolicy blocked/); + assert.doesNotThrow(() => assertWritePathAllowed({ + policy, namespace, source: 'model', operation: 'drop', category: 'management', + })); + }); +}); diff --git a/types/base.d.ts b/types/base.d.ts index e3b90c4..07870ab 100644 --- a/types/base.d.ts +++ b/types/base.d.ts @@ -43,6 +43,12 @@ export declare const enum ErrorCodes { SEARCH_INDEX_UNSUPPORTED = 'SEARCH_INDEX_UNSUPPORTED', SEARCH_INDEX_BUILD_FAILED = 'SEARCH_INDEX_BUILD_FAILED', SEARCH_INDEX_WAIT_TIMEOUT = 'SEARCH_INDEX_WAIT_TIMEOUT', + GRIDFS_FILE_NOT_FOUND = 'GRIDFS_FILE_NOT_FOUND', + GRIDFS_FILE_EXISTS = 'GRIDFS_FILE_EXISTS', + GRIDFS_INTEGRITY_MISMATCH = 'GRIDFS_INTEGRITY_MISMATCH', + GRIDFS_UPLOAD_INCOMPLETE = 'GRIDFS_UPLOAD_INCOMPLETE', + GRIDFS_RESUME_MISMATCH = 'GRIDFS_RESUME_MISMATCH', + GRIDFS_RANGE_INVALID = 'GRIDFS_RANGE_INVALID', INVALID_RELATION_SELECT = 'INVALID_RELATION_SELECT', RELATION_IN_USE = 'RELATION_IN_USE', RELATION_USAGE_UNAVAILABLE = 'RELATION_USAGE_UNAVAILABLE', diff --git a/types/monsqlize.d.ts b/types/monsqlize.d.ts index 4d2749c..a184e58 100644 --- a/types/monsqlize.d.ts +++ b/types/monsqlize.d.ts @@ -92,6 +92,12 @@ export type WritePathPolicyRawMode = 'inherit' | 'allow' | 'block'; /** Controls whether collection/database management writes are available under a write path policy. */ export type WritePathPolicyManagementMode = 'inherit' | 'allow' | 'block'; +/** + * Controls GridFS writes. `inherit` follows `mode`, except `model-only` still blocks + * GridFS writes from every source until the rule is `allow`. + */ +export type WritePathPolicyGridfsMode = 'inherit' | 'allow' | 'block'; + /** Controls violation handling; `warn` logs and allows for migration observation. */ export type WritePathPolicyViolationAction = 'throw' | 'warn'; @@ -103,6 +109,11 @@ export interface WritePathPolicyRule { raw?: WritePathPolicyRawMode; /** Management write policy. `inherit` allows model management and blocks external management in `model-only`. */ management?: WritePathPolicyManagementMode; + /** + * GridFS write policy. Default: `inherit`. + * `inherit` allows GridFS writes in `allow-both` and blocks them in `model-only` even when `source` is `model`. + */ + gridfs?: WritePathPolicyGridfsMode; /** Violation action. Default: `throw`. */ onViolation?: WritePathPolicyViolationAction; } From 646711acc949118a4b70b25d7645d21ea4f60e8e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 27 Oct 2026 03:14:18 +0000 Subject: [PATCH 2/7] feat(gridfs): implement GridFS bucket core and resumable uploads Add msq.gridfs() and model.gridfs() with upload, download, ranges, ensureIndexes, and phase-1 resumable upload. Hash buffers before any write, and keep a file invisible until finalize succeeds. Co-authored-by: rocky --- src/capabilities/gridfs/bucket.ts | 112 ++++++++++ src/capabilities/gridfs/bytes.ts | 13 ++ src/capabilities/gridfs/driver.ts | 27 +++ src/capabilities/gridfs/errors.ts | 153 ++++++++++++++ src/capabilities/gridfs/index.ts | 42 ++++ src/capabilities/gridfs/indexes.ts | 36 ++++ src/capabilities/gridfs/normalize.ts | 46 +++++ src/capabilities/gridfs/operations.ts | 285 ++++++++++++++++++++++++++ src/capabilities/gridfs/resumable.ts | 238 +++++++++++++++++++++ src/capabilities/gridfs/types.ts | 127 ++++++++++++ src/capabilities/gridfs/validate.ts | 188 +++++++++++++++++ src/capabilities/model/index.ts | 3 + src/entry/runtime-core.ts | 3 + src/entry/runtime-gridfs.ts | 106 ++++++++++ types/gridfs.d.ts | 146 +++++++++++++ types/index.d.ts | 1 + types/model.d.ts | 2 + types/monsqlize.d.ts | 9 + types/runtime.d.ts | 2 + 19 files changed, 1539 insertions(+) create mode 100644 src/capabilities/gridfs/bucket.ts create mode 100644 src/capabilities/gridfs/bytes.ts create mode 100644 src/capabilities/gridfs/driver.ts create mode 100644 src/capabilities/gridfs/errors.ts create mode 100644 src/capabilities/gridfs/index.ts create mode 100644 src/capabilities/gridfs/indexes.ts create mode 100644 src/capabilities/gridfs/normalize.ts create mode 100644 src/capabilities/gridfs/operations.ts create mode 100644 src/capabilities/gridfs/resumable.ts create mode 100644 src/capabilities/gridfs/types.ts create mode 100644 src/capabilities/gridfs/validate.ts create mode 100644 src/entry/runtime-gridfs.ts create mode 100644 types/gridfs.d.ts diff --git a/src/capabilities/gridfs/bucket.ts b/src/capabilities/gridfs/bucket.ts new file mode 100644 index 0000000..9093b72 --- /dev/null +++ b/src/capabilities/gridfs/bucket.ts @@ -0,0 +1,112 @@ +import type { Readable, Writable } from 'node:stream'; +import type { ObjectId } from 'mongodb'; +import { assertWritePathAllowed } from '../write-path-policy'; +import type { + AppendChunksResult, + BeginResumableUploadOptions, + BeginResumableUploadResult, + GridFSContext, + GridFSDeleteOptions, + GridFSDownloadRange, + GridFSFileInfo, + GridFSFindFilesFilter, + GridFSFindFilesOptions, + GridFSIdOrName, + GridFSUploadOptions, + GridFSUploadStreamOptions, +} from './types'; +import { deleteFile, downloadFile, dropBucket, findFiles, getFile, openDownloadStream, openUploadStream, renameFile, uploadFile } from './operations'; +import { abortResumableUpload, appendChunks, beginResumableUpload, finalizeResumableUpload } from './resumable'; + +export class MonSQLizeGridFSBucketImpl { + constructor(private readonly ctx: GridFSContext) {} + + get bucketName(): string { return this.ctx.bucketName; } + get dbName(): string { return this.ctx.dbName; } + get chunkSizeBytes(): number { return this.ctx.chunkSizeBytes; } + + upload(filename: string, data: Buffer | Uint8Array | Readable, opts?: GridFSUploadOptions): Promise { + this.assertGridfs('upload'); + return uploadFile(this.ctx, filename, data, opts); + } + + uploadStream(filename: string, opts?: GridFSUploadStreamOptions): Writable & { id: ObjectId; done: Promise } { + this.assertGridfs('uploadStream'); + return openUploadStream(this.ctx, filename, opts); + } + + download(idOrName: GridFSIdOrName): Promise { + return downloadFile(this.ctx, idOrName); + } + + downloadStream(idOrName: GridFSIdOrName, range?: GridFSDownloadRange): Readable { + return openDownloadStream(this.ctx, idOrName, range); + } + + getFile(id: ObjectId): Promise { + return getFile(this.ctx, id); + } + + findFiles(filter?: GridFSFindFilesFilter, opts?: GridFSFindFilesOptions): Promise { + return findFiles(this.ctx, filter, opts); + } + + delete(id: ObjectId, opts?: GridFSDeleteOptions): Promise { + this.assertGridfs('delete'); + return deleteFile(this.ctx, id, opts); + } + + rename(id: ObjectId, filename: string): Promise { + this.assertGridfs('rename'); + return renameFile(this.ctx, id, filename); + } + + drop(): Promise { + this.assertAllowed('drop', 'management'); + return dropBucket(this.ctx); + } + + beginResumableUpload(filename: string, opts: BeginResumableUploadOptions): Promise { + this.assertGridfs('beginResumableUpload'); + return beginResumableUpload(this.ctx, filename, opts); + } + + appendChunks(uploadId: ObjectId, fromChunk: number, data: Buffer | Uint8Array | Readable): Promise { + this.assertGridfs('appendChunks'); + return appendChunks(this.ctx, uploadId, fromChunk, data); + } + + finalizeResumableUpload(uploadId: ObjectId): Promise { + this.assertGridfs('finalizeResumableUpload'); + return finalizeResumableUpload(this.ctx, uploadId); + } + + abortResumableUpload(uploadId: ObjectId): Promise { + this.assertGridfs('abortResumableUpload'); + return abortResumableUpload(this.ctx, uploadId); + } + + ensureIndexes(): Promise { + this.assertGridfs('ensureIndexes'); + return this.ctx.indexes.ensure(); + } + + private assertGridfs(operation: string): void { + this.assertAllowed(operation, 'gridfs'); + } + + private assertAllowed(operation: string, category: 'gridfs' | 'management'): void { + assertWritePathAllowed({ + policy: this.ctx.policy, + namespace: { + ...(this.ctx.poolName ? { pool: this.ctx.poolName } : {}), + db: this.ctx.dbName, + collection: `${this.ctx.bucketName}.files`, + }, + source: this.ctx.source, + operation, + category, + logger: this.ctx.logger, + }); + } +} diff --git a/src/capabilities/gridfs/bytes.ts b/src/capabilities/gridfs/bytes.ts new file mode 100644 index 0000000..9faace0 --- /dev/null +++ b/src/capabilities/gridfs/bytes.ts @@ -0,0 +1,13 @@ +import { Readable } from 'node:stream'; + +export async function readReadable(source: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of source) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return Buffer.concat(chunks); +} + +export function releaseReadable(source: Readable): void { + if (!source.destroyed) source.destroy(); +} diff --git a/src/capabilities/gridfs/driver.ts b/src/capabilities/gridfs/driver.ts new file mode 100644 index 0000000..98918c1 --- /dev/null +++ b/src/capabilities/gridfs/driver.ts @@ -0,0 +1,27 @@ +import { GridFSBucket, type Db, type ObjectId } from 'mongodb'; +import type { GridFSDriver } from './types'; + +export function createGridFSDriver(db: Db, bucketName: string, chunkSizeBytes: number): GridFSDriver { + const bucket = new GridFSBucket(db, { bucketName, chunkSizeBytes }); + return { + openUploadStream(id, filename, options) { + return bucket.openUploadStreamWithId(id, filename, { + chunkSizeBytes: options.chunkSizeBytes, + metadata: options.metadata, + contentType: options.contentType, + }); + }, + openDownloadStream(id: ObjectId, range) { + return bucket.openDownloadStream(id, range); + }, + delete(id) { + return bucket.delete(id); + }, + rename(id, filename) { + return bucket.rename(id, filename); + }, + drop() { + return bucket.drop(); + }, + }; +} diff --git a/src/capabilities/gridfs/errors.ts b/src/capabilities/gridfs/errors.ts new file mode 100644 index 0000000..0a13498 --- /dev/null +++ b/src/capabilities/gridfs/errors.ts @@ -0,0 +1,153 @@ +import type { ObjectId } from 'mongodb'; +import { ErrorCodes, createError, type MonSQLizeError } from '../../core/errors'; +import type { GridFSContext, GridFSIdOrName } from './types'; + +export interface GridFSErrorDetails { + operation: string; + bucket: string; + id?: ObjectId; + filename?: string; + sha256?: string; + uploadId?: ObjectId; + expected?: unknown; + actual?: unknown; + missingChunks?: number[]; + fromChunk?: number; + expectedNextChunk?: number; + start?: number; + end?: number; + length?: number; +} + +const GRIDFS_CODES = new Set([ + ErrorCodes.GRIDFS_FILE_NOT_FOUND, + ErrorCodes.GRIDFS_FILE_EXISTS, + ErrorCodes.GRIDFS_INTEGRITY_MISMATCH, + ErrorCodes.GRIDFS_UPLOAD_INCOMPLETE, + ErrorCodes.GRIDFS_RESUME_MISMATCH, + ErrorCodes.GRIDFS_RANGE_INVALID, + ErrorCodes.INVALID_ARGUMENT, + ErrorCodes.INVALID_OPERATION, + ErrorCodes.NOT_CONNECTED, +]); + +export function isGridFSDomainError(error: unknown): error is MonSQLizeError { + return error instanceof Error && GRIDFS_CODES.has((error as MonSQLizeError).code); +} + +function withMeta(error: MonSQLizeError, details: GridFSErrorDetails): MonSQLizeError { + Object.defineProperty(error, 'monsqlize', { + value: { operation: details.operation, bucket: details.bucket, id: details.id, filename: details.filename, uploadId: details.uploadId }, + enumerable: true, + }); + return error; +} + +export function gridfsError(code: string, message: string, details: GridFSErrorDetails, cause?: Error): MonSQLizeError { + return withMeta(createError(code, message, [stripUndefined(details)], cause), details); +} + +function stripUndefined(details: GridFSErrorDetails): GridFSErrorDetails { + const copy: GridFSErrorDetails = { operation: details.operation, bucket: details.bucket }; + for (const key of Object.keys(details) as Array) { + if (details[key] !== undefined) { + (copy as unknown as Record)[key] = details[key]; + } + } + return copy; +} + +export function invalidArgument(operation: string, field: string, message: string): MonSQLizeError { + return createError(ErrorCodes.INVALID_ARGUMENT, message, [{ operation, field, message }]); +} + +export function fileNotFound(operation: string, ctx: Pick, target: { id?: ObjectId; filename?: string }): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_FILE_NOT_FOUND, + 'GridFS file was not found', + { operation, bucket: ctx.bucketName, id: target.id, filename: target.filename }, + ); +} + +export function fileExists(operation: string, ctx: Pick, target: { id?: ObjectId; filename?: string; sha256?: string }): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_FILE_EXISTS, + 'GridFS file already exists', + { operation, bucket: ctx.bucketName, id: target.id, filename: target.filename, sha256: target.sha256 }, + ); +} + +export function integrityMismatch(operation: string, bucket: string, fields: { uploadId?: ObjectId; expected?: unknown; actual?: unknown }): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_INTEGRITY_MISMATCH, + 'GridFS content did not match the expected length or sha256', + { operation, bucket, uploadId: fields.uploadId, expected: fields.expected, actual: fields.actual }, + ); +} + +export function uploadIncomplete(bucket: string, uploadId: ObjectId, missingChunks: number[]): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_UPLOAD_INCOMPLETE, + 'GridFS resumable upload is missing chunks', + { operation: 'finalizeResumableUpload', bucket, uploadId, missingChunks }, + ); +} + +export function resumeMismatch(operation: string, bucket: string, fields: { uploadId?: ObjectId; fromChunk?: number; expectedNextChunk?: number }): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_RESUME_MISMATCH, + 'GridFS resumable upload did not match the open session', + { operation, bucket, uploadId: fields.uploadId, fromChunk: fields.fromChunk, expectedNextChunk: fields.expectedNextChunk }, + ); +} + +export function rangeInvalid(operation: string, bucket: string, start: number | undefined, end: number | undefined, length?: number): MonSQLizeError { + return gridfsError( + ErrorCodes.GRIDFS_RANGE_INVALID, + 'GridFS download range is invalid', + { operation, bucket, start, end, length }, + ); +} + +export function isNamespaceNotFound(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const record = error as { code?: unknown; codeName?: unknown }; + return record.code === 26 || record.codeName === 'NamespaceNotFound'; +} + +export function isDuplicateKey(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const code = (error as { code?: unknown }).code; + return code === 11000 || code === 11001; +} + +export function isDriverFileNotFound(error: unknown): boolean { + if (!error || typeof error !== 'object') return false; + const code = (error as { code?: unknown }).code; + const message = error instanceof Error ? error.message : ''; + return code === 'ENOENT' || /file not found/i.test(message) || /FileNotFound/i.test(message); +} + +export function mapDriverError(error: unknown, operation: string, ctx: Pick, target: { id?: ObjectId; filename?: string; uploadId?: ObjectId } = {}): MonSQLizeError { + if (isGridFSDomainError(error)) return error; + const cause = error instanceof Error ? error : new Error(String(error)); + if (isDriverFileNotFound(error)) { + return fileNotFound(operation, ctx, target); + } + if (isDuplicateKey(error)) { + return fileExists(operation, ctx, { id: target.id, filename: target.filename }); + } + return gridfsError(ErrorCodes.MONGODB_ERROR, cause.message || 'MongoDB GridFS operation failed', { + operation, + bucket: ctx.bucketName, + id: target.id, + filename: target.filename, + uploadId: target.uploadId, + }, cause); +} + +export function targetOf(idOrName: GridFSIdOrName | undefined): { id?: ObjectId; filename?: string } { + if (!idOrName || typeof idOrName !== 'object') return {}; + if ('filename' in idOrName && typeof idOrName.filename === 'string') return { filename: idOrName.filename }; + return { id: idOrName as ObjectId }; +} diff --git a/src/capabilities/gridfs/index.ts b/src/capabilities/gridfs/index.ts new file mode 100644 index 0000000..370edc8 --- /dev/null +++ b/src/capabilities/gridfs/index.ts @@ -0,0 +1,42 @@ +import type { Db } from 'mongodb'; +import type { Logger } from '../../core/logger'; +import type { NormalizedWritePathPolicy } from '../write-path-policy'; +import { MonSQLizeGridFSBucketImpl } from './bucket'; +import { createGridFSDriver } from './driver'; +import { GridFSIndexLatch } from './indexes'; +import { DEFAULT_CHUNK_SIZE_BYTES, type GridFSDriver } from './types'; +import { assertBucketName, assertChunkSize } from './validate'; + +export interface CreateGridFSBucketDeps { + db: Db; + bucketName: string; + dbName: string; + poolName?: string; + chunkSizeBytes?: number; + source: 'db' | 'model'; + policy?: NormalizedWritePathPolicy; + logger?: Pick; + driver?: GridFSDriver; +} + +export function createMonSQLizeGridFSBucket(deps: CreateGridFSBucketDeps): MonSQLizeGridFSBucketImpl { + assertBucketName(deps.bucketName); + const chunkSizeBytes = deps.chunkSizeBytes ?? DEFAULT_CHUNK_SIZE_BYTES; + assertChunkSize(chunkSizeBytes, 'gridfs'); + const driver = deps.driver ?? createGridFSDriver(deps.db, deps.bucketName, chunkSizeBytes); + return new MonSQLizeGridFSBucketImpl({ + db: deps.db, + driver, + bucketName: deps.bucketName, + dbName: deps.dbName, + poolName: deps.poolName, + chunkSizeBytes, + source: deps.source, + policy: deps.policy, + logger: deps.logger, + indexes: new GridFSIndexLatch(deps.db, deps.bucketName), + }); +} + +export { MonSQLizeGridFSBucketImpl }; +export type { GridFSDriver, GridFSFileInfo } from './types'; diff --git a/src/capabilities/gridfs/indexes.ts b/src/capabilities/gridfs/indexes.ts new file mode 100644 index 0000000..1d1c606 --- /dev/null +++ b/src/capabilities/gridfs/indexes.ts @@ -0,0 +1,36 @@ +import type { Db } from 'mongodb'; +import { UPLOADS_TTL_SECONDS } from './types'; +import { chunksCollectionName, filesCollectionName, uploadsCollectionName } from './normalize'; + +/** + * Idempotent GridFS indexes for one bucket instance. + * A failed attempt can be retried; concurrent callers share one in-flight promise. + */ +export class GridFSIndexLatch { + private inflight: Promise | null = null; + private ready = false; + + constructor(private readonly db: Db, private readonly bucketName: string) {} + + ensure(): Promise { + if (this.ready) return Promise.resolve(); + if (!this.inflight) { + this.inflight = this.create().then(() => { + this.ready = true; + }).finally(() => { + this.inflight = null; + }); + } + return this.inflight; + } + + private async create(): Promise { + const files = this.db.collection(filesCollectionName(this.bucketName)); + const chunks = this.db.collection(chunksCollectionName(this.bucketName)); + const uploads = this.db.collection(uploadsCollectionName(this.bucketName)); + await files.createIndex({ filename: 1, uploadDate: 1 }); + await chunks.createIndex({ files_id: 1, n: 1 }, { unique: true }); + await files.createIndex({ 'metadata.sha256': 1 }); + await uploads.createIndex({ createdAt: 1 }, { expireAfterSeconds: UPLOADS_TTL_SECONDS }); + } +} diff --git a/src/capabilities/gridfs/normalize.ts b/src/capabilities/gridfs/normalize.ts new file mode 100644 index 0000000..9b8c91c --- /dev/null +++ b/src/capabilities/gridfs/normalize.ts @@ -0,0 +1,46 @@ +import { Binary, ObjectId } from 'mongodb'; +import type { GridFSFileInfo } from './types'; +import { invalidArgument } from './errors'; + +export function filesCollectionName(bucketName: string): string { + return `${bucketName}.files`; +} + +export function chunksCollectionName(bucketName: string): string { + return `${bucketName}.chunks`; +} + +export function uploadsCollectionName(bucketName: string): string { + return `${bucketName}.uploads`; +} + +export function toFileInfo(doc: Record): GridFSFileInfo { + const info: GridFSFileInfo = { + id: doc._id as ObjectId, + filename: typeof doc.filename === 'string' ? doc.filename : '', + length: typeof doc.length === 'number' ? doc.length : 0, + chunkSize: typeof doc.chunkSize === 'number' ? doc.chunkSize : 0, + uploadDate: doc.uploadDate instanceof Date ? doc.uploadDate : new Date(doc.uploadDate as string | number), + }; + if (typeof doc.contentType === 'string') info.contentType = doc.contentType; + if (doc.metadata !== undefined && doc.metadata !== null && typeof doc.metadata === 'object') { + info.metadata = doc.metadata as Record; + } + return info; +} + +export function binaryToBuffer(value: unknown, operation: string): Buffer { + if (Buffer.isBuffer(value)) return value; + if (value instanceof Binary) { + const bytes = value.value(); + return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength); + } + if (value instanceof Uint8Array) { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength); + } + throw invalidArgument(operation, 'chunk', 'GridFS chunk data is not binary'); +} + +export function sameBytes(left: Buffer, right: Buffer): boolean { + return left.length === right.length && left.equals(right); +} diff --git a/src/capabilities/gridfs/operations.ts b/src/capabilities/gridfs/operations.ts new file mode 100644 index 0000000..bc8ab6e --- /dev/null +++ b/src/capabilities/gridfs/operations.ts @@ -0,0 +1,285 @@ +import { PassThrough, Readable, Writable } from 'node:stream'; +import { ObjectId, type Sort } from 'mongodb'; +import type { + GridFSContext, + GridFSDeleteOptions, + GridFSDownloadRange, + GridFSDriverStream, + GridFSFileInfo, + GridFSFindFilesFilter, + GridFSFindFilesOptions, + GridFSIdOrName, + GridFSUploadOptions, + GridFSUploadStreamOptions, +} from './types'; +import { readReadable, releaseReadable } from './bytes'; +import { fileExists, fileNotFound, integrityMismatch, invalidArgument, isNamespaceNotFound, mapDriverError, targetOf } from './errors'; +import { chunksCollectionName, filesCollectionName, toFileInfo, uploadsCollectionName } from './normalize'; +import { + assertChunkSize, + assertContentType, + assertFilename, + assertFindOptions, + assertIfExists, + assertObjectId, + assertSha256, + buildMetadata, + isReadableStream, + parseRangeShape, + rangeBounds, + sanitizeFilter, + sha256Hex, + toNodeBuffer, +} from './validate'; + +export async function uploadFile( + ctx: GridFSContext, + filename: string, + data: Buffer | Uint8Array | Readable, + opts?: GridFSUploadOptions, +): Promise { + assertFilename(filename, 'upload'); + const options = opts ?? {}; + if (options.ifExists !== undefined) assertIfExists(options.ifExists); + const ifExists = options.ifExists ?? 'error'; + if (options.id) assertObjectId(options.id, 'upload', 'id'); + if (options.sha256 !== undefined) assertSha256(options.sha256, 'upload'); + assertContentType(options.contentType, 'upload'); + const chunkSizeBytes = options.chunkSizeBytes ?? ctx.chunkSizeBytes; + assertChunkSize(chunkSizeBytes, 'upload'); + const metadata = buildMetadata(options.metadata, options.sha256, 'upload'); + + let payload: Buffer | undefined; + let source: Readable | undefined; + if (Buffer.isBuffer(data) || data instanceof Uint8Array) payload = toNodeBuffer(data); + else if (isReadableStream(data)) source = data; + else throw invalidArgument('upload', 'data', 'data must be a Buffer, Uint8Array, or Readable'); + + if (source && options.sha256 !== undefined) { + payload = await readReadable(source); + source = undefined; + } + if (payload && options.sha256 !== undefined) { + const actual = sha256Hex(payload); + if (actual !== options.sha256) { + throw integrityMismatch('upload', ctx.bucketName, { expected: options.sha256, actual }); + } + } + + await ctx.indexes.ensure(); + if (options.id && await getFile(ctx, options.id)) { + throw fileExists('upload', ctx, { id: options.id, filename, sha256: options.sha256 }); + } + if (ifExists !== 'duplicate') { + const existing = await findConflict(ctx, filename, options.sha256); + if (existing) { + if (source) releaseReadable(source); + if (ifExists === 'skip') return existing; + throw fileExists('upload', ctx, { id: existing.id, filename, sha256: options.sha256 }); + } + } + + const id = options.id ?? new ObjectId(); + const writeOptions = { chunkSizeBytes, metadata, contentType: options.contentType }; + if (payload) await writeBuffer(ctx, id, filename, payload, writeOptions); + else if (source) await writeReadable(ctx, id, filename, source, writeOptions); + const info = await getFile(ctx, id); + if (!info) throw fileNotFound('upload', ctx, { id, filename }); + return info; +} + +export function openUploadStream(ctx: GridFSContext, filename: string, opts?: GridFSUploadStreamOptions): Writable & { id: ObjectId; done: Promise } { + assertFilename(filename, 'uploadStream'); + const options = opts ?? {}; + if ('ifExists' in options) throw invalidArgument('uploadStream', 'ifExists', 'uploadStream does not support ifExists'); + if (options.id) assertObjectId(options.id, 'uploadStream', 'id'); + if (options.sha256 !== undefined) assertSha256(options.sha256, 'uploadStream'); + assertContentType(options.contentType, 'uploadStream'); + const chunkSizeBytes = options.chunkSizeBytes ?? ctx.chunkSizeBytes; + assertChunkSize(chunkSizeBytes, 'uploadStream'); + const metadata = buildMetadata(options.metadata, options.sha256, 'uploadStream'); + const id = options.id ?? new ObjectId(); + const input = new PassThrough(); + input.on('error', () => undefined); + const done = (async (): Promise => { + let target: GridFSDriverStream | undefined; + try { + await ctx.indexes.ensure(); + target = ctx.driver.openUploadStream(id, filename, { + chunkSizeBytes, + metadata, + contentType: options.contentType, + }); + const { pipeline } = await import('node:stream/promises'); + await pipeline(input, target); + const info = await getFile(ctx, id); + if (!info) throw fileNotFound('uploadStream', ctx, { id, filename }); + return info; + } catch (error) { + if (target) await cleanupFailedWrite(ctx, id, target); + throw mapDriverError(error, 'uploadStream', ctx, { id, filename }); + } + })(); + void done.catch(() => undefined); + return Object.assign(input, { id, done }); +} + +export async function downloadFile(ctx: GridFSContext, idOrName: GridFSIdOrName): Promise { + const file = await resolveFile(ctx, idOrName, 'download'); + if (!file) throw fileNotFound('download', ctx, targetOf(idOrName)); + const source = ctx.driver.openDownloadStream(file.id); + try { + return await readReadable(source); + } catch (error) { + throw mapDriverError(error, 'download', ctx, { id: file.id, filename: file.filename }); + } +} + +export function openDownloadStream(ctx: GridFSContext, idOrName: GridFSIdOrName, range?: GridFSDownloadRange): Readable { + const output = new PassThrough(); + queueMicrotask(() => { + void (async () => { + try { + const shape = parseRangeShape(range, 'downloadStream', ctx.bucketName); + const file = await resolveFile(ctx, idOrName, 'downloadStream'); + if (!file) throw fileNotFound('downloadStream', ctx, targetOf(idOrName)); + const bounds = rangeBounds(shape, file.length, 'downloadStream', ctx.bucketName); + const source = ctx.driver.openDownloadStream(file.id, bounds); + source.on('error', (error) => output.destroy(mapDriverError(error, 'downloadStream', ctx, { id: file.id, filename: file.filename }))); + source.pipe(output); + } catch (error) { + output.destroy(mapDriverError(error, 'downloadStream', ctx, targetOf(idOrName))); + } + })(); + }); + return output; +} + +export async function getFile(ctx: GridFSContext, id: ObjectId): Promise { + assertObjectId(id, 'getFile', 'id'); + const doc = await ctx.db.collection(filesCollectionName(ctx.bucketName)).findOne({ _id: id }); + return doc ? toFileInfo(doc as Record) : null; +} + +export async function findFiles(ctx: GridFSContext, filter?: GridFSFindFilesFilter, opts?: GridFSFindFilesOptions): Promise { + const query = sanitizeFilter(filter, 'findFiles'); + assertFindOptions(opts); + let cursor = ctx.db.collection(filesCollectionName(ctx.bucketName)).find(query); + if (opts?.sort) cursor = cursor.sort(opts.sort as Sort); + if (opts?.skip) cursor = cursor.skip(opts.skip); + if (opts?.limit) cursor = cursor.limit(opts.limit); + const docs = await cursor.toArray(); + return docs.map((doc) => toFileInfo(doc as Record)); +} + +export async function deleteFile(ctx: GridFSContext, id: ObjectId, opts?: GridFSDeleteOptions): Promise { + assertObjectId(id, 'delete', 'id'); + const existing = await getFile(ctx, id); + if (!existing) { + if (opts?.ignoreMissing) return; + throw fileNotFound('delete', ctx, { id }); + } + try { + await ctx.driver.delete(id); + } catch (error) { + throw mapDriverError(error, 'delete', ctx, { id, filename: existing.filename }); + } +} + +export async function renameFile(ctx: GridFSContext, id: ObjectId, filename: string): Promise { + assertObjectId(id, 'rename', 'id'); + assertFilename(filename, 'rename'); + const existing = await getFile(ctx, id); + if (!existing) throw fileNotFound('rename', ctx, { id, filename }); + try { + await ctx.driver.rename(id, filename); + } catch (error) { + throw mapDriverError(error, 'rename', ctx, { id, filename }); + } +} + +export async function dropBucket(ctx: GridFSContext): Promise { + try { + await ctx.driver.drop(); + } catch (error) { + if (!isNamespaceNotFound(error)) throw mapDriverError(error, 'drop', ctx); + } + await dropIfExists(ctx, chunksCollectionName(ctx.bucketName)); + await dropIfExists(ctx, filesCollectionName(ctx.bucketName)); + await dropIfExists(ctx, uploadsCollectionName(ctx.bucketName)); +} + +async function dropIfExists(ctx: GridFSContext, name: string): Promise { + try { + await ctx.db.collection(name).drop(); + } catch (error) { + if (!isNamespaceNotFound(error)) throw mapDriverError(error, 'drop', ctx); + } +} + +async function findConflict(ctx: GridFSContext, filename: string, sha256: string | undefined): Promise { + const files = ctx.db.collection(filesCollectionName(ctx.bucketName)); + const filter = sha256 !== undefined ? { 'metadata.sha256': sha256 } : { filename }; + const docs = await files.find(filter).sort({ uploadDate: -1 }).limit(1).toArray(); + return docs[0] ? toFileInfo(docs[0] as Record) : null; +} + +async function resolveFile(ctx: GridFSContext, idOrName: GridFSIdOrName, operation: string): Promise { + if (idOrName instanceof ObjectId) return getFile(ctx, idOrName); + if (!idOrName || typeof idOrName !== 'object' || typeof idOrName.filename !== 'string') { + throw invalidArgument(operation, 'idOrName', 'idOrName must be an ObjectId or { filename, revision? }'); + } + assertFilename(idOrName.filename, operation); + const revision = idOrName.revision ?? -1; + if (idOrName.revision !== undefined && (typeof revision !== 'number' || !Number.isInteger(revision))) { + throw invalidArgument(operation, 'revision', 'revision must be an integer'); + } + const files = ctx.db.collection(filesCollectionName(ctx.bucketName)); + const cursor = revision >= 0 + ? files.find({ filename: idOrName.filename }).sort({ uploadDate: 1 }).skip(revision).limit(1) + : files.find({ filename: idOrName.filename }).sort({ uploadDate: -1 }).skip(-revision - 1).limit(1); + const docs = await cursor.toArray(); + return docs[0] ? toFileInfo(docs[0] as Record) : null; +} + +async function writeBuffer(ctx: GridFSContext, id: ObjectId, filename: string, payload: Buffer, options: { chunkSizeBytes: number; metadata?: import('mongodb').Document; contentType?: string }): Promise { + const stream = ctx.driver.openUploadStream(id, filename, options); + try { + await endWritable(stream, payload); + } catch (error) { + await cleanupFailedWrite(ctx, id, stream); + throw mapDriverError(error, 'upload', ctx, { id, filename }); + } +} + +async function writeReadable(ctx: GridFSContext, id: ObjectId, filename: string, source: Readable, options: { chunkSizeBytes: number; metadata?: import('mongodb').Document; contentType?: string }): Promise { + const stream = ctx.driver.openUploadStream(id, filename, options); + try { + const { pipeline } = await import('node:stream/promises'); + await pipeline(source, stream); + } catch (error) { + await cleanupFailedWrite(ctx, id, stream); + throw mapDriverError(error, 'upload', ctx, { id, filename }); + } +} + +function endWritable(stream: Writable, payload: Buffer): Promise { + return new Promise((resolve, reject) => { + const fail = (error: Error) => reject(error); + stream.once('error', fail); + stream.once('finish', () => { + stream.removeListener('error', fail); + resolve(); + }); + stream.end(payload); + }); +} + +async function cleanupFailedWrite(ctx: GridFSContext, id: ObjectId, stream: GridFSDriverStream): Promise { + try { + if (typeof stream.abort === 'function') await stream.abort(); + } catch { /* A finished driver stream refuses abort; delete the visible file below. */ } + try { + if (await getFile(ctx, id)) await ctx.driver.delete(id); + } catch { /* The caller still receives the original write error. */ } +} diff --git a/src/capabilities/gridfs/resumable.ts b/src/capabilities/gridfs/resumable.ts new file mode 100644 index 0000000..3edb131 --- /dev/null +++ b/src/capabilities/gridfs/resumable.ts @@ -0,0 +1,238 @@ +import { createHash } from 'node:crypto'; +import type { Readable } from 'node:stream'; +import { ObjectId } from 'mongodb'; +import type { AppendChunksResult, BeginResumableUploadOptions, BeginResumableUploadResult, GridFSContext, GridFSFileInfo, GridFSUploadSession } from './types'; +import { fileExists, fileNotFound, integrityMismatch, invalidArgument, isDuplicateKey, mapDriverError, resumeMismatch, uploadIncomplete } from './errors'; +import { binaryToBuffer, chunksCollectionName, filesCollectionName, sameBytes, toFileInfo, uploadsCollectionName } from './normalize'; +import { + assertChunkSize, + assertContentType, + assertFilename, + assertNonNegativeInteger, + assertObjectId, + assertPositiveLength, + assertSha256, + buildMetadata, + chunkCount, + chunkLengthAt, + isReadableStream, + toNodeBuffer, +} from './validate'; +import { readReadable } from './bytes'; + +export async function beginResumableUpload(ctx: GridFSContext, filename: string, opts: BeginResumableUploadOptions): Promise { + assertFilename(filename, 'beginResumableUpload'); + if (!opts || typeof opts !== 'object') { + throw invalidArgument('beginResumableUpload', 'options', 'length and sha256 are required'); + } + assertPositiveLength(opts.length, 'beginResumableUpload'); + assertSha256(opts.sha256, 'beginResumableUpload'); + assertContentType(opts.contentType, 'beginResumableUpload'); + if (opts.id) assertObjectId(opts.id, 'beginResumableUpload', 'id'); + const chunkSizeBytes = opts.chunkSizeBytes ?? ctx.chunkSizeBytes; + assertChunkSize(chunkSizeBytes, 'beginResumableUpload'); + const metadata = buildMetadata(opts.metadata, undefined, 'beginResumableUpload'); + await ctx.indexes.ensure(); + + const uploadId = opts.id ?? new ObjectId(); + const files = ctx.db.collection(filesCollectionName(ctx.bucketName)); + const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); + if (await files.findOne({ _id: uploadId }, { projection: { _id: 1 } })) { + throw fileExists('beginResumableUpload', ctx, { id: uploadId, filename, sha256: opts.sha256 }); + } + if (await uploads.findOne({ _id: uploadId }, { projection: { _id: 1 } })) { + throw fileExists('beginResumableUpload', ctx, { id: uploadId, filename, sha256: opts.sha256 }); + } + + const now = new Date(); + try { + await uploads.insertOne({ + _id: uploadId, + filename, + length: opts.length, + sha256: opts.sha256, + chunkSizeBytes, + ...(metadata ? { metadata } : {}), + ...(opts.contentType !== undefined ? { contentType: opts.contentType } : {}), + status: 'pending', + nextChunk: 0, + createdAt: now, + updatedAt: now, + }); + } catch (error) { + throw mapDriverError(error, 'beginResumableUpload', ctx, { uploadId, filename }); + } + return { uploadId, nextChunk: 0, chunkSizeBytes }; +} + +export async function appendChunks( + ctx: GridFSContext, + uploadId: ObjectId, + fromChunk: number, + data: Buffer | Uint8Array | Readable, +): Promise { + assertObjectId(uploadId, 'appendChunks', 'uploadId'); + assertNonNegativeInteger(fromChunk, 'appendChunks', 'fromChunk'); + const payload = await payloadBytes(data); + if (payload.length === 0) throw badChunkSize(); + const session = await requirePending(ctx, uploadId, 'appendChunks'); + const nextChunk = await contiguousNext(ctx, uploadId); + if (fromChunk > nextChunk) { + throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); + } + + const total = chunkCount(session.length, session.chunkSizeBytes); + let offset = 0; + let index = fromChunk; + while (offset < payload.length) { + if (index >= total) throw badChunkSize(); + const expected = chunkLengthAt(index, session.length, session.chunkSizeBytes); + if (offset + expected > payload.length) throw badChunkSize(); + const slice = payload.subarray(offset, offset + expected); + if (index < nextChunk) await assertStoredChunk(ctx, uploadId, index, slice); + else await writeChunk(ctx, uploadId, index, slice); + offset += expected; + index += 1; + } + + const advanced = await contiguousNext(ctx, uploadId); + await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).updateOne( + { _id: uploadId }, + { $set: { nextChunk: advanced, updatedAt: new Date() } }, + ); + return { nextChunk: advanced, bytesAccepted: payload.length }; +} + +export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: ObjectId): Promise { + assertObjectId(uploadId, 'finalizeResumableUpload', 'uploadId'); + const session = await requirePending(ctx, uploadId, 'finalizeResumableUpload'); + const files = ctx.db.collection(filesCollectionName(ctx.bucketName)); + if (await files.findOne({ _id: uploadId }, { projection: { _id: 1 } })) { + throw fileExists('finalizeResumableUpload', ctx, { id: uploadId, filename: session.filename, sha256: session.sha256 }); + } + + const expected = chunkCount(session.length, session.chunkSizeBytes); + const hash = createHash('sha256'); + let total = 0; + const missing: number[] = []; + const cursor = ctx.db.collection(chunksCollectionName(ctx.bucketName)).find({ files_id: uploadId }).sort({ n: 1 }); + let seen = 0; + for await (const doc of cursor) { + const n = typeof doc.n === 'number' ? doc.n : -1; + while (seen < n && seen < expected) { + missing.push(seen); + seen += 1; + } + if (n !== seen || n < 0 || n >= expected) { + if (n >= 0) missing.push(n); + continue; + } + const bytes = binaryToBuffer(doc.data, 'finalizeResumableUpload'); + const want = chunkLengthAt(n, session.length, session.chunkSizeBytes); + if (bytes.length !== want) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { + uploadId, + expected: want, + actual: bytes.length, + }); + } + hash.update(bytes); + total += bytes.length; + seen += 1; + } + while (seen < expected) { + missing.push(seen); + seen += 1; + } + if (missing.length > 0) throw uploadIncomplete(ctx.bucketName, uploadId, missing); + if (total !== session.length) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.length, actual: total }); + } + const actual = hash.digest('hex'); + if (actual !== session.sha256) { + throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.sha256, actual }); + } + + const metadata = buildMetadata(session.metadata, session.sha256, 'finalizeResumableUpload'); + try { + await files.insertOne({ + _id: uploadId, + length: session.length, + chunkSize: session.chunkSizeBytes, + uploadDate: new Date(), + filename: session.filename, + ...(session.contentType !== undefined ? { contentType: session.contentType } : {}), + ...(metadata ? { metadata } : {}), + }); + } catch (error) { + if (isDuplicateKey(error)) { + throw fileExists('finalizeResumableUpload', ctx, { id: uploadId, filename: session.filename, sha256: session.sha256 }); + } + throw mapDriverError(error, 'finalizeResumableUpload', ctx, { uploadId, filename: session.filename }); + } + await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).deleteOne({ _id: uploadId }); + const info = await files.findOne({ _id: uploadId }); + if (!info) throw fileNotFound('finalizeResumableUpload', ctx, { id: uploadId, filename: session.filename }); + return toFileInfo(info as Record); +} + +export async function abortResumableUpload(ctx: GridFSContext, uploadId: ObjectId): Promise { + assertObjectId(uploadId, 'abortResumableUpload', 'uploadId'); + const uploads = ctx.db.collection(uploadsCollectionName(ctx.bucketName)); + const session = await uploads.findOne({ _id: uploadId }); + if (!session || session.status === 'aborted' || session.status === 'finalized') return; + const visible = await ctx.db.collection(filesCollectionName(ctx.bucketName)).findOne( + { _id: uploadId }, + { projection: { _id: 1 } }, + ); + if (!visible) { + await ctx.db.collection(chunksCollectionName(ctx.bucketName)).deleteMany({ files_id: uploadId }); + } + await uploads.deleteOne({ _id: uploadId }); +} + +async function payloadBytes(data: Buffer | Uint8Array | Readable): Promise { + if (Buffer.isBuffer(data) || data instanceof Uint8Array) return toNodeBuffer(data); + if (isReadableStream(data)) return readReadable(data); + throw invalidArgument('appendChunks', 'data', 'data must be a Buffer, Uint8Array, or Readable'); +} + +function badChunkSize() { + return invalidArgument('appendChunks', 'data', 'chunk payload must match the session chunk size'); +} + +async function requirePending(ctx: GridFSContext, uploadId: ObjectId, operation: string): Promise { + const doc = await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).findOne({ _id: uploadId }); + if (!doc || doc.status !== 'pending') throw fileNotFound(operation, ctx, { id: uploadId }); + return doc as unknown as GridFSUploadSession; +} + +async function contiguousNext(ctx: GridFSContext, uploadId: ObjectId): Promise { + const docs = await ctx.db.collection(chunksCollectionName(ctx.bucketName)) + .find({ files_id: uploadId }) + .sort({ n: 1 }) + .project({ n: 1 }) + .toArray(); + let expected = 0; + for (const doc of docs) { + if (doc.n !== expected) break; + expected += 1; + } + return expected; +} + +async function assertStoredChunk(ctx: GridFSContext, uploadId: ObjectId, n: number, slice: Buffer): Promise { + const existing = await ctx.db.collection(chunksCollectionName(ctx.bucketName)).findOne({ files_id: uploadId, n }); + if (!existing || !sameBytes(binaryToBuffer(existing.data, 'appendChunks'), slice)) { + throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk: n, expectedNextChunk: n }); + } +} + +async function writeChunk(ctx: GridFSContext, uploadId: ObjectId, n: number, slice: Buffer): Promise { + try { + await ctx.db.collection(chunksCollectionName(ctx.bucketName)).insertOne({ files_id: uploadId, n, data: slice }); + } catch (error) { + if (!isDuplicateKey(error)) throw mapDriverError(error, 'appendChunks', ctx, { uploadId }); + await assertStoredChunk(ctx, uploadId, n, slice); + } +} diff --git a/src/capabilities/gridfs/types.ts b/src/capabilities/gridfs/types.ts new file mode 100644 index 0000000..b0eaecb --- /dev/null +++ b/src/capabilities/gridfs/types.ts @@ -0,0 +1,127 @@ +import type { Readable, Writable } from 'node:stream'; +import type { Db, Document, ObjectId } from 'mongodb'; +import type { Logger } from '../../core/logger'; +import type { NormalizedWritePathPolicy, WritePathSource } from '../write-path-policy'; + +/** Driver default: 255 KiB. */ +export const DEFAULT_CHUNK_SIZE_BYTES = 255 * 1024; + +/** `.uploads` session TTL. MongoDB deletes the session document only. */ +export const UPLOADS_TTL_SECONDS = 604800; + +export interface GridFSFileInfo { + id: ObjectId; + filename: string; + length: number; + chunkSize: number; + uploadDate: Date; + contentType?: string; + metadata?: Record; +} + +export type GridFSIfExists = 'error' | 'skip' | 'duplicate'; + +export interface GridFSUploadOptions { + id?: ObjectId; + metadata?: Record; + contentType?: string; + sha256?: string; + ifExists?: GridFSIfExists; + chunkSizeBytes?: number; +} + +export type GridFSUploadStreamOptions = Omit; + +export type GridFSIdOrName = ObjectId | { filename: string; revision?: number }; + +export interface GridFSDownloadRange { + start?: number; + end?: number; +} + +export interface GridFSFindFilesFilter { + filename?: string; + [key: string]: unknown; +} + +export interface GridFSFindFilesOptions { + limit?: number; + skip?: number; + sort?: Record; +} + +export interface GridFSDeleteOptions { + ignoreMissing?: boolean; +} + +export interface BeginResumableUploadOptions { + length: number; + sha256: string; + chunkSizeBytes?: number; + metadata?: Record; + contentType?: string; + id?: ObjectId; +} + +export interface BeginResumableUploadResult { + uploadId: ObjectId; + nextChunk: number; + chunkSizeBytes: number; +} + +export interface AppendChunksResult { + nextChunk: number; + bytesAccepted: number; +} + +export interface GridFSUploadSession { + _id: ObjectId; + filename: string; + length: number; + sha256: string; + chunkSizeBytes: number; + metadata?: Record; + contentType?: string; + status: 'pending' | 'aborted'; + nextChunk: number; + createdAt: Date; + updatedAt: Date; +} + +export interface GridFSDriverStream extends Writable { + abort?: () => Promise; +} + +export interface GridFSDriver { + openUploadStream(id: ObjectId, filename: string, options: { + chunkSizeBytes: number; + metadata?: Document; + contentType?: string; + }): GridFSDriverStream; + openDownloadStream(id: ObjectId, range?: { start?: number; end?: number }): Readable; + delete(id: ObjectId): Promise; + rename(id: ObjectId, filename: string): Promise; + drop(): Promise; +} + +export interface GridFSIndexLatch { + ensure(): Promise; +} + +export interface GridFSContext { + db: Db; + driver: GridFSDriver; + bucketName: string; + dbName: string; + poolName?: string; + chunkSizeBytes: number; + source: Extract; + policy?: NormalizedWritePathPolicy; + logger?: Pick; + indexes: GridFSIndexLatch; +} + +export interface ParsedRange { + start?: number; + end?: number; +} diff --git a/src/capabilities/gridfs/validate.ts b/src/capabilities/gridfs/validate.ts new file mode 100644 index 0000000..3e03f46 --- /dev/null +++ b/src/capabilities/gridfs/validate.ts @@ -0,0 +1,188 @@ +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { ObjectId, type Document } from 'mongodb'; +import type { GridFSDownloadRange, GridFSFindFilesFilter, GridFSFindFilesOptions, GridFSIfExists, ParsedRange } from './types'; +import { invalidArgument, rangeInvalid } from './errors'; + +const SHA256_HEX = /^[0-9a-f]{64}$/; +const FORBIDDEN_KEYS = new Set(['__proto__', 'prototype', 'constructor']); + +export function isPlainRecord(value: unknown): value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) return false; + if (Buffer.isBuffer(value) || value instanceof Date || value instanceof ObjectId) return false; + const proto = Object.getPrototypeOf(value); + return proto === Object.prototype || proto === null; +} + +export function assertFilename(filename: unknown, operation: string): asserts filename is string { + if (typeof filename !== 'string' || filename.length === 0 || filename.includes('\0')) { + throw invalidArgument(operation, 'filename', 'filename must be a non-empty string'); + } +} + +export function assertBucketName(bucketName: unknown): asserts bucketName is string { + if (typeof bucketName !== 'string' || bucketName.length === 0 || bucketName.includes('\0') || bucketName.includes('$')) { + throw invalidArgument('gridfs', 'bucketName', 'bucketName must be a non-empty string without "$" or NUL'); + } +} + +export function assertChunkSize(value: unknown, operation: string): asserts value is number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw invalidArgument(operation, 'chunkSizeBytes', 'chunkSizeBytes must be a positive integer'); + } +} + +export function assertSha256(value: unknown, operation: string): asserts value is string { + if (typeof value !== 'string' || !SHA256_HEX.test(value)) { + throw invalidArgument(operation, 'sha256', 'sha256 must be a lowercase hex string of 64 characters'); + } +} + +export function assertPositiveLength(value: unknown, operation: string): asserts value is number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value <= 0) { + throw invalidArgument(operation, 'length', 'length must be a positive integer'); + } +} + +export function assertObjectId(value: unknown, operation: string, field: string): asserts value is ObjectId { + if (!(value instanceof ObjectId)) { + throw invalidArgument(operation, field, `${field} must be an ObjectId`); + } +} + +export function assertIfExists(value: unknown): asserts value is GridFSIfExists { + if (value !== 'error' && value !== 'skip' && value !== 'duplicate') { + throw invalidArgument('upload', 'ifExists', 'ifExists must be "error", "skip", or "duplicate"'); + } +} + +export function assertContentType(value: unknown, operation: string): void { + if (value !== undefined && typeof value !== 'string') { + throw invalidArgument(operation, 'contentType', 'contentType must be a string'); + } +} + +export function assertNonNegativeInteger(value: unknown, operation: string, field: string): asserts value is number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw invalidArgument(operation, field, `${field} must be a non-negative integer`); + } +} + +export function isReadableStream(value: unknown): value is Readable { + if (Buffer.isBuffer(value) || value instanceof Uint8Array) return false; + if (value instanceof Readable) return true; + return typeof value === 'object' && value !== null + && typeof (value as Readable).read === 'function' + && typeof (value as Readable).pipe === 'function'; +} + +export function toNodeBuffer(data: Buffer | Uint8Array): Buffer { + if (Buffer.isBuffer(data)) return data; + return Buffer.from(data.buffer, data.byteOffset, data.byteLength); +} + +export function sha256Hex(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +export function chunkCount(length: number, chunkSize: number): number { + return Math.ceil(length / chunkSize); +} + +export function chunkLengthAt(index: number, length: number, chunkSize: number): number { + const count = chunkCount(length, chunkSize); + if (index < 0 || index >= count) return -1; + if (index < count - 1) return chunkSize; + const remainder = length % chunkSize; + return remainder === 0 ? chunkSize : remainder; +} + +export function assertSafeKey(key: string, operation: string): void { + if (FORBIDDEN_KEYS.has(key)) { + throw invalidArgument(operation, key, `key "${key}" is not allowed`); + } +} + +export function buildMetadata(metadata: Record | undefined, sha256: string | undefined, operation: string): Document | undefined { + if (metadata !== undefined && !isPlainRecord(metadata)) { + throw invalidArgument(operation, 'metadata', 'metadata must be a plain object'); + } + if (!metadata && sha256 === undefined) return undefined; + const out: Document = {}; + if (metadata) { + for (const key of Object.keys(metadata)) { + assertSafeKey(key, operation); + out[key] = metadata[key]; + } + } + if (sha256 !== undefined) out.sha256 = sha256; + return out; +} + +export function parseRangeShape(range: GridFSDownloadRange | undefined, operation: string, bucket: string): ParsedRange { + if (range === undefined) return {}; + if (!isPlainRecord(range)) throw rangeInvalid(operation, bucket, undefined, undefined); + const start = range.start; + const end = range.end; + if (start !== undefined && (typeof start !== 'number' || !Number.isSafeInteger(start) || start < 0)) { + throw rangeInvalid(operation, bucket, typeof start === 'number' ? start : undefined, typeof end === 'number' ? end : undefined); + } + if (end !== undefined && (typeof end !== 'number' || !Number.isSafeInteger(end) || end < 0)) { + throw rangeInvalid(operation, bucket, start, typeof end === 'number' ? end : undefined); + } + if (start !== undefined && end !== undefined && start >= end) { + throw rangeInvalid(operation, bucket, start, end); + } + return { start, end }; +} + +export function rangeBounds(range: ParsedRange, length: number, operation: string, bucket: string): { start: number; end: number } | undefined { + if (range.start === undefined && range.end === undefined) return undefined; + const start = range.start ?? 0; + const end = range.end ?? length; + if (start >= end || end > length) { + throw rangeInvalid(operation, bucket, range.start, range.end, length); + } + return { start, end }; +} + +export function sanitizeFilter(filter: GridFSFindFilesFilter | undefined, operation: string): Record { + if (filter === undefined) return {}; + if (!isPlainRecord(filter)) throw invalidArgument(operation, 'filter', 'filter must be a plain object'); + return sanitizeRecord(filter, operation) as Record; +} + +function sanitizeRecord(record: Record, operation: string): Document { + const out: Document = {}; + for (const key of Object.keys(record)) { + assertSafeKey(key, operation); + out[key] = sanitizeValue(record[key], operation); + } + return out; +} + +function sanitizeValue(value: unknown, operation: string): unknown { + if (Array.isArray(value)) return value.map((entry) => sanitizeValue(entry, operation)); + if (isPlainRecord(value)) return sanitizeRecord(value, operation); + return value; +} + +export function assertFindOptions(options: GridFSFindFilesOptions | undefined): void { + if (options === undefined) return; + if (!isPlainRecord(options)) throw invalidArgument('findFiles', 'options', 'options must be a plain object'); + if (options.limit !== undefined) { + if (typeof options.limit !== 'number' || !Number.isSafeInteger(options.limit) || options.limit <= 0) { + throw invalidArgument('findFiles', 'limit', 'limit must be a positive integer'); + } + } + if (options.skip !== undefined) assertNonNegativeInteger(options.skip, 'findFiles', 'skip'); + if (options.sort !== undefined) { + if (!isPlainRecord(options.sort)) throw invalidArgument('findFiles', 'sort', 'sort must be a plain object'); + for (const key of Object.keys(options.sort)) { + assertSafeKey(key, 'findFiles'); + if (options.sort[key] !== 1 && options.sort[key] !== -1) { + throw invalidArgument('findFiles', 'sort', 'sort values must be 1 or -1'); + } + } + } +} diff --git a/src/capabilities/model/index.ts b/src/capabilities/model/index.ts index 25320a4..3b97cbb 100644 --- a/src/capabilities/model/index.ts +++ b/src/capabilities/model/index.ts @@ -96,6 +96,7 @@ import { orchestrateModelUpsertOne, } from './model-mutation-orchestrator'; import { runWithModelWriteSource } from '../write-path-policy'; +import { openModelGridFS } from '../../entry/runtime-gridfs'; import { executeModelVectorSearch, executeProtectedRelationDelete, @@ -233,6 +234,8 @@ export class ModelInstance> { return this.collection.raw(); } + gridfs(bucketName?: string, opts?: { chunkSizeBytes?: number }) { return openModelGridFS(this.runtime, this.dbName, this.poolName, bucketName, opts?.chunkSizeBytes); } + private extendedCollection(): ExtendedModelCollectionLike { return this.collection as ExtendedModelCollectionLike; } diff --git a/src/entry/runtime-core.ts b/src/entry/runtime-core.ts index d70bcf2..024db1d 100644 --- a/src/entry/runtime-core.ts +++ b/src/entry/runtime-core.ts @@ -144,6 +144,7 @@ import { normalizeRuntimeCacheWithLifecycle } from './runtime-cache-normalizer'; import { prepareSshTunnelConnectConfig } from './runtime-ssh'; import { createInitialRuntimeSchemaDslEngine, disposeRuntimeSchemaDslEngine, replaceRuntimeSchemaDslEngine, type SchemaDslEngine } from './runtime-schema-dsl'; import { cleanupFailedRuntimeConnection } from './runtime-connect-cleanup'; +import { openRuntimeGridFS } from './runtime-gridfs'; // All public symbols are re-exported from the barrel file to keep the public API unchanged export * from './runtime-exports'; @@ -466,6 +467,8 @@ export class MonSQLizeRuntime { return dbInstance.collection(name) as CollectionFacade; } + gridfs(bucketName?: string, opts?: { db?: string; pool?: string; chunkSizeBytes?: number }) { return openRuntimeGridFS(this, bucketName, opts); } + db(name?: string): DbFacade { if (name !== undefined) { if (!name || typeof name !== 'string' || !name.trim()) { diff --git a/src/entry/runtime-gridfs.ts b/src/entry/runtime-gridfs.ts new file mode 100644 index 0000000..90960f9 --- /dev/null +++ b/src/entry/runtime-gridfs.ts @@ -0,0 +1,106 @@ +import type { MongoClient } from 'mongodb'; +import type { Logger } from '../core/logger'; +import { ErrorCodes, createError } from '../core/errors'; +import type { MonSQLizeOptions } from '../../types/monsqlize'; +import type { RuntimeDefaults } from '../types/internal/query'; +import { createMonSQLizeGridFSBucket, type MonSQLizeGridFSBucketImpl } from '../capabilities/gridfs'; +import { resolveDatabaseName } from './runtime-db-facade'; + +type GridFSRuntimeState = { + ensureConnected?: () => void; + _connected?: boolean; + options: MonSQLizeOptions; + logger?: Logger; + _logger?: Logger; + _client: MongoClient | null; + _poolManager: Record | null; + _runtimeDefaults?: RuntimeDefaults; +}; + +export function openRuntimeGridFS( + runtime: unknown, + bucketName?: string, + opts?: { db?: string; pool?: string; chunkSizeBytes?: number }, +): MonSQLizeGridFSBucketImpl { + return openGridFS(runtime, { + bucketName, + dbName: opts?.db, + poolName: opts?.pool, + chunkSizeBytes: opts?.chunkSizeBytes, + source: 'db', + }); +} + +export function openModelGridFS( + runtime: unknown, + dbName: string, + poolName: string | undefined, + bucketName?: string, + chunkSizeBytes?: number, +): MonSQLizeGridFSBucketImpl { + return openGridFS(runtime, { bucketName, dbName, poolName, chunkSizeBytes, source: 'model' }); +} + +function openGridFS(runtime: unknown, request: { + bucketName?: string; + dbName?: string; + poolName?: string; + chunkSizeBytes?: number; + source: 'db' | 'model'; +}): MonSQLizeGridFSBucketImpl { + const state = runtime as GridFSRuntimeState; + if (typeof state.ensureConnected === 'function') state.ensureConnected(); + else if (!state._connected) { + throw createError(ErrorCodes.NOT_CONNECTED, 'Database is not connected. Please call connect() first.'); + } + const bucketName = request.bucketName === undefined ? 'fs' : request.bucketName; + const dbName = request.dbName ?? resolveDatabaseName(state.options); + if (!dbName || typeof dbName !== 'string' || !dbName.trim()) { + throw createError(ErrorCodes.INVALID_DATABASE_NAME, 'Database name must be a non-empty string'); + } + const client = resolveClient(state, request.poolName); + return createMonSQLizeGridFSBucket({ + db: client.db(dbName), + bucketName, + dbName, + poolName: request.poolName, + chunkSizeBytes: request.chunkSizeBytes, + source: request.source, + policy: state._runtimeDefaults?.writePathPolicy, + logger: state.logger ?? state._logger, + }); +} + +function resolveClient(state: GridFSRuntimeState, poolName?: string): MongoClient { + if (!poolName) { + if (!state._client) throw createError(ErrorCodes.NOT_CONNECTED, 'Database is not connected. Please call connect() first.'); + return state._client; + } + const poolManager = state._poolManager; + if (!poolManager) { + throw createError(ErrorCodes.NO_POOL_MANAGER, `GridFS requires pool '${poolName}' but no pools are configured.`); + } + const client = readPoolClient(poolManager, poolName); + if (!client) { + const getNames = poolManager['getPoolNames']; + const available = typeof getNames === 'function' ? (getNames as () => string[]).call(poolManager) : []; + const error = createError(ErrorCodes.POOL_NOT_FOUND, `Pool '${poolName}' not found. Available pools: [${available.join(', ')}]`); + (error as unknown as Record)['available'] = available; + throw error; + } + return client; +} + +function readPoolClient(poolManager: Record, poolName: string): MongoClient | null { + const getPoolV1 = poolManager['_getPool'] as ((name: string) => MongoClient | null) | undefined; + const getPoolV2 = poolManager['getPool'] as ((name: string) => MongoClient | null) | undefined; + if (typeof getPoolV1 === 'function') return getPoolV1.call(poolManager, poolName) ?? null; + if (typeof getPoolV2 === 'function') { + try { + return getPoolV2.call(poolManager, poolName) ?? null; + } catch { + return null; + } + } + return null; +} diff --git a/types/gridfs.d.ts b/types/gridfs.d.ts new file mode 100644 index 0000000..b12812a --- /dev/null +++ b/types/gridfs.d.ts @@ -0,0 +1,146 @@ +/** + * GridFS bucket types. + * @since v3.5.0 + */ + +import type { ObjectId } from 'mongodb'; +import type { Readable, Writable } from 'node:stream'; + +/** Normalized GridFS file metadata. Driver-private state is omitted. */ +export interface GridFSFileInfo { + /** `files._id`. */ + id: ObjectId; + filename: string; + /** Byte length. Always >= 0. */ + length: number; + /** Chunk size in bytes. Always > 0. */ + chunkSize: number; + uploadDate: Date; + /** Present when the file was stored with a content type. */ + contentType?: string; + metadata?: Record; +} + +/** Options for `msq.gridfs()`. `db` and `pool` select the native database. */ +export interface GridFSBucketOpenOptions { + /** Defaults to the runtime database. */ + db?: string; + /** Defaults to the runtime connection. A named pool uses that pool's client. */ + pool?: string; + /** Defaults to 261120 (255 KiB). Must be a positive integer. */ + chunkSizeBytes?: number; +} + +/** Options for `model.gridfs()`. Pool and database come from the Model. */ +export interface GridFSModelBucketOpenOptions { + /** Defaults to 261120 (255 KiB). Must be a positive integer. */ + chunkSizeBytes?: number; +} + +/** What to do when uploaded content matches an existing visible file. Default `error`. */ +export type GridFSIfExists = 'error' | 'skip' | 'duplicate'; + +export interface GridFSUploadOptions { + /** Custom `files._id`. An existing id rejects with `GRIDFS_FILE_EXISTS`. */ + id?: ObjectId; + metadata?: Record; + contentType?: string; + /** + * Lowercase 64-character sha256 hex. Stored as `metadata.sha256` and used as the idempotency key. + * When the bytes do not match, upload rejects with `GRIDFS_INTEGRITY_MISMATCH` before the file is visible. + */ + sha256?: string; + /** Default `error`. */ + ifExists?: GridFSIfExists; + /** Overrides the bucket chunk size for this file only. */ + chunkSizeBytes?: number; +} + +/** `uploadStream` writes `sha256` into metadata and does not support `ifExists`. */ +export interface GridFSUploadStreamOptions extends Omit {} + +/** + * Download target. `revision` follows the driver: `0` is the oldest file with that name, + * `-1` (default) is the newest. + */ +export type GridFSIdOrName = ObjectId | { filename: string; revision?: number }; + +/** + * Inclusive `start` and exclusive `end`, matching `GridFSBucketReadStreamOptions`. + * Omitted `start` means 0. Omitted `end` means the file length. + */ +export interface GridFSDownloadRange { + start?: number; + end?: number; +} + +export interface GridFSFindFilesFilter { + filename?: string; + [key: string]: unknown; +} + +export interface GridFSFindFilesOptions { + /** Positive integer. Omit for no limit. */ + limit?: number; + /** Non-negative integer. Default 0. */ + skip?: number; + sort?: Record; +} + +export interface GridFSDeleteOptions { + /** Default false. When true, a missing file resolves instead of `GRIDFS_FILE_NOT_FOUND`. */ + ignoreMissing?: boolean; +} + +export interface BeginResumableUploadOptions { + /** Final byte length. Positive integer. */ + length: number; + /** Final lowercase sha256 hex. Checked by `finalizeResumableUpload`. */ + sha256: string; + chunkSizeBytes?: number; + metadata?: Record; + contentType?: string; + /** Also the id returned as `uploadId` and the final `files._id`. */ + id?: ObjectId; +} + +export interface BeginResumableUploadResult { + uploadId: ObjectId; + /** Next chunk index to write. A new session returns 0. */ + nextChunk: number; + chunkSizeBytes: number; +} + +export interface AppendChunksResult { + nextChunk: number; + bytesAccepted: number; +} + +/** + * Bucket returned by `msq.gridfs()` and `model.gridfs()`. + * Reads do not consult `writePathPolicy`. Writes use category `gridfs`. + * `drop` uses category `management`. + */ +export interface MonSQLizeGridFSBucket { + readonly bucketName: string; + readonly dbName: string; + readonly chunkSizeBytes: number; + upload(filename: string, data: Buffer | Uint8Array | Readable, opts?: GridFSUploadOptions): Promise; + uploadStream(filename: string, opts?: GridFSUploadStreamOptions): Writable & { id: ObjectId; done: Promise }; + download(idOrName: GridFSIdOrName): Promise; + downloadStream(idOrName: GridFSIdOrName, range?: GridFSDownloadRange): Readable; + /** Returns null when the id is absent. */ + getFile(id: ObjectId): Promise; + findFiles(filter?: GridFSFindFilesFilter, opts?: GridFSFindFilesOptions): Promise; + delete(id: ObjectId, opts?: GridFSDeleteOptions): Promise; + rename(id: ObjectId, filename: string): Promise; + /** Drops `.files`, `.chunks`, and `.uploads` when present. */ + drop(): Promise; + beginResumableUpload(filename: string, opts: BeginResumableUploadOptions): Promise; + appendChunks(uploadId: ObjectId, fromChunk: number, data: Buffer | Uint8Array | Readable): Promise; + finalizeResumableUpload(uploadId: ObjectId): Promise; + /** Deletes the session and any chunks. A missing or already aborted session resolves. */ + abortResumableUpload(uploadId: ObjectId): Promise; + /** Creates the standard files/chunks indexes, `metadata.sha256`, and the uploads TTL index. */ + ensureIndexes(): Promise; +} diff --git a/types/index.d.ts b/types/index.d.ts index 01b5b14..378bca3 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -2,6 +2,7 @@ export * from './base'; export * from './collection'; export * from './data-tasks'; export * from './expression'; +export * from './gridfs'; export * from './lock'; export * from './model'; export * from './mongodb'; diff --git a/types/model.d.ts b/types/model.d.ts index 9d3ca2f..2073267 100644 --- a/types/model.d.ts +++ b/types/model.d.ts @@ -706,6 +706,8 @@ export interface ModelInstance extends SearchIndexManagement { * Does not drop, rename, or rebuild conflicting indexes. */ ensureIndexes(options?: ModelEnsureIndexesOptions): Promise; + /** Opens a GridFS bucket bound to this Model's pool and database. Default bucket name is `fs`. @since v3.5.0 */ + gridfs(bucketName?: string, opts?: import('./gridfs').GridFSModelBucketOpenOptions): import('./gridfs').MonSQLizeGridFSBucket; /** * Drops the specified index by name. * @param name Index name. diff --git a/types/monsqlize.d.ts b/types/monsqlize.d.ts index a184e58..e48e9ee 100644 --- a/types/monsqlize.d.ts +++ b/types/monsqlize.d.ts @@ -37,6 +37,7 @@ export interface SSHConfig { } import type { Collection, CursorValueNormalizer, CursorValueType, DbAccessor, HealthView } from './collection'; +import type { GridFSBucketOpenOptions, MonSQLizeGridFSBucket } from './gridfs'; import type { DataTaskJobError } from './data-tasks'; import type { Lock, LockOptions, LockStats } from './lock'; import type { ModelAutoIndexOptions, ModelDescriptor, ModelEnsureAllIndexesOptions, ModelIndexEnsureSummary, ModelInstance } from './model'; @@ -335,6 +336,12 @@ export interface MonSQLizeInstance { * @param name Target database name; omit to use the default database. */ db(name?: string): DbAccessor; + /** + * Open a GridFS bucket. The default bucket name is `fs`. + * Call after `connect()`; otherwise this throws `NOT_CONNECTED`. + * @since v3.5.0 + */ + gridfs(bucketName?: string, opts?: GridFSBucketOpenOptions): MonSQLizeGridFSBucket; /** * Switch to the given database and return its `collection` and `model` accessors. * @param name Target database name. @@ -557,6 +564,8 @@ export default class MonSQLize implements MonSQLizeInstance { collection(name: string): Collection; collection(name: string): Collection; db(name?: string): DbAccessor; + /** @since v3.5.0 */ + gridfs(bucketName?: string, opts?: GridFSBucketOpenOptions): MonSQLizeGridFSBucket; use(name: string): { collection: (collectionName: string) => Collection; model: ModelBindingAccessor; diff --git a/types/runtime.d.ts b/types/runtime.d.ts index 1d53a4c..3f26be1 100644 --- a/types/runtime.d.ts +++ b/types/runtime.d.ts @@ -343,6 +343,8 @@ export declare class ModelInstance implements ModelInstanceCont createIndexes(specs: Array<{ key: unknown; } & Record>): Promise; listIndexes(): Promise[]>; ensureIndexes(options?: ModelEnsureIndexesOptions): Promise; + /** @since v3.5.0 */ + gridfs(bucketName?: string, opts?: import('./gridfs').GridFSModelBucketOpenOptions): import('./gridfs').MonSQLizeGridFSBucket; dropIndex(name: string): Promise; dropIndexes(): Promise; createSearchIndex(spec: import('./search-index').SearchIndexSpec, options?: import('./search-index').CreateSearchIndexOptions): Promise; From 3702ec0134e9c118d8c8e8331ce2d1667a66c5d5 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 27 Oct 2026 03:14:25 +0000 Subject: [PATCH 3/7] test(gridfs): unit, memory-server integration, and types Cover policy, ranges, ifExists, resumable upload, and Model routing. Register the suites and the tsd file in the type-check script. Co-authored-by: rocky --- package.json | 2 +- test/integration/gridfs/gridfs.test.ts | 214 ++++++++++++++ test/run-tests.cjs | 2 + test/types/gridfs-usage.test-d.ts | 74 +++++ test/unit/gridfs/gridfs.test.ts | 369 +++++++++++++++++++++++++ 5 files changed, 660 insertions(+), 1 deletion(-) create mode 100644 test/integration/gridfs/gridfs.test.ts create mode 100644 test/types/gridfs-usage.test-d.ts create mode 100644 test/unit/gridfs/gridfs.test.ts diff --git a/package.json b/package.json index 3776445..874064c 100644 --- a/package.json +++ b/package.json @@ -136,7 +136,7 @@ "test:coverage": "node scripts/run-coverage.cjs", "test:audit": "npm audit --omit=dev --registry=https://registry.npmjs.org/", "test": "npm run build && npm run build:tests && node test/run-tests.cjs", - "type-check": "npm run build && tsc -p tsconfig.json --noEmit && tsd --files test/types/root-import.test-d.ts --files test/types/cache-usage.test-d.ts --files test/types/model-usage.test-d.ts --files test/types/pool-usage.test-d.ts --files test/types/sync-usage.test-d.ts --files test/types/slow-query-log-usage.test-d.ts --files test/types/saga-usage.test-d.ts --files test/types/data-tasks-usage.test-d.ts --files test/types/search-index-usage.test-d.ts", + "type-check": "npm run build && tsc -p tsconfig.json --noEmit && tsd --files test/types/root-import.test-d.ts --files test/types/cache-usage.test-d.ts --files test/types/model-usage.test-d.ts --files test/types/pool-usage.test-d.ts --files test/types/sync-usage.test-d.ts --files test/types/slow-query-log-usage.test-d.ts --files test/types/saga-usage.test-d.ts --files test/types/data-tasks-usage.test-d.ts --files test/types/search-index-usage.test-d.ts --files test/types/gridfs-usage.test-d.ts", "check:test-language": "node scripts/check-test-language.cjs", "check:docs-examples": "node scripts/validation/check-doc-example-matrix.cjs", "check:release-candidate": "node scripts/check-release-candidate.cjs", diff --git a/test/integration/gridfs/gridfs.test.ts b/test/integration/gridfs/gridfs.test.ts new file mode 100644 index 0000000..cf26445 --- /dev/null +++ b/test/integration/gridfs/gridfs.test.ts @@ -0,0 +1,214 @@ +import { createHash } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { after, before, describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ObjectId } from 'mongodb'; +import { createMemoryServerBootstrap } from '../../bootstrap/memory-server'; + +const MonSQLize = require('../../../dist/cjs/index.cjs'); + +function sha(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +function codeOf(error: unknown): string | undefined { + return error && typeof error === 'object' && 'code' in error ? String((error as { code: unknown }).code) : undefined; +} + +function detailsOf(error: unknown): Record { + const details = error && typeof error === 'object' ? (error as { details?: unknown[] }).details : undefined; + return (details?.[0] ?? {}) as Record; +} + +async function readStream(stream: AsyncIterable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +describe('gridfs memory-server integration', () => { + const bootstrap = createMemoryServerBootstrap(); + let uri = ''; + + before(async () => { + uri = (await bootstrap.setup()).uri; + }); + + after(async () => { + await bootstrap.teardown(); + MonSQLize.Model._clear(); + }); + + async function connect(databaseName: string, options: Record = {}) { + const runtime = new MonSQLize({ type: 'mongodb', databaseName, config: { uri }, ...options }); + await runtime.connect(); + return runtime; + } + + it('round-trips at least three chunks, ranges, streams, and ifExists', async () => { + const runtime = await connect('gridfs_roundtrip'); + try { + const bucket = runtime.gridfs('artifacts', { chunkSizeBytes: 8 }); + const payload = Buffer.from('abcdefghijklmnopqrst'); + const digest = sha(payload); + const info = await bucket.upload('report.bin', payload, { + sha256: digest, + metadata: { tenantId: 't1', sha256: 'caller-value' }, + contentType: 'application/octet-stream', + }); + assert.equal(info.length, payload.length); + assert.equal(info.chunkSize, 8); + assert.equal(info.metadata?.sha256, digest); + assert.equal(info.metadata?.tenantId, 't1'); + assert.deepEqual(await bucket.download(info.id), payload); + assert.deepEqual(await bucket.download({ filename: 'report.bin' }), payload); + + const part1 = await readStream(bucket.downloadStream(info.id, { start: 0, end: 8 })); + const part2 = await readStream(bucket.downloadStream(info.id, { start: 8 })); + assert.deepEqual(Buffer.concat([part1, part2]), payload); + + await assert.rejects(() => bucket.upload('other.bin', payload, { sha256: digest }), (error: unknown) => codeOf(error) === 'GRIDFS_FILE_EXISTS'); + const skipped = await bucket.upload('other.bin', payload, { sha256: digest, ifExists: 'skip' }); + assert.equal(String(skipped.id), String(info.id)); + + const streamed = bucket.uploadStream('stream.bin', { sha256: digest }); + Readable.from([payload]).pipe(streamed); + const streamedInfo = await streamed.done; + assert.deepEqual(await readStream(bucket.downloadStream(streamedInfo.id)), payload); + const named = await bucket.upload('notes.txt', Buffer.from('notes')); + const sameName = await bucket.upload('notes.txt', Buffer.from('notes-2'), { ifExists: 'skip' }); + assert.equal(String(sameName.id), String(named.id)); + const copy = await bucket.upload('notes.txt', Buffer.from('notes-3'), { ifExists: 'duplicate' }); + assert.notEqual(String(copy.id), String(named.id)); + const latest = await bucket.download({ filename: 'notes.txt', revision: -1 }); + assert.equal(latest.toString(), 'notes-3'); + + await assert.rejects(() => bucket.upload('bad.bin', Buffer.from('nope'), { sha256: digest }), (error: unknown) => codeOf(error) === 'GRIDFS_INTEGRITY_MISMATCH'); + assert.equal(await bucket.getFile(new ObjectId()), null); + await assert.rejects(() => bucket.download({ filename: 'missing.bin' }), (error: unknown) => codeOf(error) === 'GRIDFS_FILE_NOT_FOUND'); + const ranged = bucket.downloadStream(info.id, { start: 3, end: 3 }); + const rangeError = await new Promise((resolve) => ranged.once('error', resolve)); + assert.equal(codeOf(rangeError), 'GRIDFS_RANGE_INVALID'); + assert.equal(detailsOf(rangeError).length, undefined); + const tooFar = bucket.downloadStream(info.id, { start: payload.length }); + const tooFarError = await new Promise((resolve) => tooFar.once('error', resolve)); + assert.equal(codeOf(tooFarError), 'GRIDFS_RANGE_INVALID'); + assert.equal(detailsOf(tooFarError).length, payload.length); + + await bucket.rename(named.id, 'notes-renamed.txt'); + assert.equal((await bucket.getFile(named.id))?.filename, 'notes-renamed.txt'); + await assert.rejects(() => bucket.delete(new ObjectId()), (error: unknown) => codeOf(error) === 'GRIDFS_FILE_NOT_FOUND'); + await bucket.delete(new ObjectId(), { ignoreMissing: true }); + await bucket.delete(named.id); + assert.equal(await bucket.getFile(named.id), null); + const found = await bucket.findFiles({ 'metadata.sha256': digest }, { sort: { uploadDate: 1 } }); + assert.ok(found.some((file: { id: unknown }) => String(file.id) === String(info.id))); + } finally { + await runtime.close(); + } + }); + + it('resumes across a reconnect, then rejects incomplete, mismatched, and aborted uploads', async () => { + const databaseName = 'gridfs_resume'; + const bucketName = 'resumable'; + const payload = Buffer.from('abcdefghijklmnopqrst'); + const digest = sha(payload); + const first = await connect(databaseName); + let started: { uploadId: InstanceType }; + try { + const bucket = first.gridfs(bucketName, { chunkSizeBytes: 8 }); + started = await bucket.beginResumableUpload('big.bin', { length: payload.length, sha256: digest, chunkSizeBytes: 8, contentType: 'application/octet-stream' }); + assert.equal((await bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 8))).nextChunk, 1); + assert.equal(await bucket.getFile(started.uploadId), null); + } finally { + await first.close(); + } + + const second = await connect(databaseName); + try { + const resumed = second.gridfs(bucketName); + const next = await resumed.appendChunks(started.uploadId, 1, payload.subarray(8)); + assert.equal(next.nextChunk, 3); + const info = await resumed.finalizeResumableUpload(started.uploadId); + assert.equal(info.metadata?.sha256, digest); + assert.equal(info.contentType, 'application/octet-stream'); + assert.deepEqual(await resumed.download(info.id), payload); + const skipped = await resumed.upload('again.bin', payload, { sha256: digest, ifExists: 'skip' }); + assert.equal(String(skipped.id), String(info.id)); + + const incomplete = await resumed.beginResumableUpload('partial.bin', { length: payload.length, sha256: digest, chunkSizeBytes: 8 }); + await resumed.appendChunks(incomplete.uploadId, 0, payload.subarray(0, 8)); + await assert.rejects(() => resumed.finalizeResumableUpload(incomplete.uploadId), (error: unknown) => codeOf(error) === 'GRIDFS_UPLOAD_INCOMPLETE'); + assert.equal(await resumed.getFile(incomplete.uploadId), null); + await assert.rejects(() => resumed.appendChunks(incomplete.uploadId, 5, payload.subarray(0, 8)), (error: unknown) => codeOf(error) === 'GRIDFS_RESUME_MISMATCH'); + await assert.rejects(() => resumed.appendChunks(incomplete.uploadId, 1, Buffer.from('short')), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + + const mismatched = await resumed.beginResumableUpload('wrong.bin', { length: 8, sha256: sha(Buffer.from('xxxxxxxx')), chunkSizeBytes: 8 }); + await resumed.appendChunks(mismatched.uploadId, 0, Buffer.from('abcdefgh')); + await assert.rejects(() => resumed.finalizeResumableUpload(mismatched.uploadId), (error: unknown) => codeOf(error) === 'GRIDFS_INTEGRITY_MISMATCH'); + assert.equal(await resumed.getFile(mismatched.uploadId), null); + + await resumed.abortResumableUpload(incomplete.uploadId); + await resumed.abortResumableUpload(incomplete.uploadId); + assert.equal((await second.collection(`${bucketName}.chunks`).find({ files_id: incomplete.uploadId })).length, 0); + assert.equal((await second.collection(`${bucketName}.files`).find({ _id: incomplete.uploadId })).length, 0); + await resumed.drop(); + assert.equal((await second.collection(`${bucketName}.files`).find({})).length, 0); + assert.equal((await second.collection(`${bucketName}.uploads`).find({})).length, 0); + } finally { + await second.close(); + } + }); + + it('enforces gridfs and management policy for instance and Model buckets', async () => { + MonSQLize.Model.define('gridfs_files', { schema: {} }); + const blocked = await connect('gridfs_policy_block', { writePathPolicy: { default: 'model-only' } }); + try { + const model = blocked.model('gridfs_files'); + assert.throws(() => blocked.gridfs('locked').upload('a.bin', Buffer.from('x')), (error: unknown) => { + assert.equal(codeOf(error), 'INVALID_OPERATION'); + assert.equal(detailsOf(error).category, 'gridfs'); + assert.equal(detailsOf(error).reason, 'WRITE_PATH_POLICY'); + assert.equal(detailsOf(error).source, 'db'); + return true; + }); + assert.throws(() => model.gridfs('locked').upload('a.bin', Buffer.from('x')), (error: unknown) => detailsOf(error).source === 'model'); + assert.equal(await model.gridfs('locked').getFile(new ObjectId()), null); + assert.throws(() => blocked.gridfs('locked').drop(), (error: unknown) => detailsOf(error).category === 'management'); + await model.gridfs('locked').drop(); + } finally { + await blocked.close(); + } + + const warnings: unknown[] = []; + const warned = await connect('gridfs_policy_warn', { + logger: { debug() {}, info() {}, error() {}, warn: (...args: unknown[]) => { warnings.push(args); } }, + writePathPolicy: { default: { mode: 'model-only', onViolation: 'warn' } }, + }); + try { + const warnedInfo = await warned.gridfs('warned', { chunkSizeBytes: 8 }).upload('a.bin', Buffer.from('abcdefghi')); + assert.equal(warnedInfo.length, 9); + assert.ok(warnings.length > 0); + } finally { + await warned.close(); + } + + const allowed = await connect('gridfs_policy_allow', { writePathPolicy: { default: { mode: 'model-only', gridfs: 'allow' } } }); + try { + const allowedModel = allowed.model('gridfs_files'); + const fromModel = await allowedModel.gridfs('shared').upload('from-model.bin', Buffer.from('model-bytes'), { sha256: sha(Buffer.from('model-bytes')) }); + assert.deepEqual(await allowed.gridfs('shared').download(fromModel.id), Buffer.from('model-bytes')); + await allowed.gridfs('shared').ensureIndexes(); + const indexes = await allowed.collection('shared.files').listIndexes(); + assert.ok(indexes.some((index: { key?: Record }) => index.key?.['metadata.sha256'] === 1)); + } finally { + await allowed.close(); + MonSQLize.Model._clear(); + } + }); + + it('throws NOT_CONNECTED before connect', () => { + const runtime = new MonSQLize({ type: 'mongodb', databaseName: 'gridfs_offline', config: { uri } }); + assert.throws(() => runtime.gridfs(), (error: unknown) => codeOf(error) === 'NOT_CONNECTED'); + }); +}); diff --git a/test/run-tests.cjs b/test/run-tests.cjs index c0cdf37..f8c298f 100644 --- a/test/run-tests.cjs +++ b/test/run-tests.cjs @@ -53,6 +53,7 @@ const UNIT = [ 'test/unit/expression/expression.test.js', 'test/unit/expression/operators.test.js', 'test/unit/errors/errors.test.js', + 'test/unit/gridfs/gridfs.test.js', 'test/unit/search-indexes/search-indexes.test.js', 'test/unit/management/management.test.js', 'test/unit/writes/batch.test.js', @@ -133,6 +134,7 @@ const UNIT = [ const INTEGRATION = [ 'test/integration/search-indexes/unsupported.test.js', + 'test/integration/gridfs/gridfs.test.js', 'test/integration/mongodb/review-repair-regressions.test.js', 'test/integration/model/review-model-regressions.test.js', 'test/integration/slow-query-log/review-storage-regressions.test.js', diff --git a/test/types/gridfs-usage.test-d.ts b/test/types/gridfs-usage.test-d.ts new file mode 100644 index 0000000..ae1e002 --- /dev/null +++ b/test/types/gridfs-usage.test-d.ts @@ -0,0 +1,74 @@ +import { expectError, expectType } from 'tsd'; +import { ObjectId } from 'mongodb'; +import { Readable, type Writable } from 'node:stream'; +import MonSQLize, { + ErrorCodes, + type BeginResumableUploadResult, + type GridFSFileInfo, + type MonSQLizeGridFSBucket, +} from 'monsqlize'; + +const runtime = new MonSQLize({ + type: 'mongodb', + databaseName: 'app', + config: { uri: 'mongodb://localhost:27017' }, + writePathPolicy: { default: { mode: 'model-only', gridfs: 'allow' } }, +}); + +const bucket: MonSQLizeGridFSBucket = runtime.gridfs('artifacts', { db: 'app', pool: 'archive', chunkSizeBytes: 1024 }); +const fileId = new ObjectId(); +const payload = Buffer.from('gridfs'); + +expectType>(bucket.upload('report.bin', payload, { + id: fileId, + sha256: 'a'.repeat(64), + metadata: { tenantId: 't1' }, + contentType: 'application/octet-stream', + ifExists: 'skip', + chunkSizeBytes: 1024, +})); +expectType>(bucket.upload('report.bin', Readable.from([payload]))); + +const upload = bucket.uploadStream('report.bin', { sha256: 'b'.repeat(64) }); +expectType(upload.id); +expectType }>(upload); +expectType>(upload.done); +expectError(bucket.uploadStream('report.bin', { ifExists: 'skip' })); + +expectType>(bucket.download(fileId)); +expectType>(bucket.download({ filename: 'report.bin', revision: -1 })); +expectType(bucket.downloadStream(fileId, { start: 0, end: 4 })); +expectType>(bucket.getFile(fileId)); +expectType>(bucket.findFiles({ filename: 'report.bin', 'metadata.sha256': 'a'.repeat(64) }, { limit: 10, skip: 0, sort: { uploadDate: -1 } })); +expectType>(bucket.delete(fileId, { ignoreMissing: true })); +expectType>(bucket.rename(fileId, 'renamed.bin')); +expectType>(bucket.drop()); +expectType>(bucket.ensureIndexes()); + +declare const resumed: BeginResumableUploadResult; +expectType>(bucket.beginResumableUpload('big.bin', { length: 4, sha256: 'c'.repeat(64) })); +expectType>(bucket.appendChunks(resumed.uploadId, resumed.nextChunk, payload)); +expectType>(bucket.finalizeResumableUpload(resumed.uploadId)); +expectType>(bucket.abortResumableUpload(resumed.uploadId)); + +expectType(bucket.bucketName); +expectType(bucket.dbName); +expectType(bucket.chunkSizeBytes); +expectType(ErrorCodes.GRIDFS_FILE_NOT_FOUND); +expectType(ErrorCodes.GRIDFS_FILE_EXISTS); +expectType(ErrorCodes.GRIDFS_INTEGRITY_MISMATCH); +expectType(ErrorCodes.GRIDFS_UPLOAD_INCOMPLETE); +expectType(ErrorCodes.GRIDFS_RESUME_MISMATCH); +expectType(ErrorCodes.GRIDFS_RANGE_INVALID); + +expectError(new MonSQLize({ + type: 'mongodb', + databaseName: 'app', + writePathPolicy: { default: { gridfs: 'sometimes' } }, +})); + +const model = runtime.model('files'); +expectType(model.gridfs()); +expectType(model.gridfs('artifacts', { chunkSizeBytes: 255 * 1024 })); +expectError(model.gridfs('artifacts', { pool: 'archive' })); +expectError(model.gridfs('artifacts', { db: 'other' })); diff --git a/test/unit/gridfs/gridfs.test.ts b/test/unit/gridfs/gridfs.test.ts new file mode 100644 index 0000000..521b63e --- /dev/null +++ b/test/unit/gridfs/gridfs.test.ts @@ -0,0 +1,369 @@ +import { Readable } from 'node:stream'; +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { ObjectId } from 'mongodb'; +import { createHash } from 'node:crypto'; +import { createMonSQLizeGridFSBucket } from '../../../src/capabilities/gridfs'; +import { openModelGridFS, openRuntimeGridFS } from '../../../src/entry/runtime-gridfs'; +import { normalizeWritePathPolicy } from '../../../src/capabilities/write-path-policy'; +import { sha256Hex } from '../../../src/capabilities/gridfs/validate'; + +type Doc = Record; + +function sha(data: Buffer): string { + return createHash('sha256').update(data).digest('hex'); +} + +function sameId(left: unknown, right: unknown): boolean { + if (left === right) return true; + if (left && right && typeof (left as { equals?: (value: unknown) => boolean }).equals === 'function') { + return (left as { equals: (value: unknown) => boolean }).equals(right); + } + return false; +} + +function readPath(doc: Doc, path: string): unknown { + return path.split('.').reduce((current, key) => { + if (!current || typeof current !== 'object') return undefined; + return (current as Doc)[key]; + }, doc); +} + +function matches(doc: Doc, filter: Doc): boolean { + return Object.entries(filter).every(([key, expected]) => { + const actual = readPath(doc, key); + if (expected && typeof expected === 'object' && typeof (expected as { equals?: unknown }).equals === 'function') { + return sameId(actual, expected); + } + return actual === expected; + }); +} + +function sortRows(rows: Doc[], spec?: Doc): Doc[] { + if (!spec) return rows; + const [key, direction] = Object.entries(spec)[0] ?? []; + if (!key) return rows; + const dir = direction === -1 ? -1 : 1; + return [...rows].sort((left, right) => { + const a = readPath(left, key) as number | Date; + const b = readPath(right, key) as number | Date; + if (a < b) return -1 * dir; + if (a > b) return 1 * dir; + return 0; + }); +} + +class MemoryCollection { + docs: Doc[] = []; + indexes: Doc[] = []; + + private matched(filter: Doc): Doc[] { + return this.docs.filter((doc) => matches(doc, filter)); + } + + async findOne(filter: Doc, options?: { sort?: Doc }): Promise { + const [first] = sortRows(this.matched(filter), options?.sort); + return first ? { ...first } : null; + } + + find(filter: Doc) { + let rows = this.matched(filter); + const cursor = { + sort: (spec: Doc) => { rows = sortRows(rows, spec); return cursor; }, + skip: (count: number) => { rows = rows.slice(count); return cursor; }, + limit: (count: number) => { rows = rows.slice(0, count); return cursor; }, + project: () => cursor, + toArray: async () => rows.map((row) => ({ ...row })), + async *[Symbol.asyncIterator]() { + for (const row of rows) yield { ...row }; + }, + }; + return cursor; + } + + async insertOne(doc: Doc) { + const duplicateId = doc._id !== undefined && this.docs.some((row) => sameId(row._id, doc._id)); + const duplicateChunk = doc.files_id !== undefined && doc.n !== undefined + && this.docs.some((row) => sameId(row.files_id, doc.files_id) && row.n === doc.n); + if (duplicateId || duplicateChunk) { + const error = new Error('E11000 duplicate key') as Error & { code: number }; + error.code = 11000; + throw error; + } + this.docs.push({ ...doc }); + return { insertedId: doc._id }; + } + + async updateOne(filter: Doc, update: { $set?: Doc }) { + const row = this.docs.find((doc) => matches(doc, filter)); + if (!row) return { matchedCount: 0 }; + Object.assign(row, update.$set ?? {}); + return { matchedCount: 1 }; + } + + async deleteOne(filter: Doc) { + const index = this.docs.findIndex((doc) => matches(doc, filter)); + if (index < 0) return { deletedCount: 0 }; + this.docs.splice(index, 1); + return { deletedCount: 1 }; + } + + async deleteMany(filter: Doc) { + const before = this.docs.length; + this.docs = this.docs.filter((doc) => !matches(doc, filter)); + return { deletedCount: before - this.docs.length }; + } + + async createIndex(keys: Doc, options?: Doc) { + this.indexes.push({ key: keys, ...(options ?? {}) }); + return 'index'; + } + + async drop() { + this.docs = []; + } +} + +class MemoryDb { + private readonly collections = new Map(); + collection(name: string): MemoryCollection { + let collection = this.collections.get(name); + if (!collection) { + collection = new MemoryCollection(); + this.collections.set(name, collection); + } + return collection; + } +} + +function codeOf(error: unknown): string | undefined { + return error && typeof error === 'object' && 'code' in error ? String((error as { code: unknown }).code) : undefined; +} + +function detailsOf(error: unknown): Record { + const details = error && typeof error === 'object' ? (error as { details?: unknown[] }).details : undefined; + return (details?.[0] ?? {}) as Record; +} + +function bucketFor(options: { policy?: ReturnType; source?: 'db' | 'model'; driver?: boolean } = {}) { + const db = new MemoryDb(); + const calls: string[] = []; + const bytes = new Map(); + const driver = { + openUploadStream(id: ObjectId, filename: string, streamOptions: { chunkSizeBytes: number; metadata?: Doc; contentType?: string }) { + calls.push('upload'); + const { PassThrough } = require('node:stream') as typeof import('node:stream'); + const stream = new PassThrough(); + const parts: Buffer[] = []; + stream.on('data', (chunk: Buffer) => parts.push(Buffer.from(chunk))); + stream.on('finish', () => { + const data = Buffer.concat(parts); + bytes.set(id.toHexString(), data); + void db.collection('fs.files').insertOne({ + _id: id, + filename, + length: data.length, + chunkSize: streamOptions.chunkSizeBytes, + uploadDate: new Date(), + ...(streamOptions.contentType ? { contentType: streamOptions.contentType } : {}), + ...(streamOptions.metadata ? { metadata: streamOptions.metadata } : {}), + }); + }); + (stream as unknown as { abort: () => Promise }).abort = async () => { + calls.push('abort'); + bytes.delete(id.toHexString()); + }; + return stream; + }, + openDownloadStream(id: ObjectId, range?: { start?: number; end?: number }) { + const data = bytes.get(id.toHexString()); + if (!data) return Readable.from(async function* fail() { throw Object.assign(new Error('File not found'), { code: 'ENOENT' }); }()); + const start = range?.start ?? 0; + const end = range?.end ?? data.length; + return Readable.from([data.subarray(start, end)]); + }, + async delete(id: ObjectId) { + calls.push('delete'); + const removed = await db.collection('fs.files').deleteOne({ _id: id }); + bytes.delete(id.toHexString()); + if (!removed.deletedCount) throw new Error(`File not found for id ${id}`); + }, + async rename(id: ObjectId, filename: string) { + calls.push('rename'); + const updated = await db.collection('fs.files').updateOne({ _id: id }, { $set: { filename } }); + if (!updated.matchedCount) throw new Error(`File with id ${id} not found`); + }, + async drop() { + calls.push('drop'); + await db.collection('fs.files').drop(); + await db.collection('fs.chunks').drop(); + }, + }; + const bucket = createMonSQLizeGridFSBucket({ + db: db as never, + bucketName: 'fs', + dbName: 'app', + source: options.source ?? 'db', + policy: options.policy, + driver: options.driver === false ? undefined : driver as never, + }); + return { bucket, db, calls, bytes }; +} + +describe('gridfs bucket', () => { + it('rejects an empty filename, a bad sha256, and a polluted filter', async () => { + const { bucket } = bucketFor(); + await assert.rejects(() => bucket.upload('', Buffer.from('x')), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + await assert.rejects(() => bucket.upload('a.bin', Buffer.from('x'), { sha256: 'ABC' }), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + await assert.rejects(() => bucket.findFiles({ ['__proto__']: { admin: true } } as never), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + await assert.rejects(() => bucket.findFiles({}, { limit: 0 }), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + }); + + it('checks sha256 before any GridFS write and keeps ifExists semantics', async () => { + const { bucket, calls } = bucketFor(); + const payload = Buffer.from('alpha'); + const digest = sha(payload); + await assert.rejects( + () => bucket.upload('a.bin', Buffer.from('other'), { sha256: digest }), + (error: unknown) => codeOf(error) === 'GRIDFS_INTEGRITY_MISMATCH' && detailsOf(error).actual === sha256Hex(Buffer.from('other')), + ); + assert.deepEqual(calls, []); + + const info = await bucket.upload('a.bin', payload, { sha256: digest, metadata: { tenant: 't1' }, contentType: 'text/plain' }); + assert.equal(info.filename, 'a.bin'); + assert.equal(info.metadata?.sha256, digest); + assert.equal(info.metadata?.tenant, 't1'); + assert.equal(info.contentType, 'text/plain'); + assert.deepEqual(await bucket.download(info.id), payload); + await assert.rejects(() => bucket.upload('b.bin', payload, { sha256: digest }), (error: unknown) => { + assert.equal(codeOf(error), 'GRIDFS_FILE_EXISTS'); + assert.equal(detailsOf(error).sha256, digest); + return true; + }); + const skipped = await bucket.upload('c.bin', payload, { sha256: digest, ifExists: 'skip' }); + assert.equal(String(skipped.id), String(info.id)); + const duplicate = await bucket.upload('d.bin', payload, { sha256: digest, ifExists: 'duplicate' }); + assert.notEqual(String(duplicate.id), String(info.id)); + const named = await bucket.upload('report.bin', Buffer.from('report')); + const again = await bucket.upload('report.bin', Buffer.from('other-report'), { ifExists: 'skip' }); + assert.equal(String(again.id), String(named.id)); + }); + + it('returns download misses and illegal ranges as coded errors', async () => { + const { bucket } = bucketFor(); + const missing = new ObjectId(); + await assert.rejects(() => bucket.download(missing), (error: unknown) => codeOf(error) === 'GRIDFS_FILE_NOT_FOUND'); + await assert.rejects(() => bucket.download({ filename: 'absent.txt' }), (error: unknown) => codeOf(error) === 'GRIDFS_FILE_NOT_FOUND'); + const stream = bucket.downloadStream(missing, { start: 0, end: 1 }); + const streamed = await new Promise((resolve) => stream.once('error', resolve)); + assert.equal(codeOf(streamed), 'GRIDFS_FILE_NOT_FOUND'); + const ranged = bucket.downloadStream(new ObjectId(), { start: 4, end: 4 }); + const rangeError = await new Promise((resolve) => ranged.once('error', resolve)); + assert.equal(codeOf(rangeError), 'GRIDFS_RANGE_INVALID'); + assert.equal(await bucket.getFile(missing), null); + }); + + it('blocks model-only inherit writes from both sources and still allows reads and model drop', async () => { + const policy = normalizeWritePathPolicy({ default: 'model-only' }); + const modelBucket = bucketFor({ policy, source: 'model' }); + assert.throws(() => modelBucket.bucket.upload('a.bin', Buffer.from('x')), (error: unknown) => { + assert.equal(codeOf(error), 'INVALID_OPERATION'); + assert.equal(detailsOf(error).category, 'gridfs'); + assert.equal(detailsOf(error).reason, 'WRITE_PATH_POLICY'); + assert.equal(detailsOf(error).source, 'model'); + return true; + }); + assert.equal(await modelBucket.bucket.getFile(new ObjectId()), null); + await modelBucket.bucket.drop(); + assert.deepEqual(modelBucket.calls, ['drop']); + + const instanceBucket = bucketFor({ policy, source: 'db' }); + assert.throws(() => instanceBucket.bucket.upload('a.bin', Buffer.from('x')), (error: unknown) => detailsOf(error).source === 'db' && detailsOf(error).category === 'gridfs'); + assert.throws(() => instanceBucket.bucket.drop(), (error: unknown) => detailsOf(error).category === 'management'); + const allowed = bucketFor({ policy: normalizeWritePathPolicy({ default: { mode: 'model-only', gridfs: 'allow' } }), source: 'model' }); + const info = await allowed.bucket.upload('a.bin', Buffer.from('ok')); + assert.equal(info.length, 2); + }); + + it('resumes appends, rejects gaps and bad sizes, and hides the file until finalize', async () => { + const { bucket, db } = bucketFor({ driver: false }); + const payload = Buffer.from('abcdefghij'); + const digest = sha(payload); + const started = await bucket.beginResumableUpload('big.bin', { length: payload.length, sha256: digest, chunkSizeBytes: 4, metadata: { kind: 'bin' } }); + assert.equal(started.nextChunk, 0); + assert.equal(await bucket.getFile(started.uploadId), null); + await assert.rejects(() => bucket.appendChunks(started.uploadId, 1, payload.subarray(0, 4)), (error: unknown) => { + assert.equal(codeOf(error), 'GRIDFS_RESUME_MISMATCH'); + assert.equal(detailsOf(error).expectedNextChunk, 0); + return true; + }); + await assert.rejects(() => bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 3)), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + const first = await bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 4)); + assert.equal(first.nextChunk, 1); + const replay = await bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 4)); + assert.equal(replay.nextChunk, 1); + await assert.rejects(() => bucket.appendChunks(started.uploadId, 0, Buffer.from('xxxx')), (error: unknown) => codeOf(error) === 'GRIDFS_RESUME_MISMATCH'); + await assert.rejects(() => bucket.finalizeResumableUpload(started.uploadId), (error: unknown) => { + assert.equal(codeOf(error), 'GRIDFS_UPLOAD_INCOMPLETE'); + assert.deepEqual(detailsOf(error).missingChunks, [1, 2]); + return true; + }); + assert.equal(await bucket.getFile(started.uploadId), null); + const rest = await bucket.appendChunks(started.uploadId, 1, payload.subarray(4)); + assert.equal(rest.nextChunk, 3); + const bad = await bucket.beginResumableUpload('bad.bin', { length: 4, sha256: sha(Buffer.from('nope')), chunkSizeBytes: 4 }); + await bucket.appendChunks(bad.uploadId, 0, Buffer.from('abcd')); + await assert.rejects(() => bucket.finalizeResumableUpload(bad.uploadId), (error: unknown) => codeOf(error) === 'GRIDFS_INTEGRITY_MISMATCH'); + assert.equal(await bucket.getFile(bad.uploadId), null); + + const info = await bucket.finalizeResumableUpload(started.uploadId); + assert.equal(info.metadata?.sha256, digest); + assert.equal(info.metadata?.kind, 'bin'); + assert.equal(info.length, payload.length); + assert.equal(await db.collection('fs.uploads').findOne({ _id: started.uploadId }), null); + const chunks = await db.collection('fs.chunks').find({ files_id: started.uploadId }).sort({ n: 1 }).toArray(); + assert.equal(Buffer.concat(chunks.map((chunk) => chunk.data as Buffer)).toString(), payload.toString()); + await bucket.abortResumableUpload(started.uploadId); + const retained = await db.collection('fs.chunks').find({ files_id: started.uploadId }).toArray(); + assert.equal(retained.length, 3); + await bucket.abortResumableUpload(new ObjectId()); + }); + + it('abort removes pending chunks and ensureIndexes is idempotent', async () => { + const { bucket, db } = bucketFor({ driver: false }); + const started = await bucket.beginResumableUpload('partial.bin', { length: 4, sha256: sha(Buffer.from('abcd')), chunkSizeBytes: 4 }); + await bucket.appendChunks(started.uploadId, 0, Buffer.from('abcd')); + await bucket.abortResumableUpload(started.uploadId); + await bucket.abortResumableUpload(started.uploadId); + assert.equal((await db.collection('fs.chunks').find({}).toArray()).length, 0); + assert.equal(await db.collection('fs.uploads').findOne({ _id: started.uploadId }), null); + assert.equal(await bucket.getFile(started.uploadId), null); + await bucket.ensureIndexes(); + await bucket.ensureIndexes(); + const indexes = db.collection('fs.files').indexes.concat(db.collection('fs.chunks').indexes, db.collection('fs.uploads').indexes); + assert.equal(indexes.length, 4); + assert.ok(indexes.some((index) => index.expireAfterSeconds === 604800)); + assert.ok(indexes.some((index) => index.key?.['metadata.sha256'] === 1)); + }); + + it('routes a named pool to that client and keeps Model pool and database fixed', () => { + const seen: string[] = []; + const client = (label: string) => ({ + db(name: string) { + seen.push(`${label}:${name}`); + return { collection: () => ({}) }; + }, + }); + const runtime = { + ensureConnected() { return undefined; }, + options: { type: 'mongodb' as const, databaseName: 'main' }, + _client: client('default'), + _poolManager: { getPool: (name: string) => name === 'archive' ? client('archive') : null, getPoolNames: () => ['archive'] }, + _runtimeDefaults: { writePathPolicy: normalizeWritePathPolicy() }, + }; + openRuntimeGridFS(runtime, 'photos', { pool: 'archive', db: 'assets' }); + openModelGridFS(runtime, 'assets', 'archive', undefined, 1024); + openRuntimeGridFS(runtime, undefined, undefined); + assert.deepEqual(seen, ['archive:assets', 'archive:assets', 'default:main']); + }); +}); From bf79092e4c3ffc1bc57d1ea631a05d8254183860 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 27 Oct 2026 03:14:25 +0000 Subject: [PATCH 4/7] docs(gridfs): document GridFS API and changelog Add bilingual GridFS docs, error codes, writePathPolicy notes, and the docs sidebar. Stage the work in unreleased notes without a 3.5.0 release commit. Co-authored-by: rocky --- changelogs/unreleased.md | 2 +- docs/en/api-index.md | 3 + docs/en/capability-index.md | 7 +- docs/en/error-codes.md | 18 +++ docs/en/gridfs.md | 141 +++++++++++++++++++++ docs/en/support-matrix.md | 2 + docs/en/write-path-policy.md | 5 +- docs/zh/api-index.md | 3 + docs/zh/capability-index.md | 7 +- docs/zh/error-codes.md | 18 +++ docs/zh/gridfs.md | 141 +++++++++++++++++++++ docs/zh/support-matrix.md | 2 + docs/zh/write-path-policy.md | 5 +- scripts/validation/doc-example-matrix.json | 7 + website/rspress.config.ts | 1 + 15 files changed, 353 insertions(+), 9 deletions(-) create mode 100644 docs/en/gridfs.md create mode 100644 docs/zh/gridfs.md diff --git a/changelogs/unreleased.md b/changelogs/unreleased.md index d0ea81e..ba8e2b3 100644 --- a/changelogs/unreleased.md +++ b/changelogs/unreleased.md @@ -2,7 +2,7 @@ ## Changes after v3.4.0 -No changes have been staged after 3.4.0. The Search index lifecycle API is assigned to the [3.4.0 versioned notes](./v3.4.0.md). Publication is established by the npm registry and immutable tag, not this file. +- Add GridFS buckets on `msq.gridfs()` and `model.gridfs()`, including upload, download, ranges, `ensureIndexes`, and phase-1 resumable upload. `writePathPolicy` gains a `gridfs` field and category. Policy violations stay `INVALID_OPERATION` with `reason: 'WRITE_PATH_POLICY'`. These notes stage the work after 3.4.0; the npm package version is unchanged and this file is not a 3.5.0 release. The Search index lifecycle API remains in the [3.4.0 versioned notes](./v3.4.0.md). ## Archived preparation notes for the 3.3.2 candidate diff --git a/docs/en/api-index.md b/docs/en/api-index.md index 3bd580f..d00ccd0 100644 --- a/docs/en/api-index.md +++ b/docs/en/api-index.md @@ -97,6 +97,7 @@ This index documents the current stable MongoDB adapter APIs and shared runtime | [Aggregation pipeline](aggregate.md) | `aggregate()` | Aggregation pipeline query | | [Vector search](vector-search.md) | Collection / Model `vectorSearch()` | MongoDB Vector Search query with Model soft-delete visibility when applicable | | [Search indexes](search-indexes.md) | Collection / Model `createSearchIndex()` and related methods | Create, inspect, update, drop, and wait for Search / Vector Search indexes | +| [GridFS](gridfs.md) | `msq.gridfs()` / Model `gridfs()` | Upload, download, and resume GridFS files | --- @@ -174,6 +175,7 @@ This index documents the current stable MongoDB adapter APIs and shared runtime - [Aggregation pipeline](aggregate.md) - Aggregation pipelines - [Vector search](vector-search.md) - MongoDB Vector Search queries - [Search indexes](search-indexes.md) - Search / Vector Search index lifecycle +- [GridFS](gridfs.md) - GridFS upload, download, and resumable upload - [Query plans](explain.md) - Query plans - [Chain queries](chaining-api.md) - Chain queries @@ -209,6 +211,7 @@ This index documents the current stable MongoDB adapter APIs and shared runtime | [Create indexes in bulk](create-indexes.md) | Create indexes in bulk | | [Drop an index](drop-index.md) | Drop an index | | [List indexes](list-indexes.md) | List indexes | +| [GridFS](gridfs.md) | GridFS buckets, ranges, and resumable upload | | [Find and count](find-and-count.md) | Find and count | | [Expression functions](expression-functions.md) | Expression functions | | [Quick upsert](quick-upsert.md) | Quick upsert | diff --git a/docs/en/capability-index.md b/docs/en/capability-index.md index c6d0e69..76625f6 100644 --- a/docs/en/capability-index.md +++ b/docs/en/capability-index.md @@ -14,14 +14,15 @@ Principles: | Layer | Current status | Scope | |-------|----------------|-------| | Shared runtime | Stable where exposed by the current runtime | Database cache, pools, transactions, model constraints, sync, slow-query logging, and runtime diagnostics | -| MongoDB adapter | Stable | `collection()`, `db()`, `use()`, `pool()`, query/write helpers, aggregation, indexes, Search indexes, transactions, Change Streams | +| MongoDB adapter | Stable | `collection()`, `db()`, `use()`, `pool()`, query/write helpers, aggregation, indexes, Search indexes, GridFS, transactions, Change Streams | | MySQL adapter | Planned / in development | Future database-native adapter, no current public runtime API | | PostgreSQL adapter | Planned / in development | Future database-native adapter, no current public runtime API | | Capability | Recommended entry | Notes | |------------|----------------------|-------| | Model | `msq.model()` / `MonSQLize.Model` | Supports model registration, relations, virtuals, populate, and common query entries | -| write-path-policy | `writePathPolicy` | Optional runtime guard for Model-only write namespaces | +| write-path-policy | `writePathPolicy` | Optional runtime guard for Model-only write namespaces, including the `gridfs` category | +| GridFS | `msq.gridfs()` / `model.gridfs()` | Upload, download, and phase-1 resumable GridFS upload. See [GridFS](./gridfs.md) | | cache | Query `cache` options / `msq.cache` | Database query cache, multi-layer cache access, invalidation, and statistics | | transaction | `startSession()` / `withTransaction()` | Supports MongoDB transaction wrappers and transaction statistics | | pool | `pools` / `pool()` / `ConnectionPoolManager` | Declare `pools: PoolConfig[]` in the constructor, route with `pool()`, and use the low-level manager for advanced inspection or manual management | @@ -50,7 +51,7 @@ Available entries: - `writePathPolicy.namespaces` - `WritePathPolicyOptions` -Use this when selected namespaces should be written through `msq.model()` instead of direct `collection()` writes. See [Write Path Policy](./write-path-policy.md). +Use this when selected namespaces should be written through `msq.model()` instead of direct `collection()` writes. See [Write Path Policy](./write-path-policy.md). GridFS writes use the `gridfs` field and are documented in [GridFS](./gridfs.md). ## 3. transaction diff --git a/docs/en/error-codes.md b/docs/en/error-codes.md index c9a03a9..1830de4 100644 --- a/docs/en/error-codes.md +++ b/docs/en/error-codes.md @@ -72,6 +72,12 @@ See [Data Tasks](./data-tasks.md) for the preview, approval, apply, backup, and | `SEARCH_INDEX_UNSUPPORTED` | Deployment lacks Search index support | Atlas, MongoDB 8.2+ with mongot, or mongodb-atlas-local | | `SEARCH_INDEX_BUILD_FAILED` | Search index build failed while waiting | `details[0].statusDetail`, index definition | | `SEARCH_INDEX_WAIT_TIMEOUT` | waitForSearchIndex timed out | `timeoutMs`, `details[0].last` | +| `GRIDFS_FILE_NOT_FOUND` | GridFS file or resumable session is missing | id, filename, `ignoreMissing` | +| `GRIDFS_FILE_EXISTS` | GridFS upload conflicts with a visible file or an open session | `ifExists`, `sha256`, id | +| `GRIDFS_INTEGRITY_MISMATCH` | GridFS bytes do not match the declared sha256 or chunk size | `details[0].expected`, `details[0].actual` | +| `GRIDFS_UPLOAD_INCOMPLETE` | Resumable finalize found missing chunks | `details[0].missingChunks` | +| `GRIDFS_RESUME_MISMATCH` | Resumable append does not match the stored prefix | `fromChunk`, `expectedNextChunk` | +| `GRIDFS_RANGE_INVALID` | GridFS download range is not a legal half-open interval | `start`, `end`, `length` | | `INVALID_RELATION_SELECT` | Relation default or populate selection is outside P0 inclusion syntax | Use non-empty top-level inclusion fields only | | `RELATION_IN_USE` | Protected Model deletion found declared inbound references | Inspect `error.details[0].usages`; remove or retain references intentionally | | `RELATION_USAGE_UNAVAILABLE` | Protected deletion could not complete its declared-relation scan | Inspect coverage/skipped details; do not retry as an unchecked delete | @@ -210,6 +216,12 @@ See [Search index lifecycle](search-indexes.md) for when each code is raised; ma --- +### GRIDFS_* + +See [GridFS](gridfs.md) for when each code is raised. `downloadStream` reports not-found and range failures as stream `'error'` events. Other driver failures are `MONGODB_ERROR` and keep the driver error as `cause`. `writePathPolicy` blocks are `INVALID_OPERATION`, not a GridFS code, with `details[0].reason` `WRITE_PATH_POLICY`. + +--- + ### INVALID_VECTOR_SEARCH **Explanation**: A local `vectorSearch()` option is invalid before MongoDB is called. @@ -1041,6 +1053,12 @@ try { | `SEARCH_INDEX_UNSUPPORTED` | Search indexes | Deployment lacks Search index support | | `SEARCH_INDEX_BUILD_FAILED` | Search indexes | Search index build failed while waiting | | `SEARCH_INDEX_WAIT_TIMEOUT` | Search indexes | waitForSearchIndex timed out | +| `GRIDFS_FILE_NOT_FOUND` | GridFS | GridFS file or resumable session is missing | +| `GRIDFS_FILE_EXISTS` | GridFS | GridFS upload conflicts with an existing file or session | +| `GRIDFS_INTEGRITY_MISMATCH` | GridFS | GridFS bytes do not match sha256 or chunk size | +| `GRIDFS_UPLOAD_INCOMPLETE` | GridFS | Resumable finalize found missing chunks | +| `GRIDFS_RESUME_MISMATCH` | GridFS | Resumable append does not match the stored prefix | +| `GRIDFS_RANGE_INVALID` | GridFS | GridFS download range is invalid | | `INVALID_RELATION_SELECT` | Relations | Unsupported relation inclusion selection | | `RELATION_IN_USE` | Relations | Protected delete found declared references | | `RELATION_USAGE_UNAVAILABLE` | Relations | Protected delete scan coverage is incomplete or unavailable | diff --git a/docs/en/gridfs.md b/docs/en/gridfs.md new file mode 100644 index 0000000..9ddb4fc --- /dev/null +++ b/docs/en/gridfs.md @@ -0,0 +1,141 @@ +# GridFS + +Instance and Model helpers store and read MongoDB GridFS files. The published npm package is still 3.4.0; this API is staged in the source after that release and is not a published 3.5.0 package. + +GridFS is a client convention on ordinary collections (`.files` and `.chunks`). It does not require a Search or Vector Search deployment. Integration coverage uses `mongodb-memory-server` 7.0.37. The formal server matrix remains MongoDB 7.x and 8.x. Older servers, including 4.x, are unverified. + +## Open a bucket + +```ts +const bucket = msq.gridfs(bucketName?: string, opts?: { + db?: string; + pool?: string; + chunkSizeBytes?: number; +}); + +const modelBucket = model.gridfs(bucketName?: string, opts?: { + chunkSizeBytes?: number; +}); +``` + +`bucketName` defaults to `fs`. An empty string is rejected. `chunkSizeBytes` defaults to 261120 (255 KiB) and must be a positive integer. + +`msq.gridfs()` uses the runtime connection unless `pool` names a configured pool. A named pool uses that pool's client and `client.db(db)`. `db` defaults to the runtime database. `model.gridfs()` always uses the Model `poolName` and `dbName`; it does not accept `pool` or `db`. + +Calling either method before `connect()` throws `NOT_CONNECTED`. + +## Files API + +| Method | Result | +|--------|--------| +| `upload(filename, data, opts?)` | Promise of file info. `data` is a `Buffer`, `Uint8Array`, or `Readable`. | +| `uploadStream(filename, opts?)` | Synchronous `Writable` with `id` and `done`. Await `done` for file info or the mapped error. | +| `download(idOrName)` | Promise of a `Buffer`. | +| `downloadStream(idOrName, range?)` | Synchronous `Readable`. Missing files and range errors arrive as `'error'` events. | +| `getFile(id)` | File info, or `null` when the id is absent. | +| `findFiles(filter?, opts?)` | File info documents. `limit` must be a positive integer. `skip` must be `>= 0`. Sort values are `1` or `-1`. | +| `delete(id, opts?)` | Removes the files document and its chunks. `ignoreMissing: true` resolves when the id is absent. | +| `rename(id, filename)` | Renames one visible file. | +| `drop()` | Drops `.files`, `.chunks`, and `.uploads` when they exist. | +| `ensureIndexes()` | Creates the indexes below. Safe to call more than once. | + +`idOrName` is an `ObjectId` or `{ filename, revision? }`. `revision` follows the driver: `0` is the oldest file with that name, and `-1` (the default) is the newest. `download({ filename })` therefore returns the latest file with that name. + +### `ifExists` and sha256 + +`upload` defaults to `ifExists: 'error'`. + +- When `sha256` is set, the conflict key is `metadata.sha256`. The digest must be 64 lowercase hex characters. `Buffer` and `Uint8Array` payloads are hashed before any GridFS write. A mismatch throws `GRIDFS_INTEGRITY_MISMATCH` and leaves no visible file. +- A `Readable` passed with `sha256` is buffered completely, then hashed, before any write. Large files should use the resumable API instead. +- `skip` with `sha256` returns the newest visible file with that digest. `skip` without `sha256` returns the newest visible file with the same filename. `duplicate` always creates another file. +- `uploadStream` stores `sha256` in metadata only. It does not check the bytes and it does not accept `ifExists`. + +`metadata.sha256` written by these helpers overwrites a caller-supplied `metadata.sha256`. + +### Ranges + +`downloadStream` ranges use an inclusive `start` and an exclusive `end`. Omitted `start` means `0`. Omitted `end` means the file length. A legal range satisfies `0 <= start < end <= length`. Anything else, including `start === end` and `start === length`, emits `GRIDFS_RANGE_INVALID`. When the failure depends on the file length, `details[0].length` is that length. Shape errors that can be rejected before a lookup omit `length`. `download()` rejects the promise with the same codes; it does not apply a range. + +## Resumable upload + +Phase 1 resumable upload keeps a pending session in `.uploads` until finalize. The file stays invisible until `finalizeResumableUpload` succeeds. `uploadId` is the future `files._id`. + +```ts +const session = await bucket.beginResumableUpload(filename, { + length, + sha256, + chunkSizeBytes?, + metadata?, + contentType?, + id?, +}); +const progress = await bucket.appendChunks(session.uploadId, fromChunk, data); +const info = await bucket.finalizeResumableUpload(session.uploadId); +await bucket.abortResumableUpload(session.uploadId); +``` + +`beginResumableUpload` requires `length` and `sha256`. An id that already exists in `files` or `uploads` throws `GRIDFS_FILE_EXISTS`. + +Chunk bytes must follow the session size: + +- Every chunk except the last has length `chunkSizeBytes`. +- The last chunk has length `length % chunkSizeBytes`, or `chunkSizeBytes` when the length divides evenly. +- A payload that does not split into those sizes throws `INVALID_ARGUMENT`. + +`appendChunks` is addressed by `fromChunk`: + +- `fromChunk` past the contiguous prefix throws `GRIDFS_RESUME_MISMATCH` with `expectedNextChunk`. +- Replaying an earlier chunk with the same bytes is idempotent and does not move `nextChunk` backward. +- Replaying an earlier chunk with different bytes throws `GRIDFS_RESUME_MISMATCH`. +- A duplicate `{ files_id, n }` insert is accepted only when the stored bytes match. + +`finalizeResumableUpload` requires a contiguous prefix, a total length equal to the session length, and a sha256 equal to the session digest. It then inserts the files document, sets `metadata.sha256`, and deletes the session. Missing chunk indexes throw `GRIDFS_UPLOAD_INCOMPLETE` with `details[0].missingChunks`. A bad digest or a chunk of the wrong size throws `GRIDFS_INTEGRITY_MISMATCH`. The files document is not inserted in those cases. + +`abortResumableUpload` deletes the session and, when no files document is visible, that upload's chunks. A missing or already aborted session resolves. Aborting an id that already has a visible file does not delete that file's chunks. + +`.uploads` has a TTL index on `createdAt` with `expireAfterSeconds: 604800` (7 days). The TTL deletes upload session documents only. It does not delete chunks. Abort or finalize sessions that are no longer needed. + +## Indexes + +`upload`, `uploadStream` (before the driver stream is opened), `beginResumableUpload`, and `ensureIndexes` create: + +| Collection | Key | Options | +|------------|-----|---------| +| `.files` | `{ filename: 1, uploadDate: 1 }` | | +| `.files` | `{ 'metadata.sha256': 1 }` | not unique | +| `.chunks` | `{ files_id: 1, n: 1 }` | unique | +| `.uploads` | `{ createdAt: 1 }` | `expireAfterSeconds: 604800` | + +Reads do not create indexes. A write rejected by `writePathPolicy` does not create them either. + +## writePathPolicy + +GridFS writes are category `gridfs`. The namespace is `.files`, matched with the usual order: instance id, `pool:db.collection`, `db.collection`, collection name, then `default`. + +| `gridfs` | `mode` | `msq.gridfs()` writes | `model.gridfs()` writes | +|----------|--------|----------------------|-------------------------| +| `inherit` (default) | `allow-both` | allowed | allowed | +| `inherit` | `model-only` | blocked | blocked | +| `allow` | either | allowed | allowed | +| `block` | either | blocked | blocked | + +`inherit` does not follow the management rule. Management `inherit` still allows Model management and blocks direct management. GridFS `inherit` under `model-only` blocks both sources. `model.gridfs().drop()` stays a management operation, so Model drop remains allowed under `model-only` while `msq.gridfs().drop()` is blocked. + +A blocked call throws `INVALID_OPERATION` before any GridFS write. `details[0]` includes `category: 'gridfs'` (or `'management'` for `drop`) and `reason: 'WRITE_PATH_POLICY'`. `onViolation: 'warn'` logs that detail and allows the call. `getFile`, `findFiles`, `download`, and `downloadStream` are not governed. + +## Errors + +| Code | When | +|------|------| +| `GRIDFS_FILE_NOT_FOUND` | Download, delete, rename, or resumable session target is missing | +| `GRIDFS_FILE_EXISTS` | `ifExists: 'error'` finds a conflict, or the chosen id is already present | +| `GRIDFS_INTEGRITY_MISMATCH` | Bytes do not match `sha256`, or a finalized chunk has the wrong size | +| `GRIDFS_UPLOAD_INCOMPLETE` | Finalize found gaps. `details[0].missingChunks` lists them | +| `GRIDFS_RESUME_MISMATCH` | `fromChunk` is ahead of the contiguous prefix, or a replayed chunk differs | +| `GRIDFS_RANGE_INVALID` | `start` / `end` is not a legal half-open range | +| `INVALID_ARGUMENT` | Filename, bucket name, chunk size, digest, metadata key, or chunk payload is invalid | +| `INVALID_OPERATION` | `writePathPolicy` blocked the call | +| `NOT_CONNECTED` | The runtime is not connected, or the selected pool client is missing | +| `MONGODB_ERROR` | Other driver or server failures. `cause` is the original error | + +`downloadStream` delivers `GRIDFS_FILE_NOT_FOUND` and `GRIDFS_RANGE_INVALID` on the stream. The method itself does not throw those errors. diff --git a/docs/en/support-matrix.md b/docs/en/support-matrix.md index 0b67902..7c1a944 100644 --- a/docs/en/support-matrix.md +++ b/docs/en/support-matrix.md @@ -18,6 +18,8 @@ | MongoDB Driver | 6.x / 7.x | `npm run test:server-matrix` | | MongoDB Server | 7.x / 8.x | strict `mongodb-memory-server` 7.0.37 / 8.0.26 single + replica set matrix | +GridFS is a client convention on `.files` and `.chunks`. Its integration tests run on `mongodb-memory-server` 7.0.37. That does not add MongoDB 4.x, or any server outside 7.x / 8.x, to this matrix. + ## Default verification method - **Default gate**: `npm test` diff --git a/docs/en/write-path-policy.md b/docs/en/write-path-policy.md index 18305f4..fa5e945 100644 --- a/docs/en/write-path-policy.md +++ b/docs/en/write-path-policy.md @@ -40,6 +40,7 @@ type WritePathPolicyRule = { mode?: WritePathPolicyMode; raw?: 'inherit' | 'allow' | 'block'; management?: 'inherit' | 'allow' | 'block'; + gridfs?: 'inherit' | 'allow' | 'block'; onViolation?: 'throw' | 'warn'; }; @@ -54,6 +55,7 @@ type WritePathPolicyOptions = { | `mode` | `allow-both` | `allow-both` allows collection and Model writes. `model-only` blocks direct collection, db, and legacy writes unless overridden. | | `raw` | `inherit` | Controls `collection.raw()`, `db.raw()`, and db command access. Inherits `block` from `model-only` and `allow` from `allow-both`. | | `management` | `inherit` | Controls index and collection management operations. In `model-only`, Model management methods are allowed while direct collection management is blocked. | +| `gridfs` | `inherit` | Controls GridFS writes on `.files`. `inherit` allows them only when `mode` is `allow-both`. Under `model-only`, both `msq.gridfs()` and `model.gridfs()` writes are blocked. This is separate from `management`. | | `onViolation` | `throw` | `throw` rejects the operation. `warn` logs a warning and allows the operation. | ## Namespace Matching @@ -76,7 +78,8 @@ Inside `namespaces`, the key `default` is reserved for the fallback matcher and - Collection writes: `insertOne`, `insertMany`, `updateOne`, `updateMany`, `replaceOne`, `findOneAndUpdate`, `findOneAndReplace`, `findOneAndDelete`, `upsertOne`, `deleteOne`, `deleteMany`. - Batch helpers: `insertBatch`, `updateBatch`, `deleteBatch`, `incrementOne`. -- Collection management: index creation/drop, collection creation/drop, validators, `renameCollection`, `collMod`, and capped conversion. +- Collection management: index creation/drop, collection creation/drop, validators, `renameCollection`, `collMod`, and capped conversion. `gridfs().drop()` is management. Model drop stays allowed when `management` is `inherit` and `mode` is `model-only`; instance drop is blocked. +- GridFS writes and `ensureIndexes()`: category `gridfs`, namespace `.files`. See [GridFS](gridfs.md). A violation is `INVALID_OPERATION` with `details[0].reason` set to `WRITE_PATH_POLICY` and `details[0].category` set to `gridfs`. - Raw/db/legacy write surfaces: `collection.raw()`, `db.raw()`, `db.runCommand()`, `dropDatabase()`, and legacy adapter writes. - Aggregation pipelines whose final stage writes with `$out` or `$merge`; the policy is checked against the write target namespace. diff --git a/docs/zh/api-index.md b/docs/zh/api-index.md index a0b74c3..b70c111 100644 --- a/docs/zh/api-index.md +++ b/docs/zh/api-index.md @@ -97,6 +97,7 @@ | [聚合管道查询](aggregate.md) | `aggregate()` | 聚合管道查询 | | [向量查询](vector-search.md) | Collection / Model `vectorSearch()` | MongoDB Vector Search 向量查询；Model 会应用软删除可见性 | | [Search 索引](search-indexes.md) | Collection / Model `createSearchIndex()` 等 | 创建、查看、更新、删除并等待 Search / Vector Search 索引 | +| [GridFS](gridfs.md) | `msq.gridfs()` / Model `gridfs()` | 上传、下载和恢复 GridFS 文件 | --- @@ -172,6 +173,7 @@ - [聚合管道查询](aggregate.md) - 聚合管道 - [向量查询](vector-search.md) - MongoDB Vector Search 向量查询 - [Search 索引](search-indexes.md) - Search / Vector Search 索引生命周期 +- [GridFS](gridfs.md) - GridFS 上传、下载与可恢复上传 - [查询计划分析](explain.md) - 查询计划 - [链式调用](chaining-api.md) - 链式调用 @@ -207,6 +209,7 @@ | [批量创建索引](create-indexes.md) | 批量创建索引 | | [删除索引](drop-index.md) | 删除索引 | | [列出索引](list-indexes.md) | 列出索引 | +| [GridFS](gridfs.md) | GridFS bucket、区间与可恢复上传 | | [查询并计数](find-and-count.md) | 查询并计数 | | [表达式函数](expression-functions.md) | 表达式函数 | | [快速 upsert](quick-upsert.md) | 快速 upsert | diff --git a/docs/zh/capability-index.md b/docs/zh/capability-index.md index 7daedcd..e219fd8 100644 --- a/docs/zh/capability-index.md +++ b/docs/zh/capability-index.md @@ -13,14 +13,15 @@ monSQLize 是数据库原生的生产数据运行时增强层。当前稳定适 | 层级 | 当前状态 | 范围 | |------|----------|------| | 共享运行时 | 当前 runtime 已暴露能力稳定 | 数据库缓存、连接池、事务、模型约束、同步、慢查询日志与运行时诊断 | -| MongoDB adapter | Stable | `collection()`、`db()`、`use()`、`pool()`、查询/写入增强、聚合、索引、Search 索引、事务、Change Streams | +| MongoDB adapter | Stable | `collection()`、`db()`、`use()`、`pool()`、查询/写入增强、聚合、索引、Search 索引、GridFS、事务、Change Streams | | MySQL adapter | Planned / in development | 后续数据库原生 adapter，当前暂无公开 runtime API | | PostgreSQL adapter | Planned / in development | 后续数据库原生 adapter，当前暂无公开 runtime API | | 能力 | 推荐入口 | 说明 | |------|--------------|------| | Model | `msq.model()` / `MonSQLize.Model` | 支持 Model 注册、关系、虚拟字段、populate 与常用查询入口 | -| write-path-policy | `writePathPolicy` | 可选的 Model-only 写入命名空间运行时护栏 | +| write-path-policy | `writePathPolicy` | 可选的 Model-only 写入命名空间运行时护栏，包含 `gridfs` 类别 | +| GridFS | `msq.gridfs()` / `model.gridfs()` | 上传、下载和第一阶段可恢复 GridFS 上传。见 [GridFS](./gridfs.md) | | cache | 查询 `cache` 选项 / `msq.cache` | 数据库查询缓存、多层缓存接入、缓存失效与统计 | | transaction | `startSession()` / `withTransaction()` | 支持 MongoDB 事务封装与事务统计 | | pool | `pools` / `pool()` / `ConnectionPoolManager` | 在构造函数中声明 `pools: PoolConfig[]`，通过 `pool()` 路由，底层管理器用于进阶状态查看与手动管理 | @@ -49,7 +50,7 @@ Model 相关文档与示例： - `writePathPolicy.namespaces` - `WritePathPolicyOptions` -当指定命名空间必须通过 `msq.model()` 写入，而不能直接通过 `collection()` 写入时使用。详见 [写路径策略](./write-path-policy.md)。 +当指定命名空间必须通过 `msq.model()` 写入，而不能直接通过 `collection()` 写入时使用。详见 [写路径策略](./write-path-policy.md)。GridFS 写入使用 `gridfs` 字段，说明见 [GridFS](./gridfs.md)。 ## 3. transaction diff --git a/docs/zh/error-codes.md b/docs/zh/error-codes.md index daeb40f..830c80c 100644 --- a/docs/zh/error-codes.md +++ b/docs/zh/error-codes.md @@ -69,6 +69,12 @@ try { | `SEARCH_INDEX_UNSUPPORTED` | 部署不支持 Search 索引 | Atlas、带 mongot 的 MongoDB 8.2+ 或 mongodb-atlas-local | | `SEARCH_INDEX_BUILD_FAILED` | 等待期间索引构建失败 | `details[0].statusDetail`、索引定义 | | `SEARCH_INDEX_WAIT_TIMEOUT` | waitForSearchIndex 超时 | `timeoutMs`、`details[0].last` | +| `GRIDFS_FILE_NOT_FOUND` | GridFS 文件或可恢复会话不存在 | id、filename、`ignoreMissing` | +| `GRIDFS_FILE_EXISTS` | GridFS 上传与已可见文件或未完成会话冲突 | `ifExists`、`sha256`、id | +| `GRIDFS_INTEGRITY_MISMATCH` | GridFS 字节与声明的 sha256 或分片长度不符 | `details[0].expected`、`details[0].actual` | +| `GRIDFS_UPLOAD_INCOMPLETE` | 可恢复上传 finalize 发现缺少分片 | `details[0].missingChunks` | +| `GRIDFS_RESUME_MISMATCH` | 可恢复上传的 append 与已存储前缀不符 | `fromChunk`、`expectedNextChunk` | +| `GRIDFS_RANGE_INVALID` | GridFS 下载区间不是合法半开区间 | `start`、`end`、`length` | | `INVALID_RELATION_SELECT` | relation 默认或 populate 字段选择超出 P0 包含式语法 | 只使用非空顶层包含字段 | | `RELATION_IN_USE` | 受保护 Model 删除发现已声明的入站引用 | 查看 `error.details[0].usages`，有意处理或保留引用 | | `RELATION_USAGE_UNAVAILABLE` | 受保护删除未能完成已声明关系扫描 | 查看 coverage/skipped，不要降级为未检查删除 | @@ -201,6 +207,12 @@ try { --- +#### GRIDFS_* + +各码的触发条件见 [GridFS](gridfs.md)。`downloadStream` 把不存在和区间错误发到流的 `'error'` 事件。其他驱动失败是 `MONGODB_ERROR`，驱动原错误保存在 `cause`。`writePathPolicy` 阻断仍是 `INVALID_OPERATION`，不是 GridFS 错误码，`details[0].reason` 为 `WRITE_PATH_POLICY`。 + +--- + #### INVALID_VECTOR_SEARCH **说明**: MongoDB 调用前，`vectorSearch()` 的本地参数不合法。 @@ -996,6 +1008,12 @@ try { | `SEARCH_INDEX_UNSUPPORTED` | Search 索引 | 部署不支持 Search 索引 | | `SEARCH_INDEX_BUILD_FAILED` | Search 索引 | 等待期间索引构建失败 | | `SEARCH_INDEX_WAIT_TIMEOUT` | Search 索引 | waitForSearchIndex 超时 | +| `GRIDFS_FILE_NOT_FOUND` | GridFS | GridFS 文件或可恢复会话不存在 | +| `GRIDFS_FILE_EXISTS` | GridFS | GridFS 上传与已有文件或会话冲突 | +| `GRIDFS_INTEGRITY_MISMATCH` | GridFS | GridFS 字节与 sha256 或分片长度不符 | +| `GRIDFS_UPLOAD_INCOMPLETE` | GridFS | 可恢复上传 finalize 发现缺少分片 | +| `GRIDFS_RESUME_MISMATCH` | GridFS | 可恢复上传的 append 与已存储前缀不符 | +| `GRIDFS_RANGE_INVALID` | GridFS | GridFS 下载区间无效 | | `INVALID_RELATION_SELECT` | 关系 | 不支持的 relation 包含式字段选择 | | `RELATION_IN_USE` | 关系 | 受保护删除发现已声明引用 | | `RELATION_USAGE_UNAVAILABLE` | 关系 | 受保护删除扫描 coverage 不完整或不可用 | diff --git a/docs/zh/gridfs.md b/docs/zh/gridfs.md new file mode 100644 index 0000000..bdb8535 --- /dev/null +++ b/docs/zh/gridfs.md @@ -0,0 +1,141 @@ +# GridFS + +实例和 Model 助手用于存取 MongoDB GridFS 文件。已发布的 npm 包仍是 3.4.0；本 API 位于该版本之后的源码中，还不是已发布的 3.5.0 包。 + +GridFS 是普通集合上的客户端约定（`.files` 与 `.chunks`），不依赖 Search 或 Vector Search。集成测试使用 `mongodb-memory-server` 7.0.37。正式服务器矩阵仍是 MongoDB 7.x 与 8.x。更早的服务器（包括 4.x）未经验证。 + +## 打开 bucket + +```ts +const bucket = msq.gridfs(bucketName?: string, opts?: { + db?: string; + pool?: string; + chunkSizeBytes?: number; +}); + +const modelBucket = model.gridfs(bucketName?: string, opts?: { + chunkSizeBytes?: number; +}); +``` + +`bucketName` 默认 `fs`。空字符串会被拒绝。`chunkSizeBytes` 默认 261120（255 KiB），且必须是正整数。 + +`msq.gridfs()` 使用运行时连接，除非 `pool` 指定了已配置的连接池。命名池使用该池自己的 client，再调用 `client.db(db)`。`db` 默认运行时数据库。`model.gridfs()` 固定使用 Model 的 `poolName` 与 `dbName`，不接受 `pool` 或 `db`。 + +在 `connect()` 之前调用会抛出 `NOT_CONNECTED`。 + +## 文件 API + +| 方法 | 结果 | +|------|------| +| `upload(filename, data, opts?)` | 返回文件信息的 Promise。`data` 可以是 `Buffer`、`Uint8Array` 或 `Readable`。 | +| `uploadStream(filename, opts?)` | 同步返回带 `id` 与 `done` 的 `Writable`。等待 `done` 取得文件信息或映射后的错误。 | +| `download(idOrName)` | 返回 `Buffer` 的 Promise。 | +| `downloadStream(idOrName, range?)` | 同步返回 `Readable`。文件不存在和区间错误通过 `'error'` 事件发出。 | +| `getFile(id)` | 文件信息；id 不存在时为 `null`。 | +| `findFiles(filter?, opts?)` | 文件信息列表。`limit` 必须是正整数，`skip` 必须 `>= 0`，排序值只能是 `1` 或 `-1`。 | +| `delete(id, opts?)` | 删除 files 文档及其 chunks。`ignoreMissing: true` 时，id 不存在也会成功。 | +| `rename(id, filename)` | 重命名一个已可见文件。 | +| `drop()` | 删除存在的 `.files`、`.chunks` 和 `.uploads`。 | +| `ensureIndexes()` | 创建下文索引。可以重复调用。 | + +`idOrName` 是 `ObjectId` 或 `{ filename, revision? }`。`revision` 与驱动一致：`0` 是同名最旧文件，`-1`（默认）是最新文件。因此 `download({ filename })` 返回该文件名的最新文件。 + +### `ifExists` 与 sha256 + +`upload` 的 `ifExists` 默认为 `'error'`。 + +- 提供 `sha256` 时，冲突键是 `metadata.sha256`。摘要必须是 64 位小写十六进制。`Buffer` 与 `Uint8Array` 会在任何 GridFS 写入之前完成哈希。不一致时抛出 `GRIDFS_INTEGRITY_MISMATCH`，并且不会留下可见文件。 +- 带 `sha256` 的 `Readable` 会先完整缓冲再哈希，然后才写入。大文件应改用可恢复上传。 +- 带 `sha256` 的 `skip` 返回该摘要最新的可见文件。不带 `sha256` 的 `skip` 返回同名最新可见文件。`duplicate` 总是再创建一份。 +- `uploadStream` 只把 `sha256` 写入 metadata。它不校验字节，也不接受 `ifExists`。 + +这些助手写入的 `metadata.sha256` 会覆盖调用方自己放在 `metadata.sha256` 里的值。 + +### 区间 + +`downloadStream` 的区间是包含的 `start` 和排除的 `end`。省略 `start` 表示 `0`，省略 `end` 表示文件长度。合法区间满足 `0 <= start < end <= length`。其余情况，包括 `start === end` 和 `start === length`，都会发出 `GRIDFS_RANGE_INVALID`。依赖文件长度才能判断的失败会带上 `details[0].length`。查找文件之前就能拒绝的形态错误不含 `length`。`download()` 用同一个错误码拒绝 Promise，并且不接受区间。 + +## 可恢复上传 + +第一阶段可恢复上传把未完成会话放在 `.uploads`，直到 finalize。`finalizeResumableUpload` 成功之前，文件不可见。`uploadId` 就是将来的 `files._id`。 + +```ts +const session = await bucket.beginResumableUpload(filename, { + length, + sha256, + chunkSizeBytes?, + metadata?, + contentType?, + id?, +}); +const progress = await bucket.appendChunks(session.uploadId, fromChunk, data); +const info = await bucket.finalizeResumableUpload(session.uploadId); +await bucket.abortResumableUpload(session.uploadId); +``` + +`beginResumableUpload` 必须提供 `length` 和 `sha256`。`files` 或 `uploads` 中已经存在该 id 时抛出 `GRIDFS_FILE_EXISTS`。 + +分片长度必须符合会话： + +- 除最后一片外，每片长度都是 `chunkSizeBytes`。 +- 最后一片长度是 `length % chunkSizeBytes`；能够整除时仍是 `chunkSizeBytes`。 +- 无法按这些长度切开的 payload 抛出 `INVALID_ARGUMENT`。 + +`appendChunks` 以 `fromChunk` 定位： + +- `fromChunk` 超过从 0 开始的连续前缀时，抛出 `GRIDFS_RESUME_MISMATCH`，并带上 `expectedNextChunk`。 +- 用相同字节重放已写入的分片是幂等的，不会把 `nextChunk` 往回退。 +- 用不同字节重放已写入的分片会抛出 `GRIDFS_RESUME_MISMATCH`。 +- `{ files_id, n }` 唯一索引冲突只有在已存字节相同的时候才算成功。 + +`finalizeResumableUpload` 要求分片从 0 连续、总长度等于会话长度，且 sha256 等于会话摘要。然后插入 files 文档、写入 `metadata.sha256`，并删除会话。缺少的分片序号抛出 `GRIDFS_UPLOAD_INCOMPLETE`，`details[0].missingChunks` 列出缺口。摘要不符或分片长度不对抛出 `GRIDFS_INTEGRITY_MISMATCH`。这些失败都不会插入 files 文档。 + +`abortResumableUpload` 删除会话；如果还没有可见的 files 文档，同时删除该上传的 chunks。会话不存在或已经中止时直接成功。对已经可见的文件调用 abort 不会删除该文件的 chunks。 + +`.uploads` 在 `createdAt` 上有 TTL 索引，`expireAfterSeconds` 为 604800（7 天）。TTL 只删除上传会话文档，不删除 chunks。不再需要的会话应 abort 或 finalize。 + +## 索引 + +`upload`、`uploadStream`（打开驱动流之前）、`beginResumableUpload` 和 `ensureIndexes` 会创建： + +| 集合 | 键 | 选项 | +|------|----|------| +| `.files` | `{ filename: 1, uploadDate: 1 }` | | +| `.files` | `{ 'metadata.sha256': 1 }` | 非唯一 | +| `.chunks` | `{ files_id: 1, n: 1 }` | 唯一 | +| `.uploads` | `{ createdAt: 1 }` | `expireAfterSeconds: 604800` | + +读取不会创建索引。被 `writePathPolicy` 拒绝的写入也不会创建。 + +## writePathPolicy + +GridFS 写入的类别是 `gridfs`。命名空间是 `.files`，匹配顺序与其他写入相同：实例 id、`pool:db.collection`、`db.collection`、集合名，然后 `default`。 + +| `gridfs` | `mode` | `msq.gridfs()` 写入 | `model.gridfs()` 写入 | +|----------|--------|---------------------|----------------------| +| `inherit`（默认） | `allow-both` | 允许 | 允许 | +| `inherit` | `model-only` | 阻断 | 阻断 | +| `allow` | 任意 | 允许 | 允许 | +| `block` | 任意 | 阻断 | 阻断 | + +`inherit` 不沿用 management 规则。management 的 `inherit` 仍然允许 Model 管理操作、阻断直接管理操作。`model-only` 下 GridFS 的 `inherit` 会同时阻断两个入口。`model.gridfs().drop()` 仍属于 management，因此 `model-only` 下 Model 的 drop 仍然允许，而 `msq.gridfs().drop()` 会被阻断。 + +被阻断的调用在任何 GridFS 写入之前抛出 `INVALID_OPERATION`。`details[0]` 包含 `category: 'gridfs'`（`drop` 为 `'management'`）和 `reason: 'WRITE_PATH_POLICY'`。`onViolation: 'warn'` 会记录该明细并放行。`getFile`、`findFiles`、`download` 和 `downloadStream` 不受该策略约束。 + +## 错误 + +| 错误码 | 时机 | +|--------|------| +| `GRIDFS_FILE_NOT_FOUND` | 下载、删除、重命名或可恢复会话的目标不存在 | +| `GRIDFS_FILE_EXISTS` | `ifExists: 'error'` 发现冲突，或指定 id 已经存在 | +| `GRIDFS_INTEGRITY_MISMATCH` | 字节与 `sha256` 不符，或 finalize 时分片长度不对 | +| `GRIDFS_UPLOAD_INCOMPLETE` | finalize 发现缺口。`details[0].missingChunks` 列出缺少的序号 | +| `GRIDFS_RESUME_MISMATCH` | `fromChunk` 超前于连续前缀，或重放的分片内容不同 | +| `GRIDFS_RANGE_INVALID` | `start` / `end` 不是合法的半开区间 | +| `INVALID_ARGUMENT` | 文件名、bucket 名、分片大小、摘要、metadata 键或分片 payload 不合法 | +| `INVALID_OPERATION` | `writePathPolicy` 阻断了调用 | +| `NOT_CONNECTED` | 运行时未连接，或所选连接池的 client 不存在 | +| `MONGODB_ERROR` | 其他驱动或服务器失败。`cause` 是原始错误 | + +`downloadStream` 把 `GRIDFS_FILE_NOT_FOUND` 和 `GRIDFS_RANGE_INVALID` 发到流上。方法本身不抛出这两个错误。 diff --git a/docs/zh/support-matrix.md b/docs/zh/support-matrix.md index 922e67a..f74886c 100644 --- a/docs/zh/support-matrix.md +++ b/docs/zh/support-matrix.md @@ -18,6 +18,8 @@ | MongoDB Driver | 6.x / 7.x | `npm run test:server-matrix` | | MongoDB Server | 7.x / 8.x | `mongodb-memory-server` 7.0.37 / 8.0.26 single + replica set 严格矩阵 | +GridFS 是 `.files` 与 `.chunks` 上的客户端约定。它的集成测试跑在 `mongodb-memory-server` 7.0.37 上。这不会把 MongoDB 4.x，或 7.x / 8.x 以外的服务器，写进本矩阵。 + ## 默认验证方式 - **Default gate**：`npm test` diff --git a/docs/zh/write-path-policy.md b/docs/zh/write-path-policy.md index ed3b7df..532eef8 100644 --- a/docs/zh/write-path-policy.md +++ b/docs/zh/write-path-policy.md @@ -40,6 +40,7 @@ type WritePathPolicyRule = { mode?: WritePathPolicyMode; raw?: 'inherit' | 'allow' | 'block'; management?: 'inherit' | 'allow' | 'block'; + gridfs?: 'inherit' | 'allow' | 'block'; onViolation?: 'throw' | 'warn'; }; @@ -54,6 +55,7 @@ type WritePathPolicyOptions = { | `mode` | `allow-both` | `allow-both` 允许 collection 与 Model 写入。`model-only` 会阻断直接 collection、db、legacy 写入，除非命名空间规则覆盖。 | | `raw` | `inherit` | 控制 `collection.raw()`、`db.raw()` 与 db command 入口。`model-only` 下继承为 `block`，`allow-both` 下继承为 `allow`。 | | `management` | `inherit` | 控制索引、集合、validator 等管理操作。`model-only` 下 Model 管理方法允许，直接 collection 管理方法阻断。 | +| `gridfs` | `inherit` | 控制 `.files` 上的 GridFS 写入。`inherit` 只在 `mode` 为 `allow-both` 时允许。`model-only` 下 `msq.gridfs()` 和 `model.gridfs()` 的写入都会被阻断。它与 `management` 分开。 | | `onViolation` | `throw` | `throw` 会拒绝操作；`warn` 只记录告警并放行。 | ## 命名空间匹配 @@ -76,7 +78,8 @@ type WritePathPolicyOptions = { - Collection 写操作：`insertOne`、`insertMany`、`updateOne`、`updateMany`、`replaceOne`、`findOneAndUpdate`、`findOneAndReplace`、`findOneAndDelete`、`upsertOne`、`deleteOne`、`deleteMany`。 - 批量 helper：`insertBatch`、`updateBatch`、`deleteBatch`、`incrementOne`。 -- Collection 管理操作：索引创建/删除、集合创建/删除、validator、`renameCollection`、`collMod`、capped 转换。 +- Collection 管理操作：索引创建/删除、集合创建/删除、validator、`renameCollection`、`collMod`、capped 转换。`gridfs().drop()` 属于 management。`management` 为 `inherit` 且 `mode` 为 `model-only` 时，Model 的 drop 仍允许，实例 drop 被阻断。 +- GridFS 写入和 `ensureIndexes()`：类别 `gridfs`，命名空间 `.files`。见 [GridFS](gridfs.md)。违规是 `INVALID_OPERATION`，`details[0].reason` 为 `WRITE_PATH_POLICY`，`details[0].category` 为 `gridfs`。 - Raw/db/legacy 写入口：`collection.raw()`、`db.raw()`、`db.runCommand()`、`dropDatabase()`、legacy adapter 写操作。 - 最后阶段包含 `$out` 或 `$merge` 的聚合管道；策略按实际写入目标 namespace 判断。 diff --git a/scripts/validation/doc-example-matrix.json b/scripts/validation/doc-example-matrix.json index 37c2b73..527f813 100644 --- a/scripts/validation/doc-example-matrix.json +++ b/scripts/validation/doc-example-matrix.json @@ -394,6 +394,13 @@ "runner": "test:examples", "status": "covered" }, + { + "doc": "gridfs", + "kind": "doc-check", + "target": "docs/en/gridfs.md", + "runner": "check:docs-examples", + "status": "covered" + }, { "doc": "hooks", "kind": "example", diff --git a/website/rspress.config.ts b/website/rspress.config.ts index faca0ec..aaf93d5 100644 --- a/website/rspress.config.ts +++ b/website/rspress.config.ts @@ -359,6 +359,7 @@ const sidebarSource: SidebarGroupSource[] = [ { en: 'Drop Index', zh: '删除索引', link: '/drop-index' }, { en: 'List Indexes', zh: '列出索引', link: '/list-indexes' }, { en: 'Search Indexes', zh: 'Search 索引', link: '/search-indexes' }, + { en: 'GridFS', zh: 'GridFS', link: '/gridfs' }, { en: 'Database Administration', zh: '数据库管理', link: '/admin' }, { en: 'Error Code Reference', zh: '错误码参考', link: '/error-codes' }, { en: 'Data Validation', zh: '数据验证', link: '/validation' }, From 4180e7ae9214dd9f16f280febcb85ff35d7dea1a Mon Sep 17 00:00:00 2001 From: Rocky Shi <175696301+rockyshi1993@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:43:15 +0000 Subject: [PATCH 5/7] fix(gridfs): validate appendChunks before writing Plan every chunk slice before insertOne. An invalid length writes nothing. A payload that adds chunks must start at nextChunk; a same-byte replay of the stored prefix still succeeds. Co-authored-by: rocky --- docs/en/gridfs.md | 7 +++--- docs/zh/gridfs.md | 5 ++-- src/capabilities/gridfs/resumable.ts | 36 ++++++++++++++++++---------- test/unit/gridfs/gridfs.test.ts | 26 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/docs/en/gridfs.md b/docs/en/gridfs.md index 9ddb4fc..c72ae2d 100644 --- a/docs/en/gridfs.md +++ b/docs/en/gridfs.md @@ -54,7 +54,7 @@ Calling either method before `connect()` throws `NOT_CONNECTED`. ### Ranges -`downloadStream` ranges use an inclusive `start` and an exclusive `end`. Omitted `start` means `0`. Omitted `end` means the file length. A legal range satisfies `0 <= start < end <= length`. Anything else, including `start === end` and `start === length`, emits `GRIDFS_RANGE_INVALID`. When the failure depends on the file length, `details[0].length` is that length. Shape errors that can be rejected before a lookup omit `length`. `download()` rejects the promise with the same codes; it does not apply a range. +`downloadStream` ranges use an inclusive `start` and an exclusive `end`. Omitted `start` means `0`. Omitted `end` means the file length. A legal range satisfies `0 <= start < end <= length`. Anything else, including `start === end` and `start === length`, emits `GRIDFS_RANGE_INVALID`. When the failure depends on the file length, `details[0].length` is that length. Shape errors that can be rejected before a lookup omit `length`. `download()` reads the whole file and does not accept a range. ## Resumable upload @@ -80,13 +80,14 @@ Chunk bytes must follow the session size: - Every chunk except the last has length `chunkSizeBytes`. - The last chunk has length `length % chunkSizeBytes`, or `chunkSizeBytes` when the length divides evenly. -- A payload that does not split into those sizes throws `INVALID_ARGUMENT`. +- A payload that does not split into those sizes throws `INVALID_ARGUMENT`. The whole payload is checked before any chunk is inserted. `appendChunks` is addressed by `fromChunk`: - `fromChunk` past the contiguous prefix throws `GRIDFS_RESUME_MISMATCH` with `expectedNextChunk`. -- Replaying an earlier chunk with the same bytes is idempotent and does not move `nextChunk` backward. +- Replaying already stored chunks with the same bytes is idempotent and does not move `nextChunk` backward. - Replaying an earlier chunk with different bytes throws `GRIDFS_RESUME_MISMATCH`. +- A payload that contains any new chunk must start at `nextChunk`. Starting earlier throws `GRIDFS_RESUME_MISMATCH` and writes nothing. - A duplicate `{ files_id, n }` insert is accepted only when the stored bytes match. `finalizeResumableUpload` requires a contiguous prefix, a total length equal to the session length, and a sha256 equal to the session digest. It then inserts the files document, sets `metadata.sha256`, and deletes the session. Missing chunk indexes throw `GRIDFS_UPLOAD_INCOMPLETE` with `details[0].missingChunks`. A bad digest or a chunk of the wrong size throws `GRIDFS_INTEGRITY_MISMATCH`. The files document is not inserted in those cases. diff --git a/docs/zh/gridfs.md b/docs/zh/gridfs.md index bdb8535..88fa298 100644 --- a/docs/zh/gridfs.md +++ b/docs/zh/gridfs.md @@ -54,7 +54,7 @@ const modelBucket = model.gridfs(bucketName?: string, opts?: { ### 区间 -`downloadStream` 的区间是包含的 `start` 和排除的 `end`。省略 `start` 表示 `0`，省略 `end` 表示文件长度。合法区间满足 `0 <= start < end <= length`。其余情况，包括 `start === end` 和 `start === length`，都会发出 `GRIDFS_RANGE_INVALID`。依赖文件长度才能判断的失败会带上 `details[0].length`。查找文件之前就能拒绝的形态错误不含 `length`。`download()` 用同一个错误码拒绝 Promise，并且不接受区间。 +`downloadStream` 的区间是包含的 `start` 和排除的 `end`。省略 `start` 表示 `0`，省略 `end` 表示文件长度。合法区间满足 `0 <= start < end <= length`。其余情况，包括 `start === end` 和 `start === length`，都会发出 `GRIDFS_RANGE_INVALID`。依赖文件长度才能判断的失败会带上 `details[0].length`。查找文件之前就能拒绝的形态错误不含 `length`。`download()` 读取整个文件，不接受区间。 ## 可恢复上传 @@ -80,13 +80,14 @@ await bucket.abortResumableUpload(session.uploadId); - 除最后一片外，每片长度都是 `chunkSizeBytes`。 - 最后一片长度是 `length % chunkSizeBytes`；能够整除时仍是 `chunkSizeBytes`。 -- 无法按这些长度切开的 payload 抛出 `INVALID_ARGUMENT`。 +- 无法按这些长度切开的 payload 抛出 `INVALID_ARGUMENT`。整段 payload 会在插入任何分片之前完成检查。 `appendChunks` 以 `fromChunk` 定位： - `fromChunk` 超过从 0 开始的连续前缀时，抛出 `GRIDFS_RESUME_MISMATCH`，并带上 `expectedNextChunk`。 - 用相同字节重放已写入的分片是幂等的，不会把 `nextChunk` 往回退。 - 用不同字节重放已写入的分片会抛出 `GRIDFS_RESUME_MISMATCH`。 +- 只要 payload 含有新分片，`fromChunk` 就必须等于 `nextChunk`。从更早的序号开始会抛出 `GRIDFS_RESUME_MISMATCH`，并且不会写入。 - `{ files_id, n }` 唯一索引冲突只有在已存字节相同的时候才算成功。 `finalizeResumableUpload` 要求分片从 0 连续、总长度等于会话长度，且 sha256 等于会话摘要。然后插入 files 文档、写入 `metadata.sha256`，并删除会话。缺少的分片序号抛出 `GRIDFS_UPLOAD_INCOMPLETE`，`details[0].missingChunks` 列出缺口。摘要不符或分片长度不对抛出 `GRIDFS_INTEGRITY_MISMATCH`。这些失败都不会插入 files 文档。 diff --git a/src/capabilities/gridfs/resumable.ts b/src/capabilities/gridfs/resumable.ts index 3edb131..816f46a 100644 --- a/src/capabilities/gridfs/resumable.ts +++ b/src/capabilities/gridfs/resumable.ts @@ -81,18 +81,14 @@ export async function appendChunks( throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); } - const total = chunkCount(session.length, session.chunkSizeBytes); - let offset = 0; - let index = fromChunk; - while (offset < payload.length) { - if (index >= total) throw badChunkSize(); - const expected = chunkLengthAt(index, session.length, session.chunkSizeBytes); - if (offset + expected > payload.length) throw badChunkSize(); - const slice = payload.subarray(offset, offset + expected); - if (index < nextChunk) await assertStoredChunk(ctx, uploadId, index, slice); - else await writeChunk(ctx, uploadId, index, slice); - offset += expected; - index += 1; + const slices = planChunkSlices(payload, fromChunk, session.length, session.chunkSizeBytes); + const lastIndex = slices[slices.length - 1].index; + if (lastIndex >= nextChunk && fromChunk !== nextChunk) { + throw resumeMismatch('appendChunks', ctx.bucketName, { uploadId, fromChunk, expectedNextChunk: nextChunk }); + } + for (const slice of slices) { + if (slice.index < nextChunk) await assertStoredChunk(ctx, uploadId, slice.index, slice.bytes); + else await writeChunk(ctx, uploadId, slice.index, slice.bytes); } const advanced = await contiguousNext(ctx, uploadId); @@ -201,6 +197,22 @@ function badChunkSize() { return invalidArgument('appendChunks', 'data', 'chunk payload must match the session chunk size'); } +function planChunkSlices(payload: Buffer, fromChunk: number, length: number, chunkSizeBytes: number): Array<{ index: number; bytes: Buffer }> { + const total = chunkCount(length, chunkSizeBytes); + const slices: Array<{ index: number; bytes: Buffer }> = []; + let offset = 0; + let index = fromChunk; + while (offset < payload.length) { + if (index >= total) throw badChunkSize(); + const expected = chunkLengthAt(index, length, chunkSizeBytes); + if (offset + expected > payload.length) throw badChunkSize(); + slices.push({ index, bytes: payload.subarray(offset, offset + expected) }); + offset += expected; + index += 1; + } + return slices; +} + async function requirePending(ctx: GridFSContext, uploadId: ObjectId, operation: string): Promise { const doc = await ctx.db.collection(uploadsCollectionName(ctx.bucketName)).findOne({ _id: uploadId }); if (!doc || doc.status !== 'pending') throw fileNotFound(operation, ctx, { id: uploadId }); diff --git a/test/unit/gridfs/gridfs.test.ts b/test/unit/gridfs/gridfs.test.ts index 521b63e..d3b6274 100644 --- a/test/unit/gridfs/gridfs.test.ts +++ b/test/unit/gridfs/gridfs.test.ts @@ -329,6 +329,32 @@ describe('gridfs bucket', () => { await bucket.abortResumableUpload(new ObjectId()); }); + it('validates an append before writing and does not extend a replay', async () => { + const { bucket, db } = bucketFor({ driver: false }); + const payload = Buffer.from('abcdefghijkl'); + const started = await bucket.beginResumableUpload('big.bin', { length: payload.length, sha256: sha(payload), chunkSizeBytes: 4 }); + await assert.rejects(() => bucket.appendChunks(started.uploadId, 0, Buffer.from('abcdef')), (error: unknown) => codeOf(error) === 'INVALID_ARGUMENT'); + assert.equal((await db.collection('fs.chunks').find({ files_id: started.uploadId }).toArray()).length, 0); + + await bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 8)); + await assert.rejects( + () => bucket.appendChunks(started.uploadId, 0, Buffer.concat([payload.subarray(0, 8), Buffer.from('WXYZ')])), + (error: unknown) => { + assert.equal(codeOf(error), 'GRIDFS_RESUME_MISMATCH'); + assert.equal(detailsOf(error).expectedNextChunk, 2); + return true; + }, + ); + const stored = await db.collection('fs.chunks').find({ files_id: started.uploadId }).sort({ n: 1 }).toArray(); + assert.deepEqual(stored.map((chunk) => chunk.n), [0, 1]); + assert.equal((await db.collection('fs.uploads').findOne({ _id: started.uploadId }))?.nextChunk, 2); + + const replay = await bucket.appendChunks(started.uploadId, 0, payload.subarray(0, 8)); + assert.equal(replay.nextChunk, 2); + const advanced = await bucket.appendChunks(started.uploadId, 2, payload.subarray(8)); + assert.equal(advanced.nextChunk, 3); + }); + it('abort removes pending chunks and ensureIndexes is idempotent', async () => { const { bucket, db } = bucketFor({ driver: false }); const started = await bucket.beginResumableUpload('partial.bin', { length: 4, sha256: sha(Buffer.from('abcd')), chunkSizeBytes: 4 }); From 70eb64f913ee5ede53964e8b66de0b2023f03d84 Mon Sep 17 00:00:00 2001 From: Rocky Shi <175696301+rockyshi1993@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:43:15 +0000 Subject: [PATCH 6/7] chore(budget): raise runtime bundle ceilings for GridFS 3.5.0 The sync gridfs() API stays in the single-file bundle, so the CJS and ESM ceilings move to 872000 and 870000 bytes. Co-authored-by: rocky --- changelogs/unreleased.md | 1 + config/package-budget.json | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/changelogs/unreleased.md b/changelogs/unreleased.md index ba8e2b3..6c76a08 100644 --- a/changelogs/unreleased.md +++ b/changelogs/unreleased.md @@ -3,6 +3,7 @@ ## Changes after v3.4.0 - Add GridFS buckets on `msq.gridfs()` and `model.gridfs()`, including upload, download, ranges, `ensureIndexes`, and phase-1 resumable upload. `writePathPolicy` gains a `gridfs` field and category. Policy violations stay `INVALID_OPERATION` with `reason: 'WRITE_PATH_POLICY'`. These notes stage the work after 3.4.0; the npm package version is unchanged and this file is not a 3.5.0 release. The Search index lifecycle API remains in the [3.4.0 versioned notes](./v3.4.0.md). +- Raise the CJS and ESM runtime package budgets for GridFS 3.5.0. `gridfs()` stays synchronous, so the single-file build keeps that code in the main bundle. ## Archived preparation notes for the 3.3.2 candidate diff --git a/config/package-budget.json b/config/package-budget.json index b6a66ef..bafec33 100644 --- a/config/package-budget.json +++ b/config/package-budget.json @@ -1,7 +1,8 @@ { + "comment": "GridFS 3.5.0 stays in the main bundle: sync gridfs() cannot be split out of the single-file build.", "artifacts": { - "dist/cjs/index.cjs": 840000, - "dist/esm/index.mjs": 840000, + "dist/cjs/index.cjs": 872000, + "dist/esm/index.mjs": 870000, "dist/cjs/cli/data-task.cjs": 50000 } } From 5d803808c2246b2abc42d82dad6df070b95acbe0 Mon Sep 17 00:00:00 2001 From: Rocky Shi <175696301+rockyshi1993@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:59:27 +0000 Subject: [PATCH 7/7] fix(gridfs): store contentType on metadata Driver 7 removed contentType from the upload stream options. The public option is written as metadata.contentType and still returned on FileInfo. The CJS bundle ceiling moves to 873000 bytes. Co-authored-by: rocky --- changelogs/unreleased.md | 2 +- config/package-budget.json | 2 +- docs/en/gridfs.md | 2 +- docs/zh/gridfs.md | 2 +- src/capabilities/gridfs/driver.ts | 21 +++++++++++----- src/capabilities/gridfs/normalize.ts | 4 +++- src/capabilities/gridfs/resumable.ts | 7 ++++-- src/capabilities/gridfs/validate.ts | 6 +++++ test/integration/gridfs/gridfs.test.ts | 2 ++ test/unit/gridfs/gridfs.test.ts | 33 ++++++++++++++++++++++++++ 10 files changed, 68 insertions(+), 13 deletions(-) diff --git a/changelogs/unreleased.md b/changelogs/unreleased.md index 6c76a08..9c81d86 100644 --- a/changelogs/unreleased.md +++ b/changelogs/unreleased.md @@ -3,7 +3,7 @@ ## Changes after v3.4.0 - Add GridFS buckets on `msq.gridfs()` and `model.gridfs()`, including upload, download, ranges, `ensureIndexes`, and phase-1 resumable upload. `writePathPolicy` gains a `gridfs` field and category. Policy violations stay `INVALID_OPERATION` with `reason: 'WRITE_PATH_POLICY'`. These notes stage the work after 3.4.0; the npm package version is unchanged and this file is not a 3.5.0 release. The Search index lifecycle API remains in the [3.4.0 versioned notes](./v3.4.0.md). -- Raise the CJS and ESM runtime package budgets for GridFS 3.5.0. `gridfs()` stays synchronous, so the single-file build keeps that code in the main bundle. +- Raise the CJS and ESM runtime package budgets for GridFS 3.5.0. `gridfs()` stays synchronous, so the single-file build keeps that code in the main bundle. The CJS ceiling is 873000 bytes after `contentType` moved onto file metadata. ## Archived preparation notes for the 3.3.2 candidate diff --git a/config/package-budget.json b/config/package-budget.json index bafec33..4011720 100644 --- a/config/package-budget.json +++ b/config/package-budget.json @@ -1,7 +1,7 @@ { "comment": "GridFS 3.5.0 stays in the main bundle: sync gridfs() cannot be split out of the single-file build.", "artifacts": { - "dist/cjs/index.cjs": 872000, + "dist/cjs/index.cjs": 873000, "dist/esm/index.mjs": 870000, "dist/cjs/cli/data-task.cjs": 50000 } diff --git a/docs/en/gridfs.md b/docs/en/gridfs.md index c72ae2d..602dcd5 100644 --- a/docs/en/gridfs.md +++ b/docs/en/gridfs.md @@ -50,7 +50,7 @@ Calling either method before `connect()` throws `NOT_CONNECTED`. - `skip` with `sha256` returns the newest visible file with that digest. `skip` without `sha256` returns the newest visible file with the same filename. `duplicate` always creates another file. - `uploadStream` stores `sha256` in metadata only. It does not check the bytes and it does not accept `ifExists`. -`metadata.sha256` written by these helpers overwrites a caller-supplied `metadata.sha256`. +`metadata.sha256` written by these helpers overwrites a caller-supplied `metadata.sha256`. A provided `contentType` is stored as `metadata.contentType` and overwrites a caller-supplied value of that field. `FileInfo.contentType` returns it. An older file that still has a top-level `contentType` is returned the same way. ### Ranges diff --git a/docs/zh/gridfs.md b/docs/zh/gridfs.md index 88fa298..f003267 100644 --- a/docs/zh/gridfs.md +++ b/docs/zh/gridfs.md @@ -50,7 +50,7 @@ const modelBucket = model.gridfs(bucketName?: string, opts?: { - 带 `sha256` 的 `skip` 返回该摘要最新的可见文件。不带 `sha256` 的 `skip` 返回同名最新可见文件。`duplicate` 总是再创建一份。 - `uploadStream` 只把 `sha256` 写入 metadata。它不校验字节，也不接受 `ifExists`。 -这些助手写入的 `metadata.sha256` 会覆盖调用方自己放在 `metadata.sha256` 里的值。 +这些助手写入的 `metadata.sha256` 会覆盖调用方自己放在 `metadata.sha256` 里的值。提供的 `contentType` 写入 `metadata.contentType`，并覆盖调用方在该字段上的值。`FileInfo.contentType` 会返回它。旧文件上残留的顶层 `contentType` 同样会返回。 ### 区间 diff --git a/src/capabilities/gridfs/driver.ts b/src/capabilities/gridfs/driver.ts index 98918c1..176e60a 100644 --- a/src/capabilities/gridfs/driver.ts +++ b/src/capabilities/gridfs/driver.ts @@ -1,15 +1,24 @@ -import { GridFSBucket, type Db, type ObjectId } from 'mongodb'; +import { GridFSBucket, type Db, type Document, type GridFSBucketWriteStreamOptions, type ObjectId } from 'mongodb'; import type { GridFSDriver } from './types'; +import { withContentType } from './validate'; + +/** Options passed to the driver. `contentType` is folded into `metadata` and is not a stream option. */ +export function gridFSUploadStreamOptions(options: { + chunkSizeBytes: number; + metadata?: Document; + contentType?: string; +}): GridFSBucketWriteStreamOptions { + const streamOptions: GridFSBucketWriteStreamOptions = { chunkSizeBytes: options.chunkSizeBytes }; + const metadata = withContentType(options.metadata, options.contentType); + if (metadata !== undefined) streamOptions.metadata = metadata; + return streamOptions; +} export function createGridFSDriver(db: Db, bucketName: string, chunkSizeBytes: number): GridFSDriver { const bucket = new GridFSBucket(db, { bucketName, chunkSizeBytes }); return { openUploadStream(id, filename, options) { - return bucket.openUploadStreamWithId(id, filename, { - chunkSizeBytes: options.chunkSizeBytes, - metadata: options.metadata, - contentType: options.contentType, - }); + return bucket.openUploadStreamWithId(id, filename, gridFSUploadStreamOptions(options)); }, openDownloadStream(id: ObjectId, range) { return bucket.openDownloadStream(id, range); diff --git a/src/capabilities/gridfs/normalize.ts b/src/capabilities/gridfs/normalize.ts index 9b8c91c..5dc9fb0 100644 --- a/src/capabilities/gridfs/normalize.ts +++ b/src/capabilities/gridfs/normalize.ts @@ -22,10 +22,12 @@ export function toFileInfo(doc: Record): GridFSFileInfo { chunkSize: typeof doc.chunkSize === 'number' ? doc.chunkSize : 0, uploadDate: doc.uploadDate instanceof Date ? doc.uploadDate : new Date(doc.uploadDate as string | number), }; - if (typeof doc.contentType === 'string') info.contentType = doc.contentType; if (doc.metadata !== undefined && doc.metadata !== null && typeof doc.metadata === 'object') { info.metadata = doc.metadata as Record; } + const storedContentType = info.metadata?.contentType; + if (typeof doc.contentType === 'string') info.contentType = doc.contentType; + else if (typeof storedContentType === 'string') info.contentType = storedContentType; return info; } diff --git a/src/capabilities/gridfs/resumable.ts b/src/capabilities/gridfs/resumable.ts index 816f46a..a74970f 100644 --- a/src/capabilities/gridfs/resumable.ts +++ b/src/capabilities/gridfs/resumable.ts @@ -17,6 +17,7 @@ import { chunkLengthAt, isReadableStream, toNodeBuffer, + withContentType, } from './validate'; import { readReadable } from './bytes'; @@ -149,7 +150,10 @@ export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: Obje throw integrityMismatch('finalizeResumableUpload', ctx.bucketName, { uploadId, expected: session.sha256, actual }); } - const metadata = buildMetadata(session.metadata, session.sha256, 'finalizeResumableUpload'); + const metadata = withContentType( + buildMetadata(session.metadata, session.sha256, 'finalizeResumableUpload'), + session.contentType, + ); try { await files.insertOne({ _id: uploadId, @@ -157,7 +161,6 @@ export async function finalizeResumableUpload(ctx: GridFSContext, uploadId: Obje chunkSize: session.chunkSizeBytes, uploadDate: new Date(), filename: session.filename, - ...(session.contentType !== undefined ? { contentType: session.contentType } : {}), ...(metadata ? { metadata } : {}), }); } catch (error) { diff --git a/src/capabilities/gridfs/validate.ts b/src/capabilities/gridfs/validate.ts index 3e03f46..9ed3278 100644 --- a/src/capabilities/gridfs/validate.ts +++ b/src/capabilities/gridfs/validate.ts @@ -119,6 +119,12 @@ export function buildMetadata(metadata: Record | undefined, sha return out; } +/** Driver 7 dropped the upload-stream `contentType` option. Store it on metadata instead. */ +export function withContentType(metadata: Document | undefined, contentType: string | undefined): Document | undefined { + if (contentType === undefined) return metadata; + return { ...(metadata ?? {}), contentType }; +} + export function parseRangeShape(range: GridFSDownloadRange | undefined, operation: string, bucket: string): ParsedRange { if (range === undefined) return {}; if (!isPlainRecord(range)) throw rangeInvalid(operation, bucket, undefined, undefined); diff --git a/test/integration/gridfs/gridfs.test.ts b/test/integration/gridfs/gridfs.test.ts index cf26445..b090e2d 100644 --- a/test/integration/gridfs/gridfs.test.ts +++ b/test/integration/gridfs/gridfs.test.ts @@ -59,6 +59,7 @@ describe('gridfs memory-server integration', () => { assert.equal(info.length, payload.length); assert.equal(info.chunkSize, 8); assert.equal(info.metadata?.sha256, digest); + assert.equal(info.metadata?.contentType, 'application/octet-stream'); assert.equal(info.metadata?.tenantId, 't1'); assert.deepEqual(await bucket.download(info.id), payload); assert.deepEqual(await bucket.download({ filename: 'report.bin' }), payload); @@ -131,6 +132,7 @@ describe('gridfs memory-server integration', () => { assert.equal(next.nextChunk, 3); const info = await resumed.finalizeResumableUpload(started.uploadId); assert.equal(info.metadata?.sha256, digest); + assert.equal(info.metadata?.contentType, 'application/octet-stream'); assert.equal(info.contentType, 'application/octet-stream'); assert.deepEqual(await resumed.download(info.id), payload); const skipped = await resumed.upload('again.bin', payload, { sha256: digest, ifExists: 'skip' }); diff --git a/test/unit/gridfs/gridfs.test.ts b/test/unit/gridfs/gridfs.test.ts index d3b6274..163524e 100644 --- a/test/unit/gridfs/gridfs.test.ts +++ b/test/unit/gridfs/gridfs.test.ts @@ -4,6 +4,8 @@ import assert from 'node:assert/strict'; import { ObjectId } from 'mongodb'; import { createHash } from 'node:crypto'; import { createMonSQLizeGridFSBucket } from '../../../src/capabilities/gridfs'; +import { gridFSUploadStreamOptions } from '../../../src/capabilities/gridfs/driver'; +import { toFileInfo } from '../../../src/capabilities/gridfs/normalize'; import { openModelGridFS, openRuntimeGridFS } from '../../../src/entry/runtime-gridfs'; import { normalizeWritePathPolicy } from '../../../src/capabilities/write-path-policy'; import { sha256Hex } from '../../../src/capabilities/gridfs/validate'; @@ -372,6 +374,37 @@ describe('gridfs bucket', () => { assert.ok(indexes.some((index) => index.key?.['metadata.sha256'] === 1)); }); + it('stores contentType on metadata and omits it from driver upload options', () => { + const options = gridFSUploadStreamOptions({ + chunkSizeBytes: 4, + metadata: { kind: 'bin', contentType: 'from-metadata' }, + contentType: 'text/plain', + }); + assert.equal(Object.prototype.hasOwnProperty.call(options, 'contentType'), false); + assert.equal(options.metadata?.contentType, 'text/plain'); + assert.equal(options.metadata?.kind, 'bin'); + const unchanged = gridFSUploadStreamOptions({ chunkSizeBytes: 4, metadata: { kind: 'bin' } }); + assert.equal(unchanged.metadata?.contentType, undefined); + const info = toFileInfo({ + _id: new ObjectId(), + filename: 'a.bin', + length: 1, + chunkSize: 4, + uploadDate: new Date(), + metadata: { contentType: 'text/plain' }, + }); + assert.equal(info.contentType, 'text/plain'); + const legacy = toFileInfo({ + _id: new ObjectId(), + filename: 'b.bin', + length: 1, + chunkSize: 4, + uploadDate: new Date(), + contentType: 'application/octet-stream', + }); + assert.equal(legacy.contentType, 'application/octet-stream'); + }); + it('routes a named pool to that client and keeps Model pool and database fixed', () => { const seen: string[] = []; const client = (label: string) => ({