From d0c268a9a1580500faf108c83dd8e1a86980f35c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 11 Nov 2026 14:37:05 +0000 Subject: [PATCH] fix: open /privacy and /terms to crawlers The x402 crawl gate answered 402 to ClaudeBot and other crawlers on the legal pages, which a connector directory's review crawler must read. Co-Authored-By: Claude Opus 5.5 --- src/crawl.mjs | 4 +++- test/crawl.test.mjs | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/crawl.mjs b/src/crawl.mjs index ce9dec3..0881fdc 100644 --- a/src/crawl.mjs +++ b/src/crawl.mjs @@ -16,7 +16,9 @@ export function createCrawlGate(env = process.env) { coinpay: { apiKey }, payTo, contact: 'mailto:anthony@profullstack.com', - openPaths: ['/healthz', '/llms.txt', '/install.sh', '/install', '/commands.json', '/blog/rss.xml', '/blog/feed.xml'], + openPaths: ['/healthz', '/llms.txt', '/install.sh', '/install', '/commands.json', '/blog/rss.xml', '/blog/feed.xml', + // The legal pages: a connector directory's review crawler must read them. + '/privacy', '/terms'], }); } diff --git a/test/crawl.test.mjs b/test/crawl.test.mjs index 1b7ae4b..86ddd10 100644 --- a/test/crawl.test.mjs +++ b/test/crawl.test.mjs @@ -44,3 +44,11 @@ test('a browser and the health check are not charged', async () => { assert.equal((await get('/', CHROME)).status, 200); assert.equal((await get('/healthz', { 'user-agent': GPT })).status, 200); }); + +test('legal pages are open to crawlers and browsers alike', async () => { + const CLAUDEBOT = 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'; + for (const path of ['/privacy', '/terms']) { + for (const ua of [GPT, CLAUDEBOT]) assert.equal((await get(path, { 'user-agent': ua })).status, 200, `${path} ${ua}`); + assert.equal((await get(path, CHROME)).status, 200); + } +});