diff --git a/src/crawl.mjs b/src/crawl.mjs index ce9dec3..0881fdc 100644 --- a/src/crawl.mjs +++ b/src/crawl.mjs @@ -16,7 +16,9 @@ export function createCrawlGate(env = process.env) { coinpay: { apiKey }, payTo, contact: 'mailto:anthony@profullstack.com', - openPaths: ['/healthz', '/llms.txt', '/install.sh', '/install', '/commands.json', '/blog/rss.xml', '/blog/feed.xml'], + openPaths: ['/healthz', '/llms.txt', '/install.sh', '/install', '/commands.json', '/blog/rss.xml', '/blog/feed.xml', + // The legal pages: a connector directory's review crawler must read them. + '/privacy', '/terms'], }); } diff --git a/test/crawl.test.mjs b/test/crawl.test.mjs index 1b7ae4b..86ddd10 100644 --- a/test/crawl.test.mjs +++ b/test/crawl.test.mjs @@ -44,3 +44,11 @@ test('a browser and the health check are not charged', async () => { assert.equal((await get('/', CHROME)).status, 200); assert.equal((await get('/healthz', { 'user-agent': GPT })).status, 200); }); + +test('legal pages are open to crawlers and browsers alike', async () => { + const CLAUDEBOT = 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'; + for (const path of ['/privacy', '/terms']) { + for (const ua of [GPT, CLAUDEBOT]) assert.equal((await get(path, { 'user-agent': ua })).status, 200, `${path} ${ua}`); + assert.equal((await get(path, CHROME)).status, 200); + } +});