diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts new file mode 100644 index 0000000000..06d2b92764 --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -0,0 +1,446 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { + IDeactivatedOctolensKeywordMapping, + IOctolensMappingDiscoveryHealth, +} from '@crowd/data-access-layer/src/octolens/health' +import type { Logger } from '@crowd/logging' +import { SlackPersona } from '@crowd/slack' + +import { + DEACTIVATIONS_CURSOR_KEY, + IOctolensMappingHealthAlert, + IOctolensMappingHealthDeps, + MAX_SECTION_CHARS, + STALE_ALERTED_KEY, + capLines, + formatName, + hasDeactivationBurst, + isDeactivationBurst, + runOctolensMappingHealthCheck, +} from './octolensMappingHealth' + +const CHECKED_AT = new Date('2026-10-15T12:30:00.000Z') +const UNTIL = '2026-10-15T12:25:00.000Z' +const FIRST_LOOKBACK = '2026-10-14T12:25:00.000Z' +const RUN_AT = new Date('2026-10-15T12:00:00.000Z') +const STALE_AT = new Date('2026-10-15T08:00:00.000Z') + +function health( + overrides: Partial = {}, +): IOctolensMappingDiscoveryHealth { + return { + activeMappings: 20, + integrations: 4, + lastDiscoveredAt: RUN_AT, + checkedAt: CHECKED_AT, + ...overrides, + } +} + +function deactivation( + keywordId: number, + overrides: Partial = {}, +): IDeactivatedOctolensKeywordMapping { + return { + integrationId: 'int-1', + keywordId, + keyword: `keyword-${keywordId}`, + segmentId: 'seg-1', + segmentName: 'Project One', + segmentSlug: 'project-one', + deactivatedAt: RUN_AT, + integrationDeleted: false, + nowMappedToSegmentName: null, + ...overrides, + } +} + +function setup(options: { + health: IOctolensMappingDiscoveryHealth | null + deactivations?: IDeactivatedOctolensKeywordMapping[] + store?: Record + sendSucceeds?: boolean +}) { + const store = new Map(Object.entries(options.store ?? {})) + const ttls = new Map() + const sent: IOctolensMappingHealthAlert[] = [] + const log = { warn: vi.fn(), error: vi.fn() } + const listDeactivatedBetween = vi.fn(async () => options.deactivations ?? []) + const deps: IOctolensMappingHealthDeps = { + log: log as unknown as Logger, + getHealth: async () => options.health, + listDeactivatedBetween, + getKey: async (key) => store.get(key) ?? null, + setKey: async (key, value, expireAfterSeconds) => { + store.set(key, value) + ttls.set(key, expireAfterSeconds) + }, + sendAlert: async (alert) => { + sent.push(alert) + return options.sendSucceeds ?? true + }, + } + return { deps, store, ttls, sent, log, listDeactivatedBetween } +} + +function alertText(alert: IOctolensMappingHealthAlert): string { + return alert.sections.map((s) => `${s.title}\n${s.text}`).join('\n') +} + +describe('runOctolensMappingHealthCheck', () => { + it('stays quiet and keeps no state while there are no active mappings', async () => { + const { deps, store, sent, listDeactivatedBetween } = setup({ health: null }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + expect(store.size).toBe(0) + expect(listDeactivatedBetween).not.toHaveBeenCalled() + }) + + it('sends nothing when healthy and moves the cursor', async () => { + const { deps, store, sent } = setup({ health: health() }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) + }) + + it.each([ + { name: 'first run looks back 24h', cursor: undefined, since: FIRST_LOOKBACK }, + { name: 'later runs continue from the cursor', cursor: '2026-10-15T11:25:00.000Z' }, + { name: 'an unreadable cursor looks back 24h', cursor: 'not-a-date', since: FIRST_LOOKBACK }, + ])('queries the window: $name', async ({ cursor, since }) => { + const { deps, store, listDeactivatedBetween } = setup({ + health: health(), + store: cursor ? { [DEACTIVATIONS_CURSOR_KEY]: cursor } : {}, + }) + + await runOctolensMappingHealthCheck(deps) + + expect(listDeactivatedBetween).toHaveBeenCalledWith( + new Date(since ?? (cursor as string)), + new Date(UNTIL), + ) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) + }) + + it('does not query when the cursor is already at the window end', async () => { + const { deps, listDeactivatedBetween } = setup({ + health: health(), + store: { [DEACTIVATIONS_CURSOR_KEY]: UNTIL }, + }) + + await runOctolensMappingHealthCheck(deps) + + expect(listDeactivatedBetween).not.toHaveBeenCalled() + }) + + it.each([ + { name: 'first alert of an episode', lastDiscoveredAt: STALE_AT, alerted: undefined, sends: 1 }, + { + name: 'same episode already alerted', + lastDiscoveredAt: STALE_AT, + alerted: STALE_AT, + sends: 0, + }, + { + name: 'new episode after an older one', + lastDiscoveredAt: STALE_AT, + alerted: new Date('2026-10-14T08:00:00.000Z'), + sends: 1, + }, + { + name: 'within the threshold', + lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z'), + alerted: undefined, + sends: 0, + }, + ])('staleness: $name', async ({ lastDiscoveredAt, alerted, sends }) => { + const { deps, store, ttls, sent } = setup({ + health: health({ lastDiscoveredAt }), + store: alerted ? { [STALE_ALERTED_KEY]: alerted.toISOString() } : {}, + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(sends) + if (sends > 0) { + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + expect(store.get(STALE_ALERTED_KEY)).toBe(lastDiscoveredAt.toISOString()) + expect(ttls.get(STALE_ALERTED_KEY)).toBe(24 * 60 * 60) + } + }) + + it('describes the stale state and where to look', async () => { + const { deps, sent } = setup({ health: health({ lastDiscoveredAt: STALE_AT }) }) + + await runOctolensMappingHealthCheck(deps) + + const text = alertText(sent[0]) + expect(text).toContain('Last write: 2026-10-15 08:00 UTC (4h ago)') + expect(text).toContain('Active mappings: 20 across 4 projects') + expect(text).toContain('CROWD_OCTOLENS_API_TOKEN') + expect(text).toContain('`--dry-run` first') + expect(text).toContain( + '`pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions -- --from 2026-10-15T08:00:00.000Z`', + ) + expect(text).toContain('script:backfill-octolens-mentions -- --keywords KEYWORD_ID,KEYWORD_ID') + expect(text).not.toContain('nothing needs recovering') + }) + + it('sends the stale alert before the deactivation alert', async () => { + const { deps, sent } = setup({ + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [deactivation(1)], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent.map((a) => a.title)).toEqual([ + 'Octolens mapping discovery looks stale', + 'Octolens keyword mappings deactivated', + ]) + }) + + it('runs both checks when one throws', async () => { + const { deps, sent, log, store } = setup({ + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [deactivation(1)], + }) + const failing = { ...deps, listDeactivatedBetween: vi.fn().mockRejectedValue(new Error('db')) } + + const result = await runOctolensMappingHealthCheck(failing) + + expect(sent.map((a) => a.title)).toEqual(['Octolens mapping discovery looks stale']) + expect(result.failures).toBe(1) + expect(log.error).toHaveBeenCalledOnce() + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(FIRST_LOOKBACK) + + const staleThrows = { ...deps, getKey: vi.fn(deps.getKey) } + staleThrows.getKey.mockRejectedValueOnce(new Error('redis')) + await runOctolensMappingHealthCheck(staleThrows) + + expect(sent[sent.length - 1].title).toBe('Octolens keyword mappings deactivated') + }) + + it.each([ + { + name: 'stale alert stays unmarked', + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [], + store: {}, + key: STALE_ALERTED_KEY, + expected: undefined, + }, + { + name: 'cursor stays put', + health: health(), + deactivations: [deactivation(1)], + store: { [DEACTIVATIONS_CURSOR_KEY]: '2026-10-15T11:25:00.000Z' }, + key: DEACTIVATIONS_CURSOR_KEY, + expected: '2026-10-15T11:25:00.000Z', + }, + { + name: 'first lookback is pinned', + health: health(), + deactivations: [deactivation(1)], + store: {}, + key: DEACTIVATIONS_CURSOR_KEY, + expected: FIRST_LOOKBACK, + }, + ])('a failed send leaves state for a retry: $name', async (c) => { + const { deps, store, log } = setup({ ...c, sendSucceeds: false }) + + const result = await runOctolensMappingHealthCheck(deps) + + expect(result.failures).toBe(1) + expect(store.get(c.key)).toBe(c.expected) + expect(log.warn).toHaveBeenCalledOnce() + }) + + it('logs the window start when the deactivation alert fails to send', async () => { + const { deps, log } = setup({ + health: health(), + deactivations: [deactivation(1)], + sendSucceeds: false, + }) + + await runOctolensMappingHealthCheck(deps) + + expect(log.warn).toHaveBeenCalledWith( + { windowStart: FIRST_LOOKBACK, deactivations: 1 }, + expect.any(String), + ) + }) + + it('reports a burst in one run as a warning listing every keyword and project', async () => { + const { deps, store, sent } = setup({ + health: health(), + deactivations: [ + deactivation(1), + deactivation(2), + deactivation(3, { + integrationId: 'int-2', + segmentName: 'Project Two', + segmentSlug: 'project-two', + integrationDeleted: true, + }), + ], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + expect(sent[0].title).toBe('Octolens keyword mappings deactivated in bulk') + const text = alertText(sent[0]) + expect(text).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') + expect(text).toContain('Deactivated mappings (3, 20 still active)') + expect(text).toContain( + '`keyword-1` (1) → `Project One` (`project-one`), deactivated 2026-10-15 12:00 UTC', + ) + expect(text).toContain( + '`keyword-3` (3) → `Project Two` (`project-two`), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', + ) + expect(text).toContain('Nothing is re-fetched automatically') + expect(text).toContain('script:backfill-octolens-mentions -- --keywords KEYWORD_ID,KEYWORD_ID') + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) + }) + + it('reports deactivations spread over runs as info', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: [10, 11, 12].map((hour, i) => + deactivation(i + 1, { deactivatedAt: new Date(`2026-10-15T${hour}:00:00.000Z`) }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) + }) + + it('names the new project of a moved keyword and leaves moves out of the burst', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: [1, 2, 3].map((id) => + deactivation(id, { nowMappedToSegmentName: id === 1 ? 'Project Two' : null }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) + expect(alertText(sent[0])).toContain( + '`keyword-1` (1) → `Project One` (`project-one`), deactivated 2026-10-15 12:00 UTC, now mapped to `Project Two`', + ) + }) + + it('caps the listed mappings by count and keeps the full count', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: Array.from({ length: 30 }, (_, i) => deactivation(i + 1)), + }) + + await runOctolensMappingHealthCheck(deps) + + const text = alertText(sent[0]) + expect(text).toContain('Deactivated mappings (30, 20 still active)') + expect(text).toContain('`keyword-25` (25)') + expect(text).not.toContain('`keyword-26` (26)') + expect(text).toContain('… and 5 more') + }) + + it('lists the newest deactivations first when the list is capped', async () => { + const at = (minute: number) => new Date(RUN_AT.getTime() + minute * 60 * 1000) + const { deps, sent } = setup({ + health: health(), + deactivations: Array.from({ length: 30 }, (_, i) => + deactivation(i + 1, { deactivatedAt: at(i) }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + const lines = sent[0].sections[1].text.split('\n') + expect(lines[0]).toContain('`keyword-30` (30)') + expect(lines[24]).toContain('`keyword-6` (6)') + expect(lines[25]).toBe('… and 5 more') + }) + + it('keeps every section under the Slack block size after escaping long names', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: Array.from({ length: 25 }, (_, i) => + deactivation(i + 1, { keyword: '&'.repeat(500), segmentName: '<'.repeat(500) }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + for (const section of sent[0].sections) { + expect(section.text.length).toBeLessThanOrEqual(MAX_SECTION_CHARS) + } + expect(JSON.stringify(sent[0]).length).toBeLessThan(20 * 1024) + expect(alertText(sent[0])).toMatch(/… and \d+ more$/) + }) +}) + +describe('formatName', () => { + it.each([ + { name: 'plain', input: 'kubernetes', expected: '`kubernetes`' }, + { name: 'mrkdwn markers', input: '*bold* _it_ ~st~', expected: '`*bold* _it_ ~st~`' }, + { name: 'backtick', input: 'a`b', expected: '`aˋb`' }, + { name: 'html entities', input: '', expected: '`<a&b>`' }, + { name: 'newlines', input: 'a\nb', expected: '`a b`' }, + ])('$name', ({ input, expected }) => { + expect(formatName(input)).toBe(expected) + }) +}) + +describe('capLines', () => { + it.each([ + { name: 'fits', lines: ['a', 'b'], maxLines: 5, maxChars: 100, expected: 'a\nb' }, + { + name: 'line cap', + lines: ['a', 'b', 'c'], + maxLines: 2, + maxChars: 100, + expected: 'a\nb\n… and 1 more', + }, + { + name: 'char cap', + lines: ['x'.repeat(30), 'y'.repeat(30)], + maxLines: 5, + maxChars: 60, + expected: `${'x'.repeat(30)}\n… and 1 more`, + }, + ])('$name', ({ lines, maxLines, maxChars, expected }) => { + expect(capLines(lines, maxLines, maxChars)).toBe(expected) + }) +}) + +describe('isDeactivationBurst', () => { + it.each([ + [3, 1000, true], + [1, 3, true], + [2, 6, true], + [2, 7, false], + ])('%i deactivated with %i still active is a burst: %s', (deactivated, active, expected) => { + expect(isDeactivationBurst(deactivated, active)).toBe(expected) + }) +}) + +describe('hasDeactivationBurst', () => { + it('judges each run separately by its shared deactivation timestamp', () => { + const at = (hour: number) => new Date(`2026-10-15T${hour}:00:00.000Z`) + const spread = [10, 11, 12].map((hour, i) => deactivation(i + 1, { deactivatedAt: at(hour) })) + const oneRun = [1, 2, 3].map((id) => deactivation(id, { deactivatedAt: at(10) })) + + expect(hasDeactivationBurst(spread, 100)).toBe(false) + expect(hasDeactivationBurst(oneRun, 100)).toBe(true) + }) +}) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts new file mode 100644 index 0000000000..ce4b497802 --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -0,0 +1,302 @@ +import type { + IDeactivatedOctolensKeywordMapping, + IOctolensMappingDiscoveryHealth, +} from '@crowd/data-access-layer/src/octolens/health' +import { Logger } from '@crowd/logging' +import { SlackMessageSection, SlackPersona, escapeSlackText } from '@crowd/slack' + +const STALE_AFTER_HOURS = 3 +const STALE_REMINDER_HOURS = 24 +const BURST_MIN_DEACTIVATIONS = 3 +const BURST_MIN_SHARE = 0.25 +const CURSOR_SAFETY_MARGIN_MINUTES = 5 +const INITIAL_LOOKBACK_HOURS = 24 +const MAX_LISTED_KEYWORDS = 25 +// Under @crowd/slack's 2,900-char section block, so the list is never split or pruned. +export const MAX_SECTION_CHARS = 2800 +const MAX_NAME_LENGTH = 100 +const OVERFLOW_NOTE_RESERVE = 20 + +export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' +export const STALE_ALERTED_KEY = 'octolens-mapping-health:stale-alerted' + +const BACKFILL_COMMAND = + 'pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions --' + +const HOUR_MS = 60 * 60 * 1000 +const MINUTE_MS = 60 * 1000 + +export interface IOctolensMappingHealthAlert { + persona: SlackPersona + title: string + sections: SlackMessageSection[] +} + +export interface IOctolensMappingHealthDeps { + log: Logger + getHealth: () => Promise + listDeactivatedBetween: ( + since: Date, + until: Date, + ) => Promise + getKey: (key: string) => Promise + setKey: (key: string, value: string, expireAfterSeconds?: number) => Promise + sendAlert: (alert: IOctolensMappingHealthAlert) => Promise +} + +export interface IOctolensMappingHealthResult { + activeMappings: number + staleAlerted: boolean + deactivationsReported: number + failures: number +} + +function hoursSince(from: Date, now: Date): number { + return (now.getTime() - from.getTime()) / HOUR_MS +} + +function isStale(health: IOctolensMappingDiscoveryHealth): boolean { + return hoursSince(health.lastDiscoveredAt, health.checkedAt) > STALE_AFTER_HOURS +} + +export function isDeactivationBurst(deactivated: number, activeMappings: number): boolean { + if (deactivated >= BURST_MIN_DEACTIVATIONS) { + return true + } + const total = activeMappings + deactivated + return total > 0 && deactivated / total >= BURST_MIN_SHARE +} + +// One discovery run deactivates in one transaction, so its rows share one now() timestamp. +// A keyword moved to another project is a deliberate change, not discovery losing it. +export function hasDeactivationBurst( + deactivations: IDeactivatedOctolensKeywordMapping[], + activeMappings: number, +): boolean { + const perRun = new Map() + for (const row of deactivations.filter((r) => r.nowMappedToSegmentName === null)) { + const at = row.deactivatedAt.getTime() + perRun.set(at, (perRun.get(at) ?? 0) + 1) + } + return [...perRun.values()].some((count) => isDeactivationBurst(count, activeMappings)) +} + +function deactivationWindow( + checkedAt: Date, + cursor: string | null, +): { since: Date; until: Date; fromCursor: boolean } { + const until = new Date(checkedAt.getTime() - CURSOR_SAFETY_MARGIN_MINUTES * MINUTE_MS) + const cursorDate = cursor ? new Date(cursor) : null + if (cursorDate && !Number.isNaN(cursorDate.getTime())) { + return { since: cursorDate, until, fromCursor: true } + } + return { + since: new Date(until.getTime() - INITIAL_LOOKBACK_HOURS * HOUR_MS), + until, + fromCursor: false, + } +} + +function formatUtc(date: Date): string { + return `${date.toISOString().slice(0, 16).replace('T', ' ')} UTC` +} + +// Names come from Octolens and project data. Inside a code span Slack applies no `*`, `_` or +// `~` formatting; a backtick would end the span, so it becomes a look-alike. +export function formatName(name: string): string { + const chars = Array.from(name.replace(/\s+/g, ' ')) + const short = + chars.length > MAX_NAME_LENGTH + ? `${chars.slice(0, MAX_NAME_LENGTH - 1).join('')}…` + : chars.join('') + return `\`${escapeSlackText(short.replace(/`/g, 'ˋ'))}\`` +} + +function buildStaleAlert(health: IOctolensMappingDiscoveryHealth): IOctolensMappingHealthAlert { + return { + persona: SlackPersona.WARNING_PROPAGATOR, + title: 'Octolens mapping discovery looks stale', + sections: [ + { + title: '', + text: [ + `Keyword-to-project mapping discovery has not written any mappings in over ${STALE_AFTER_HOURS} hours.`, + `Last write: ${formatUtc(health.lastDiscoveredAt)} (${Math.floor(hoursSince(health.lastDiscoveredAt, health.checkedAt))}h ago)`, + `Active mappings: ${health.activeMappings} across ${health.integrations} projects`, + 'Existing mappings keep routing mentions, but keywords added, moved or removed in ' + + 'Octolens are not picked up, and the mention poll skips mentions of new keywords.', + 'Check the `connectors-octolens-discovery` schedule and the connectors_worker logs: a ' + + 'failed run, an incomplete Octolens snapshot or a run that finds no mappings writes ' + + 'nothing, and so does a missing `CROWD_OCTOLENS_API_TOKEN`.', + `Once discovery writes again, recover the skipped mentions with \`${BACKFILL_COMMAND} --from ${health.lastDiscoveredAt.toISOString()}\`, ` + + 'running it with `--dry-run` first. If it refuses because the date is not earlier ' + + 'than the current start date, the poll may already have read past some skipped mentions, ' + + `so re-fetch the keywords mapped since then with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`.`, + ].join('\n'), + }, + ], + } +} + +// Capped because @crowd/slack silently prunes oversized messages while still reporting +// success, which would move the cursor past deactivations nobody saw. +export function capLines(lines: string[], maxLines: number, maxChars: number): string { + const budget = maxChars - OVERFLOW_NOTE_RESERVE + const kept: string[] = [] + let length = 0 + for (const line of lines) { + if (kept.length === maxLines || length + line.length + 1 > budget) { + break + } + kept.push(line) + length += line.length + 1 + } + const omitted = lines.length - kept.length + return [...kept, ...(omitted > 0 ? [`… and ${omitted} more`] : [])].join('\n') +} + +function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { + return ( + `• ${formatName(row.keyword)} (${row.keywordId}) → ` + + `${formatName(row.segmentName)} (${formatName(row.segmentSlug)}), ` + + `deactivated ${formatUtc(row.deactivatedAt)}` + + (row.integrationDeleted ? ', project has no Octolens keywords left' : '') + + (row.nowMappedToSegmentName ? `, now mapped to ${formatName(row.nowMappedToSegmentName)}` : '') + ) +} + +// Stable, so rows of one run keep the query's project and keyword order. +function newestFirst( + deactivations: IDeactivatedOctolensKeywordMapping[], +): IDeactivatedOctolensKeywordMapping[] { + return [...deactivations].sort((a, b) => b.deactivatedAt.getTime() - a.deactivatedAt.getTime()) +} + +function buildDeactivationAlert( + deactivations: IDeactivatedOctolensKeywordMapping[], + activeMappings: number, + since: Date, + until: Date, +): IOctolensMappingHealthAlert { + const burst = hasDeactivationBurst(deactivations, activeMappings) + + return { + persona: burst ? SlackPersona.WARNING_PROPAGATOR : SlackPersona.INFO_NOTIFIER, + title: burst + ? 'Octolens keyword mappings deactivated in bulk' + : 'Octolens keyword mappings deactivated', + sections: [ + { + title: '', + text: [ + `Window: ${formatUtc(since)} → ${formatUtc(until)}`, + 'Discovery no longer saw these keywords mapped to the project, so their mentions stop ' + + 'reaching it. A keyword now mapped to another project goes there instead; any other ' + + 'keyword is skipped by the mention poll.', + 'Nothing is re-fetched automatically: once a skipped keyword is mapped again, re-fetch ' + + `its history with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`, running ` + + 'it with `--dry-run` first.', + ].join('\n'), + }, + { + title: `Deactivated mappings (${deactivations.length}, ${activeMappings} still active)`, + text: capLines( + newestFirst(deactivations).map(formatDeactivation), + MAX_LISTED_KEYWORDS, + MAX_SECTION_CHARS, + ), + }, + ], + } +} + +async function reportDeactivations( + deps: IOctolensMappingHealthDeps, + health: IOctolensMappingDiscoveryHealth, + result: IOctolensMappingHealthResult, +): Promise { + const { since, until, fromCursor } = deactivationWindow( + health.checkedAt, + await deps.getKey(DEACTIVATIONS_CURSOR_KEY), + ) + if (since >= until) { + return + } + // Pin the initial lookback so a retry after a failed send does not slide the window forward. + if (!fromCursor) { + await deps.setKey(DEACTIVATIONS_CURSOR_KEY, since.toISOString()) + } + + const deactivations = await deps.listDeactivatedBetween(since, until) + if (deactivations.length > 0) { + const alert = buildDeactivationAlert(deactivations, health.activeMappings, since, until) + if (!(await deps.sendAlert(alert))) { + result.failures++ + deps.log.warn( + { windowStart: since.toISOString(), deactivations: deactivations.length }, + 'octolens deactivation alert failed to send, the next run retries from the same window start', + ) + return + } + result.deactivationsReported = deactivations.length + } + + await deps.setKey(DEACTIVATIONS_CURSOR_KEY, until.toISOString()) +} + +async function reportStaleness( + deps: IOctolensMappingHealthDeps, + health: IOctolensMappingDiscoveryHealth, + result: IOctolensMappingHealthResult, +): Promise { + if (!isStale(health)) { + return + } + const episode = health.lastDiscoveredAt.toISOString() + if ((await deps.getKey(STALE_ALERTED_KEY)) === episode) { + return + } + + if (!(await deps.sendAlert(buildStaleAlert(health)))) { + result.failures++ + deps.log.warn({ lastDiscoveredAt: episode }, 'octolens stale alert failed to send') + return + } + await deps.setKey(STALE_ALERTED_KEY, episode, STALE_REMINDER_HOURS * 60 * 60) + result.staleAlerted = true +} + +async function runIsolated( + deps: IOctolensMappingHealthDeps, + result: IOctolensMappingHealthResult, + name: string, + check: () => Promise, +): Promise { + try { + await check() + } catch (err) { + result.failures++ + deps.log.error({ err }, `octolens mapping health: ${name} check failed`) + } +} + +// null means no active mapping, so discovery never completed a run (or the token was never set): +// a run that sees no mappings writes nothing, so it cannot empty a populated table. +export async function runOctolensMappingHealthCheck( + deps: IOctolensMappingHealthDeps, +): Promise { + const health = await deps.getHealth() + const result: IOctolensMappingHealthResult = { + activeMappings: health?.activeMappings ?? 0, + staleAlerted: false, + deactivationsReported: 0, + failures: 0, + } + if (!health) { + return result + } + + await runIsolated(deps, result, 'staleness', () => reportStaleness(deps, health, result)) + await runIsolated(deps, result, 'deactivations', () => reportDeactivations(deps, health, result)) + return result +} diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts new file mode 100644 index 0000000000..beaf80e22f --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -0,0 +1,52 @@ +import CronTime from 'cron-time-generator' + +import { IS_DEV_ENV, IS_PROD_ENV } from '@crowd/common' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import { + getOctolensMappingDiscoveryHealth, + listOctolensKeywordMappingsDeactivatedBetween, +} from '@crowd/data-access-layer/src/octolens/health' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { REDIS_CONFIG, getRedisClient } from '@crowd/redis' +import { SlackChannel, sendSlackNotificationAsync } from '@crowd/slack' + +import { IJobDefinition } from '../types' +import { runOctolensMappingHealthCheck } from './octolensMappingHealth' + +const job: IJobDefinition = { + name: 'octolens-mapping-health-check', + cronTime: IS_DEV_ENV ? CronTime.every(15).minutes() : CronTime.everyHourAt(30), + timeout: 5 * 60, + enabled: async () => IS_PROD_ENV, + process: async (ctx) => { + ctx.log.info('Running octolens-mapping-health-check job...') + + // Primary, not replica: the cursor margin only covers the reconcile transaction's duration + // (deactivatedAt is its start time), so replica lag on top could skip deactivations. + const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 3, 0) + const qx = pgpQx(dbConnection) + const redis = await getRedisClient(REDIS_CONFIG()) + + const result = await runOctolensMappingHealthCheck({ + log: ctx.log, + getHealth: () => getOctolensMappingDiscoveryHealth(qx), + listDeactivatedBetween: (since, until) => + listOctolensKeywordMappingsDeactivatedBetween(qx, since, until), + getKey: (key) => redis.get(key), + setKey: async (key, value, expireAfterSeconds) => { + await redis.set(key, value, expireAfterSeconds ? { EX: expireAfterSeconds } : undefined) + }, + sendAlert: (alert) => + sendSlackNotificationAsync( + SlackChannel.CDP_INTEGRATIONS_ALERTS, + alert.persona, + alert.title, + alert.sections, + ), + }) + + ctx.log.info(result, 'Octolens mapping health check done') + }, +} + +export default job diff --git a/services/libs/data-access-layer/src/octolens/health.test.ts b/services/libs/data-access-layer/src/octolens/health.test.ts new file mode 100644 index 0000000000..19b44f2520 --- /dev/null +++ b/services/libs/data-access-layer/src/octolens/health.test.ts @@ -0,0 +1,291 @@ +import { test as base, describe, expect } from 'vitest' + +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' +import type { QueryExecutor } from '@crowd/database' +import { withQx } from '@crowd/test-kit/db' +import { createSegments } from '@crowd/test-kit/factories' +import { PlatformType } from '@crowd/types' + +import { + getOctolensMappingDiscoveryHealth, + listOctolensKeywordMappingsDeactivatedBetween, +} from './health' +import { + findOrCreateOctolensAnchorIntegration, + findOrCreateOctolensSubprojectIntegration, + reconcileOctolensKeywordMappings, + upsertOctolensKeywordMapping, +} from './index' + +const test = withQx(base) + +async function createSubproject(qx: QueryExecutor, name: string): Promise { + const { projectGroups } = await createSegments(qx, [ + { + name: `${name}-group`, + slug: `${name}-group`, + projects: [ + { + name: `${name}-project`, + slug: `${name}-project`, + subprojects: [{ name, slug: `${name}-slug` }], + }, + ], + }, + ]) + return projectGroups[0].projects[0].subprojects[0].id +} + +async function createIntegration( + qx: QueryExecutor, + options: { segmentId: string | null; platform?: PlatformType; deleted?: boolean }, +): Promise { + const id = generateUUIDv1() + await qx.result( + ` + INSERT INTO public.integrations + (id, platform, status, "segmentId", "tenantId", "createdAt", "updatedAt", "deletedAt") + VALUES ($(id), $(platform), 'done', $(segmentId), $(tenantId), NOW(), NOW(), + CASE WHEN $(deleted) THEN NOW() END) + `, + { + id, + platform: options.platform ?? PlatformType.OCTOLENS, + segmentId: options.segmentId, + tenantId: DEFAULT_TENANT_ID, + deleted: options.deleted ?? false, + }, + ) + return id +} + +async function createActiveSyncUnit(qx: QueryExecutor, integrationId: string): Promise { + await qx.result( + ` + INSERT INTO integration.sync_units + ("integrationId", platform, "channelId", "channelName", "syncName", status) + VALUES ($(integrationId), $(platform), 'octolens', 'octolens', 'mentionPoll', 'active') + `, + { integrationId, platform: PlatformType.OCTOLENS }, + ) +} + +async function setTimestamps( + qx: QueryExecutor, + keywordId: number, + timestamps: { lastSeenAt: string; deactivatedAt?: string }, +): Promise { + await qx.result( + ` + UPDATE integration."octolensKeywordMappings" + SET "lastSeenAt" = $(lastSeenAt), "deactivatedAt" = $(deactivatedAt) + WHERE "keywordId" = $(keywordId) + `, + { + keywordId, + lastSeenAt: timestamps.lastSeenAt, + deactivatedAt: timestamps.deactivatedAt ?? null, + }, + ) +} + +async function mapKeyword( + qx: QueryExecutor, + integrationId: string, + segmentId: string, + deactivatedAt: string | null = null, +): Promise { + await upsertOctolensKeywordMapping(qx, { integrationId, segmentId, keywordId: 7, keyword: 'k7' }) + await qx.result( + ` + UPDATE integration."octolensKeywordMappings" + SET "deactivatedAt" = $(deactivatedAt) + WHERE "integrationId" = $(integrationId) AND "keywordId" = 7 + `, + { integrationId, deactivatedAt }, + ) +} + +const IN_WINDOW = '2026-01-01T10:30:00Z' +const WINDOW: [Date, Date] = [new Date('2026-01-01T10:00:00Z'), new Date('2026-01-01T11:00:00Z')] + +async function readDeactivatedAt(qx: QueryExecutor, keywordId: number): Promise { + const row: { deactivatedAt: string } = await qx.selectOne( + `SELECT "deactivatedAt" FROM integration."octolensKeywordMappings" WHERE "keywordId" = $(keywordId)`, + { keywordId }, + ) + return new Date(row.deactivatedAt) +} + +describe('getOctolensMappingDiscoveryHealth', () => { + test('reports nothing to judge without mappings, even with the anchor in place', async ({ + qx, + }) => { + const anchor = await findOrCreateOctolensAnchorIntegration(qx) + await createActiveSyncUnit(qx, anchor.id) + + expect(await getOctolensMappingDiscoveryHealth(qx)).toBeNull() + }) + + test('takes the newest lastSeenAt across active mappings of every live subproject integration', async ({ + qx, + }) => { + const segmentA = await createSubproject(qx, 'a') + const segmentB = await createSubproject(qx, 'b') + await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: segmentA }, + { keywordId: 2, keyword: 'k2', segmentId: segmentA }, + { keywordId: 3, keyword: 'k3', segmentId: segmentB }, + ]) + await setTimestamps(qx, 1, { lastSeenAt: '2026-01-01T09:00:00Z' }) + await setTimestamps(qx, 2, { + lastSeenAt: '2026-01-01T12:00:00Z', + deactivatedAt: '2026-01-01T12:00:00Z', + }) + await setTimestamps(qx, 3, { lastSeenAt: '2026-01-01T10:00:00Z' }) + + const health = await getOctolensMappingDiscoveryHealth(qx) + + expect(health).toMatchObject({ activeMappings: 2, integrations: 2 }) + expect(health?.lastDiscoveredAt.toISOString()).toBe('2026-01-01T10:00:00.000Z') + }) + + test('ignores soft-deleted, sync-unit-owning and non-octolens integrations', async ({ qx }) => { + const segmentId = await createSubproject(qx, 's') + const { id: live } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) + const deleted = await createIntegration(qx, { segmentId, deleted: true }) + const legacyOwner = await createIntegration(qx, { segmentId }) + await createActiveSyncUnit(qx, legacyOwner) + const github = await createIntegration(qx, { segmentId, platform: PlatformType.GITHUB }) + const integrationByKeywordId = { 1: live, 2: deleted, 3: legacyOwner, 4: github } + for (const [keywordId, integrationId] of Object.entries(integrationByKeywordId)) { + await upsertOctolensKeywordMapping(qx, { + integrationId, + segmentId, + keywordId: Number(keywordId), + keyword: `k${keywordId}`, + }) + } + await setTimestamps(qx, 1, { lastSeenAt: '2026-01-01T09:00:00Z' }) + + const health = await getOctolensMappingDiscoveryHealth(qx) + + expect(health).toMatchObject({ activeMappings: 1, integrations: 1 }) + expect(health?.lastDiscoveredAt.toISOString()).toBe('2026-01-01T09:00:00.000Z') + }) +}) + +describe('listOctolensKeywordMappingsDeactivatedBetween', () => { + test('excludes since, includes until, and joins the project', async ({ qx }) => { + const segmentId = await createSubproject(qx, 'project') + await reconcileOctolensKeywordMappings( + qx, + [1, 2, 3, 4, 5].map((keywordId) => ({ keywordId, keyword: `k${keywordId}`, segmentId })), + ) + await setTimestamps(qx, 1, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T10:00:00Z', + }) + await setTimestamps(qx, 2, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T10:30:00.000123Z', + }) + await setTimestamps(qx, 3, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T11:00:00Z', + }) + await setTimestamps(qx, 4, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T11:00:00.000001Z', + }) + + const rows = await listOctolensKeywordMappingsDeactivatedBetween( + qx, + new Date('2026-01-01T10:00:00Z'), + new Date('2026-01-01T11:00:00Z'), + ) + + expect(rows.map((r) => r.keyword)).toEqual(['k2', 'k3']) + expect(rows[0]).toMatchObject({ + keywordId: 2, + segmentId, + segmentName: 'project', + segmentSlug: 'project-slug', + integrationDeleted: false, + }) + expect(rows[0].deactivatedAt.toISOString()).toBe('2026-01-01T10:30:00.000Z') + }) + + test('keeps mappings of a subproject discovery soft-deleted in the same run and names moves', async ({ + qx, + }) => { + const kept = await createSubproject(qx, 'kept') + const emptied = await createSubproject(qx, 'emptied') + await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: kept }, + { keywordId: 2, keyword: 'k2', segmentId: emptied }, + { keywordId: 3, keyword: 'k3', segmentId: emptied }, + ]) + const outcome = await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: kept }, + { keywordId: 2, keyword: 'k2', segmentId: kept }, + ]) + expect(outcome.softDeleted).toHaveLength(1) + const deactivatedAt = await readDeactivatedAt(qx, 3) + + const rows = await listOctolensKeywordMappingsDeactivatedBetween( + qx, + new Date(deactivatedAt.getTime() - 1000), + new Date(deactivatedAt.getTime() + 1000), + ) + + expect( + rows.map((r) => [r.keyword, r.segmentName, r.integrationDeleted, r.nowMappedToSegmentName]), + ).toEqual([ + ['k2', 'emptied', true, 'kept'], + ['k3', 'emptied', true, null], + ]) + expect(rows[0].deactivatedAt.getTime()).toBe(rows[1].deactivatedAt.getTime()) + }) + + test('excludes the anchor and other platforms', async ({ qx }) => { + const segmentId = await createSubproject(qx, 'project') + const { id: subproject } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) + const anchor = await findOrCreateOctolensAnchorIntegration(qx) + const github = await createIntegration(qx, { segmentId, platform: PlatformType.GITHUB }) + for (const integrationId of [subproject, anchor.id, github]) { + await mapKeyword(qx, integrationId, segmentId, IN_WINDOW) + } + + const rows = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + + expect(rows.map((r) => r.integrationId)).toEqual([subproject]) + }) + + test('names a move only to a live subproject integration', async ({ qx }) => { + const segmentId = await createSubproject(qx, 'project') + const other = await createSubproject(qx, 'other') + const { id: subproject } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) + await mapKeyword(qx, subproject, segmentId, IN_WINDOW) + const legacyOwner = await createIntegration(qx, { segmentId: other }) + await createActiveSyncUnit(qx, legacyOwner) + const notSubprojects = [ + await createIntegration(qx, { segmentId: other, deleted: true }), + await createIntegration(qx, { segmentId: other, platform: PlatformType.GITHUB }), + legacyOwner, + ] + for (const integrationId of notSubprojects) { + await mapKeyword(qx, integrationId, other) + } + + const [ignored] = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + expect(ignored.nowMappedToSegmentName).toBeNull() + + const control = await createSubproject(qx, 'control') + const { id: controlIntegration } = await findOrCreateOctolensSubprojectIntegration(qx, control) + await mapKeyword(qx, controlIntegration, control) + + const [moved] = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + expect(moved.nowMappedToSegmentName).toBe('control') + }) +}) diff --git a/services/libs/data-access-layer/src/octolens/health.ts b/services/libs/data-access-layer/src/octolens/health.ts new file mode 100644 index 0000000000..4edf178948 --- /dev/null +++ b/services/libs/data-access-layer/src/octolens/health.ts @@ -0,0 +1,117 @@ +import { PlatformType } from '@crowd/types' + +import { QueryExecutor } from '../queryExecutor' +import { OWNS_NO_ACTIVE_SYNC_UNITS } from './index' + +const OCTOLENS_SUBPROJECT_INTEGRATION = ` + i.platform = $(platform) + AND i."segmentId" IS NOT NULL + AND ${OWNS_NO_ACTIVE_SYNC_UNITS} +` + +export interface IOctolensMappingDiscoveryHealth { + activeMappings: number + integrations: number + lastDiscoveredAt: Date + checkedAt: Date +} + +// Discovery stamps every mapping it keeps in one transaction, so the newest lastSeenAt is its last +// complete run. +export async function getOctolensMappingDiscoveryHealth( + qx: QueryExecutor, +): Promise { + const row: { + activeMappings: number + integrations: number + lastDiscoveredAt: string | null + checkedAt: string + } = await qx.selectOne( + ` + SELECT + count(*)::int AS "activeMappings", + count(DISTINCT m."integrationId")::int AS "integrations", + max(m."lastSeenAt") AS "lastDiscoveredAt", + now() AS "checkedAt" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" + WHERE m."deactivatedAt" IS NULL + AND i."deletedAt" IS NULL + AND ${OCTOLENS_SUBPROJECT_INTEGRATION} + `, + { platform: PlatformType.OCTOLENS }, + ) + if (row.lastDiscoveredAt === null) { + return null + } + return { + activeMappings: row.activeMappings, + integrations: row.integrations, + lastDiscoveredAt: new Date(row.lastDiscoveredAt), + checkedAt: new Date(row.checkedAt), + } +} + +export interface IDeactivatedOctolensKeywordMapping { + integrationId: string + keywordId: number + keyword: string + segmentId: string + segmentName: string + segmentSlug: string + deactivatedAt: Date + integrationDeleted: boolean + nowMappedToSegmentName: string | null +} + +// Soft-deleted integrations stay in: discovery soft-deletes a subproject in the same run that +// deactivates its last mapping, and that is the deactivation most worth reporting. +export async function listOctolensKeywordMappingsDeactivatedBetween( + qx: QueryExecutor, + since: Date, + until: Date, +): Promise { + const rows: (Omit & { + deactivatedAt: string + })[] = await qx.select( + ` + SELECT + m."integrationId", + m."keywordId", + m.keyword, + m."segmentId", + s.name AS "segmentName", + s.slug AS "segmentSlug", + m."deactivatedAt", + i."deletedAt" IS NOT NULL AS "integrationDeleted", + moved.name AS "nowMappedToSegmentName" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" + JOIN public.segments s ON s.id = m."segmentId" + LEFT JOIN LATERAL ( + SELECT ms.name + FROM integration."octolensKeywordMappings" a + JOIN public.integrations ai ON ai.id = a."integrationId" + JOIN public.segments ms ON ms.id = a."segmentId" + WHERE a."keywordId" = m."keywordId" + AND a."deactivatedAt" IS NULL + AND ai."deletedAt" IS NULL + AND ai.platform = $(platform) + AND ai."segmentId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM integration.sync_units asu + WHERE asu."integrationId" = ai.id AND asu.status = 'active' + ) + ORDER BY ms.name + LIMIT 1 + ) moved ON true + WHERE m."deactivatedAt" > $(since) + AND m."deactivatedAt" <= $(until) + AND ${OCTOLENS_SUBPROJECT_INTEGRATION} + ORDER BY m."deactivatedAt", s.name, m.keyword, m."keywordId" + `, + { platform: PlatformType.OCTOLENS, since, until }, + ) + return rows.map((row) => ({ ...row, deactivatedAt: new Date(row.deactivatedAt) })) +} diff --git a/services/libs/data-access-layer/src/octolens/index.ts b/services/libs/data-access-layer/src/octolens/index.ts index 0ca1652dcc..8062f39d6c 100644 --- a/services/libs/data-access-layer/src/octolens/index.ts +++ b/services/libs/data-access-layer/src/octolens/index.ts @@ -56,7 +56,7 @@ export interface IOctolensKeywordMappingRef { } // An integration owning active sync units (the anchor, or a legacy one) is never a subproject integration. -const OWNS_NO_ACTIVE_SYNC_UNITS = `NOT EXISTS ( +export const OWNS_NO_ACTIVE_SYNC_UNITS = `NOT EXISTS ( SELECT 1 FROM integration.sync_units su WHERE su."integrationId" = i.id AND su.status = 'active' )`