From 6e02013c354fe0ffaa394981435239025f4b932b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Thu, 15 Oct 2026 15:02:52 +0200 Subject: [PATCH 01/10] feat: alert when octolens mapping discovery looks unhealthy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an hourly cron_service job that posts to the integrations alerts channel when keyword-to-project mapping discovery has not written for over 3 hours, and lists every mapping it deactivated since the last report (warning persona on a burst). Redis keeps a reported-until cursor and a per-integration stale marker so alerts do not repeat. Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 227 +++++++++++++++++ .../src/jobs/octolensMappingHealth.ts | 231 ++++++++++++++++++ .../jobs/octolensMappingHealthCheck.job.ts | 55 +++++ .../src/octolens/index.test.ts | 138 +++++++++++ .../data-access-layer/src/octolens/index.ts | 73 ++++++ 5 files changed, 724 insertions(+) create mode 100644 services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts create mode 100644 services/apps/cron_service/src/jobs/octolensMappingHealth.ts create mode 100644 services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts new file mode 100644 index 0000000000..8e7da94e2d --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it, vi } from 'vitest' + +import type { + IDeactivatedOctolensKeywordMapping, + IOctolensMappingDiscoveryState, +} from '@crowd/data-access-layer' +import { SlackPersona } from '@crowd/slack' + +import { + DEACTIVATIONS_CURSOR_KEY, + IOctolensMappingHealthAlert, + IOctolensMappingHealthDeps, + isDeactivationBurst, + runOctolensMappingHealthCheck, + staleAlertedKey, +} from './octolensMappingHealth' + +const CHECKED_AT = new Date('2026-10-15T12:30:00.000Z') +const UNTIL = '2026-10-15T12:25:00.000Z' + +function state(overrides: Partial = {}) { + return { + integrationId: 'int-1', + activeMappings: 20, + lastDiscoveredAt: new Date('2026-10-15T12:05:00.000Z'), + checkedAt: CHECKED_AT, + ...overrides, + } +} + +function deactivation( + keywordId: number, + overrides: Partial = {}, +): IDeactivatedOctolensKeywordMapping { + return { + integrationId: 'int-1', + keywordId, + keyword: `keyword-${keywordId}`, + segmentId: 'seg-1', + segmentName: 'Project One', + segmentSlug: 'project-one', + deactivatedAt: new Date('2026-10-15T12:05:00.000Z'), + ...overrides, + } +} + +function setup(options: { + states: IOctolensMappingDiscoveryState[] + deactivations?: IDeactivatedOctolensKeywordMapping[] + store?: Record + sendSucceeds?: boolean +}) { + const store = new Map(Object.entries(options.store ?? {})) + const ttls = new Map() + const sent: IOctolensMappingHealthAlert[] = [] + const listDeactivatedBetween = vi.fn(async () => options.deactivations ?? []) + const deps: IOctolensMappingHealthDeps = { + listStates: async () => options.states, + listDeactivatedBetween, + getKey: async (key) => store.get(key) ?? null, + setKey: async (key, value, expireAfterSeconds) => { + store.set(key, value) + ttls.set(key, expireAfterSeconds) + }, + sendAlert: async (alert) => { + sent.push(alert) + return options.sendSucceeds ?? true + }, + } + return { deps, store, ttls, sent, listDeactivatedBetween } +} + +function alertText(alert: IOctolensMappingHealthAlert): string { + return alert.sections.map((s) => `${s.title}\n${s.text}`).join('\n') +} + +describe('runOctolensMappingHealthCheck', () => { + it('does nothing when no integration has mappings yet', async () => { + const { deps, store, sent, listDeactivatedBetween } = setup({ states: [] }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + expect(store.size).toBe(0) + expect(listDeactivatedBetween).not.toHaveBeenCalled() + }) + + it('sends nothing for a healthy integration and moves the cursor', async () => { + const { deps, store, sent } = setup({ states: [state()] }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) + }) + + it('looks back 24h on the first run and continues from the cursor afterwards', async () => { + const first = setup({ states: [state()] }) + await runOctolensMappingHealthCheck(first.deps) + expect(first.listDeactivatedBetween).toHaveBeenCalledWith( + new Date('2026-10-14T12:25:00.000Z'), + new Date(UNTIL), + ) + + const next = setup({ + states: [state()], + store: { [DEACTIVATIONS_CURSOR_KEY]: '2026-10-15T11:25:00.000Z' }, + }) + await runOctolensMappingHealthCheck(next.deps) + expect(next.listDeactivatedBetween).toHaveBeenCalledWith( + new Date('2026-10-15T11:25:00.000Z'), + new Date(UNTIL), + ) + }) + + it('alerts on a stale integration once, then again only after the reminder expires', async () => { + const stale = state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }) + const { deps, store, ttls, sent } = setup({ states: [stale] }) + + await runOctolensMappingHealthCheck(deps) + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + expect(alertText(sent[0])).toContain('2026-10-15 08:00 UTC (4h ago)') + expect(alertText(sent[0])).toContain('Active mappings: 20') + expect(ttls.get(staleAlertedKey('int-1'))).toBe(24 * 60 * 60) + + store.delete(staleAlertedKey('int-1')) + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(2) + }) + + it('alerts again when a new stale episode starts', async () => { + const { deps, sent } = setup({ + states: [state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') })], + store: { [staleAlertedKey('int-1')]: '2026-10-14T08:00:00.000Z' }, + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + }) + + it('does not treat discovery within the stale threshold as stale', async () => { + const { deps, sent } = setup({ + states: [state({ lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z') })], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + }) + + it('keeps alerting a stale integration while the Slack send fails', async () => { + const { deps, store, sent } = setup({ + states: [state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') })], + sendSucceeds: false, + }) + + await runOctolensMappingHealthCheck(deps) + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(2) + expect(store.has(staleAlertedKey('int-1'))).toBe(false) + }) + + it('reports a burst of deactivations as a warning listing every keyword', async () => { + const { deps, store, sent } = setup({ + states: [state()], + deactivations: [deactivation(1), deactivation(2), deactivation(3)], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + const text = alertText(sent[0]) + expect(text).toContain('3 of 23 mappings deactivated') + for (const keywordId of [1, 2, 3]) { + expect(text).toContain( + `keyword-${keywordId} (${keywordId}) → Project One (project-one), deactivated 2026-10-15 12:05 UTC`, + ) + } + expect(text).toContain('re-fetches its whole history automatically') + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) + }) + + it('reports a single deactivation as info', async () => { + const { deps, sent } = setup({ states: [state()], deactivations: [deactivation(1)] }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) + expect(alertText(sent[0])).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') + }) + + it('keeps the cursor when the deactivation alert fails so the next run retries', async () => { + const cursor = '2026-10-15T11:25:00.000Z' + const { deps, store, sent } = setup({ + states: [state()], + deactivations: [deactivation(1)], + store: { [DEACTIVATIONS_CURSOR_KEY]: cursor }, + sendSucceeds: false, + }) + + const result = await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(result.sendFailures).toBe(1) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(cursor) + }) +}) + +describe('isDeactivationBurst', () => { + it('flags three or more deactivations regardless of size', () => { + expect(isDeactivationBurst(3, 1000)).toBe(true) + }) + + it('flags a quarter or more of the mappings', () => { + expect(isDeactivationBurst(1, 3)).toBe(true) + expect(isDeactivationBurst(2, 6)).toBe(true) + expect(isDeactivationBurst(2, 7)).toBe(false) + }) +}) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts new file mode 100644 index 0000000000..8363ba93a6 --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -0,0 +1,231 @@ +import type { + IDeactivatedOctolensKeywordMapping, + IOctolensMappingDiscoveryState, +} from '@crowd/data-access-layer' +import { SlackMessageSection, SlackPersona, escapeSlackText } from '@crowd/slack' + +export const STALE_AFTER_HOURS = 3 +export const STALE_REMINDER_HOURS = 24 +export const BURST_MIN_DEACTIVATIONS = 3 +export const BURST_MIN_SHARE = 0.25 +export const CURSOR_SAFETY_MARGIN_MINUTES = 5 +export const INITIAL_LOOKBACK_HOURS = 24 + +export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' + +export function staleAlertedKey(integrationId: string): string { + return `octolens-mapping-health:stale-alerted:${integrationId}` +} + +const HOUR_MS = 60 * 60 * 1000 +const MINUTE_MS = 60 * 1000 + +export interface IOctolensMappingHealthAlert { + persona: SlackPersona + title: string + sections: SlackMessageSection[] +} + +export interface IOctolensMappingHealthDeps { + listStates: () => Promise + listDeactivatedBetween: ( + since: Date, + until: Date, + ) => Promise + getKey: (key: string) => Promise + setKey: (key: string, value: string, expireAfterSeconds?: number) => Promise + sendAlert: (alert: IOctolensMappingHealthAlert) => Promise +} + +export interface IOctolensMappingHealthResult { + integrations: number + staleAlerted: number + deactivationsReported: number + sendFailures: number +} + +export function hoursSince(from: Date, now: Date): number { + return (now.getTime() - from.getTime()) / HOUR_MS +} + +export function isStale(state: IOctolensMappingDiscoveryState): boolean { + return hoursSince(state.lastDiscoveredAt, state.checkedAt) > STALE_AFTER_HOURS +} + +export function isDeactivationBurst(deactivated: number, activeMappings: number): boolean { + if (deactivated >= BURST_MIN_DEACTIVATIONS) { + return true + } + const total = activeMappings + deactivated + return total > 0 && deactivated / total >= BURST_MIN_SHARE +} + +export function deactivationWindow( + checkedAt: Date, + cursor: string | null, +): { since: Date; until: Date } { + const until = new Date(checkedAt.getTime() - CURSOR_SAFETY_MARGIN_MINUTES * MINUTE_MS) + const cursorDate = cursor ? new Date(cursor) : null + const since = + cursorDate && !Number.isNaN(cursorDate.getTime()) + ? cursorDate + : new Date(until.getTime() - INITIAL_LOOKBACK_HOURS * HOUR_MS) + return { since, until } +} + +function formatUtc(date: Date): string { + return `${date.toISOString().slice(0, 16).replace('T', ' ')} UTC` +} + +export function buildStaleAlert( + states: IOctolensMappingDiscoveryState[], +): IOctolensMappingHealthAlert { + return { + persona: SlackPersona.WARNING_PROPAGATOR, + title: 'Octolens mapping discovery looks stale', + sections: [ + { + title: '', + text: + `Keyword-to-project mapping discovery has not completed a run in over ${STALE_AFTER_HOURS} hours. ` + + 'Existing mappings stay as they are, but keywords added, remapped or ' + + 'removed in Octolens are not picked up until discovery succeeds again.', + }, + ...states.map((state) => ({ + title: `Integration ${state.integrationId}`, + text: [ + `Last successful discovery: ${formatUtc(state.lastDiscoveredAt)} (${Math.floor(hoursSince(state.lastDiscoveredAt, state.checkedAt))}h ago)`, + `Active mappings: ${state.activeMappings}`, + ].join('\n'), + })), + ], + } +} + +export function buildDeactivationAlert( + deactivations: IDeactivatedOctolensKeywordMapping[], + states: IOctolensMappingDiscoveryState[], + since: Date, + until: Date, +): IOctolensMappingHealthAlert { + const activeByIntegration = new Map(states.map((s) => [s.integrationId, s.activeMappings])) + const byIntegration = new Map() + for (const row of deactivations) { + const bucket = byIntegration.get(row.integrationId) ?? [] + bucket.push(row) + byIntegration.set(row.integrationId, bucket) + } + + let burst = false + const integrationSections: SlackMessageSection[] = [] + for (const [integrationId, rows] of byIntegration) { + const active = activeByIntegration.get(integrationId) ?? 0 + const integrationBurst = isDeactivationBurst(rows.length, active) + burst = burst || integrationBurst + integrationSections.push({ + title: `${integrationBurst ? '⚠️ ' : ''}Integration ${integrationId}: ${rows.length} of ${active + rows.length} mappings deactivated`, + text: rows + .map( + (row) => + `• ${escapeSlackText(row.keyword)} (${row.keywordId}) → ` + + `${escapeSlackText(row.segmentName)} (${escapeSlackText(row.segmentSlug)}), ` + + `deactivated ${formatUtc(row.deactivatedAt)}`, + ) + .join('\n'), + }) + } + + return { + persona: burst ? SlackPersona.WARNING_PROPAGATOR : SlackPersona.INFO_NOTIFIER, + title: burst + ? 'Octolens keyword mappings deactivated in bulk' + : 'Octolens keyword mappings deactivated', + sections: [ + { + title: '', + text: [ + `Window: ${formatUtc(since)} → ${formatUtc(until)}`, + 'Discovery no longer saw these keywords mapped to a project, so their mentions stop ' + + 'reaching it. If a keyword gets mapped again, the mention poll re-fetches its whole ' + + 'history automatically.', + ].join('\n'), + }, + ...integrationSections, + ], + } +} + +async function reportDeactivations( + deps: IOctolensMappingHealthDeps, + states: IOctolensMappingDiscoveryState[], + result: IOctolensMappingHealthResult, +): Promise { + const { since, until } = deactivationWindow( + states[0].checkedAt, + await deps.getKey(DEACTIVATIONS_CURSOR_KEY), + ) + if (since >= until) { + return + } + + const deactivations = await deps.listDeactivatedBetween(since, until) + if (deactivations.length > 0) { + const sent = await deps.sendAlert(buildDeactivationAlert(deactivations, states, since, until)) + if (!sent) { + result.sendFailures++ + return + } + result.deactivationsReported = deactivations.length + } + + await deps.setKey(DEACTIVATIONS_CURSOR_KEY, until.toISOString()) +} + +async function reportStaleIntegrations( + deps: IOctolensMappingHealthDeps, + states: IOctolensMappingDiscoveryState[], + result: IOctolensMappingHealthResult, +): Promise { + const toAlert: IOctolensMappingDiscoveryState[] = [] + for (const state of states.filter(isStale)) { + const alertedFor = await deps.getKey(staleAlertedKey(state.integrationId)) + if (alertedFor !== state.lastDiscoveredAt.toISOString()) { + toAlert.push(state) + } + } + if (toAlert.length === 0) { + return + } + + if (!(await deps.sendAlert(buildStaleAlert(toAlert)))) { + result.sendFailures++ + return + } + for (const state of toAlert) { + await deps.setKey( + staleAlertedKey(state.integrationId), + state.lastDiscoveredAt.toISOString(), + STALE_REMINDER_HOURS * 60 * 60, + ) + } + result.staleAlerted = toAlert.length +} + +export async function runOctolensMappingHealthCheck( + deps: IOctolensMappingHealthDeps, +): Promise { + const states = await deps.listStates() + const result: IOctolensMappingHealthResult = { + integrations: states.length, + staleAlerted: 0, + deactivationsReported: 0, + sendFailures: 0, + } + if (states.length === 0) { + return result + } + + await reportDeactivations(deps, states, result) + await reportStaleIntegrations(deps, states, result) + return result +} diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts new file mode 100644 index 0000000000..6506ed89e2 --- /dev/null +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -0,0 +1,55 @@ +import CronTime from 'cron-time-generator' + +import { IS_DEV_ENV, IS_PROD_ENV } from '@crowd/common' +import { + listOctolensKeywordMappingsDeactivatedBetween, + listOctolensMappingDiscoveryStates, +} from '@crowd/data-access-layer' +import { READ_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' +import { REDIS_CONFIG, getRedisClient } from '@crowd/redis' +import { SlackChannel, sendSlackNotificationAsync } from '@crowd/slack' + +import { IJobDefinition } from '../types' +import { runOctolensMappingHealthCheck } from './octolensMappingHealth' + +const job: IJobDefinition = { + name: 'octolens-mapping-health-check', + cronTime: IS_DEV_ENV ? CronTime.every(15).minutes() : CronTime.everyHourAt(30), + timeout: 5 * 60, + enabled: async () => IS_PROD_ENV, + process: async (ctx) => { + ctx.log.info('Running octolens-mapping-health-check job...') + + const dbConnection = await getDbConnection(READ_DB_CONFIG(), 3, 0) + const qx = pgpQx(dbConnection) + const redis = await getRedisClient(REDIS_CONFIG()) + + const result = await runOctolensMappingHealthCheck({ + listStates: () => listOctolensMappingDiscoveryStates(qx), + listDeactivatedBetween: (since, until) => + listOctolensKeywordMappingsDeactivatedBetween(qx, since, until), + getKey: (key) => redis.get(key), + setKey: async (key, value, expireAfterSeconds) => { + await redis.set(key, value, expireAfterSeconds ? { EX: expireAfterSeconds } : undefined) + }, + sendAlert: (alert) => + sendSlackNotificationAsync( + SlackChannel.CDP_INTEGRATIONS_ALERTS, + alert.persona, + alert.title, + alert.sections, + ), + }) + + if (result.sendFailures > 0) { + ctx.log.warn('octolens mapping health alert failed to send, will retry next run') + } + + ctx.log.info( + `Octolens mapping health check done: integrations=${result.integrations}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, sendFailures=${result.sendFailures}`, + ) + }, +} + +export default job diff --git a/services/libs/data-access-layer/src/octolens/index.test.ts b/services/libs/data-access-layer/src/octolens/index.test.ts index c67e6ab7c1..4da57d0e44 100644 --- a/services/libs/data-access-layer/src/octolens/index.test.ts +++ b/services/libs/data-access-layer/src/octolens/index.test.ts @@ -8,6 +8,8 @@ import { deactivateOctolensKeywordMappingsNotSeen, findOctolensKeywordMapping, listOctolensKeywordMappings, + listOctolensKeywordMappingsDeactivatedBetween, + listOctolensMappingDiscoveryStates, upsertOctolensKeywordMapping, } from './index' @@ -263,3 +265,139 @@ describe('octolens keyword mappings', () => { expect(await findOctolensKeywordMapping(qx, integrationId, 1)).not.toBeNull() }) }) + +async function setMappingTimestamps( + qx: QueryExecutor, + integrationId: string, + keywordId: number, + timestamps: { lastSeenAt: string; deactivatedAt?: string }, +): Promise { + await qx.result( + ` + UPDATE integration."octolensKeywordMappings" + SET "lastSeenAt" = $(lastSeenAt), "deactivatedAt" = $(deactivatedAt) + WHERE "integrationId" = $(integrationId) AND "keywordId" = $(keywordId) + `, + { + integrationId, + keywordId, + lastSeenAt: timestamps.lastSeenAt, + deactivatedAt: timestamps.deactivatedAt ?? null, + }, + ) +} + +describe('octolens mapping discovery health', () => { + test('listOctolensMappingDiscoveryStates returns nothing when there are no mappings', async ({ + qx, + }) => { + await createIntegration(qx) + + expect(await listOctolensMappingDiscoveryStates(qx)).toEqual([]) + }) + + test('listOctolensMappingDiscoveryStates counts active mappings and takes lastDiscoveredAt over all rows', async ({ + qx, + }) => { + const integrationId = await createIntegration(qx) + const segmentId = await createSegment(qx) + for (const keywordId of [1, 2, 3]) { + await upsertOctolensKeywordMapping(qx, { + integrationId, + segmentId, + keywordId, + keyword: `k${keywordId}`, + }) + } + await setMappingTimestamps(qx, integrationId, 1, { lastSeenAt: '2026-01-01T10:00:00Z' }) + await setMappingTimestamps(qx, integrationId, 2, { lastSeenAt: '2026-01-01T09:00:00Z' }) + await setMappingTimestamps(qx, integrationId, 3, { + lastSeenAt: '2026-01-01T11:00:00Z', + deactivatedAt: '2026-01-01T12:00:00Z', + }) + + const states = await listOctolensMappingDiscoveryStates(qx) + + expect(states).toHaveLength(1) + expect(states[0]).toMatchObject({ integrationId, activeMappings: 2 }) + expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T11:00:00.000Z') + expect(states[0].checkedAt).toBeInstanceOf(Date) + }) + + test('listOctolensMappingDiscoveryStates skips deleted integrations', async ({ qx }) => { + const integrationId = await createIntegration(qx) + const deletedIntegrationId = await createIntegration(qx) + const segmentId = await createSegment(qx) + for (const id of [integrationId, deletedIntegrationId]) { + await upsertOctolensKeywordMapping(qx, { + integrationId: id, + segmentId, + keywordId: 1, + keyword: 'k', + }) + } + await qx.result(`UPDATE public.integrations SET "deletedAt" = NOW() WHERE id = $(id)`, { + id: deletedIntegrationId, + }) + + const states = await listOctolensMappingDiscoveryStates(qx) + + expect(states.map((s) => s.integrationId)).toEqual([integrationId]) + }) + + test('listOctolensKeywordMappingsDeactivatedBetween excludes since, includes until, joins the segment', async ({ + qx, + }) => { + const integrationId = await createIntegration(qx) + const deletedIntegrationId = await createIntegration(qx) + const segmentId = await createSegment(qx) + const deactivatedAtByKeywordId: Record = { + 1: '2026-01-01T10:00:00Z', + 2: '2026-01-01T10:30:00Z', + 3: '2026-01-01T11:00:00Z', + 4: '2026-01-01T11:00:01Z', + 5: undefined, + } + for (const [keywordId, deactivatedAt] of Object.entries(deactivatedAtByKeywordId)) { + await upsertOctolensKeywordMapping(qx, { + integrationId, + segmentId, + keywordId: Number(keywordId), + keyword: `k${keywordId}`, + }) + await setMappingTimestamps(qx, integrationId, Number(keywordId), { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt, + }) + } + await upsertOctolensKeywordMapping(qx, { + integrationId: deletedIntegrationId, + segmentId, + keywordId: 1, + keyword: 'deleted', + }) + await setMappingTimestamps(qx, deletedIntegrationId, 1, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T10:30:00Z', + }) + await qx.result(`UPDATE public.integrations SET "deletedAt" = NOW() WHERE id = $(id)`, { + id: deletedIntegrationId, + }) + + const rows = await listOctolensKeywordMappingsDeactivatedBetween( + qx, + new Date('2026-01-01T10:00:00Z'), + new Date('2026-01-01T11:00:00Z'), + ) + + expect(rows.map((r) => r.keyword)).toEqual(['k2', 'k3']) + expect(rows[0]).toMatchObject({ + integrationId, + keywordId: 2, + segmentId, + segmentName: `segment-${segmentId}`, + segmentSlug: `segment-${segmentId}`, + }) + expect(rows[0].deactivatedAt.toISOString()).toBe('2026-01-01T10:30:00.000Z') + }) +}) diff --git a/services/libs/data-access-layer/src/octolens/index.ts b/services/libs/data-access-layer/src/octolens/index.ts index ddc271c525..8a1512db39 100644 --- a/services/libs/data-access-layer/src/octolens/index.ts +++ b/services/libs/data-access-layer/src/octolens/index.ts @@ -95,3 +95,76 @@ export async function deactivateOctolensKeywordMappingsNotSeen( { integrationId, seenKeywordIds }, ) } + +// The pg type parsers hand timestamptz back as strings. +type Timestamped = Omit & Record + +export interface IOctolensMappingDiscoveryState { + integrationId: string + activeMappings: number + lastDiscoveredAt: Date + checkedAt: Date +} + +// A writing discovery run stamps lastSeenAt on every mapping it sees, so the max over active +// and deactivated rows is when discovery last wrote anything. +export async function listOctolensMappingDiscoveryStates( + qx: QueryExecutor, +): Promise { + const rows: Timestamped[] = + await qx.select( + ` + SELECT + m."integrationId", + count(*) FILTER (WHERE m."deactivatedAt" IS NULL)::int AS "activeMappings", + max(m."lastSeenAt") AS "lastDiscoveredAt", + now() AS "checkedAt" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" AND i."deletedAt" IS NULL + GROUP BY m."integrationId" + ORDER BY m."integrationId" + `, + ) + return rows.map((row) => ({ + ...row, + lastDiscoveredAt: new Date(row.lastDiscoveredAt), + checkedAt: new Date(row.checkedAt), + })) +} + +export interface IDeactivatedOctolensKeywordMapping { + integrationId: string + keywordId: number + keyword: string + segmentId: string + segmentName: string + segmentSlug: string + deactivatedAt: Date +} + +export async function listOctolensKeywordMappingsDeactivatedBetween( + qx: QueryExecutor, + since: Date, + until: Date, +): Promise { + const rows: Timestamped[] = await qx.select( + ` + SELECT + m."integrationId", + m."keywordId", + m.keyword, + m."segmentId", + s.name AS "segmentName", + s.slug AS "segmentSlug", + m."deactivatedAt" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" AND i."deletedAt" IS NULL + JOIN public.segments s ON s.id = m."segmentId" + WHERE m."deactivatedAt" > $(since) + AND m."deactivatedAt" <= $(until) + ORDER BY m."integrationId", m."deactivatedAt", m.keyword + `, + { since, until }, + ) + return rows.map((row) => ({ ...row, deactivatedAt: new Date(row.deactivatedAt) })) +} From d6f4991d85f617803fffa2d705f2c8fef95c413a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Thu, 15 Oct 2026 15:10:12 +0200 Subject: [PATCH 02/10] fix: cap octolens deactivation alert rows and read from the primary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 30 +++++++++++++++++++ .../src/jobs/octolensMappingHealth.ts | 30 ++++++++++++++----- .../jobs/octolensMappingHealthCheck.job.ts | 5 ++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 8e7da94e2d..09fb9cfdda 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -124,6 +124,7 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) expect(alertText(sent[0])).toContain('2026-10-15 08:00 UTC (4h ago)') expect(alertText(sent[0])).toContain('Active mappings: 20') + expect(alertText(sent[0])).not.toContain('No active mappings') expect(ttls.get(staleAlertedKey('int-1'))).toBe(24 * 60 * 60) store.delete(staleAlertedKey('int-1')) @@ -187,6 +188,35 @@ describe('runOctolensMappingHealthCheck', () => { expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) + it('caps listed deactivations per integration but keeps the full count', async () => { + const { deps, sent } = setup({ + states: [state()], + deactivations: Array.from({ length: 30 }, (_, i) => deactivation(i + 1)), + }) + + await runOctolensMappingHealthCheck(deps) + + const text = alertText(sent[0]) + expect(text).toContain('30 of 50 mappings deactivated') + expect(text).toContain('keyword-25 (25)') + expect(text).not.toContain('keyword-26 (26)') + expect(text).toContain('… and 5 more') + }) + + it('flags a stale integration that has no active mappings left', async () => { + const { deps, sent } = setup({ + states: [ + state({ activeMappings: 0, lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), + ], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(alertText(sent[0])).toContain( + 'No active mappings: every keyword is unmapped, so no mentions are being ingested.', + ) + }) + it('reports a single deactivation as info', async () => { const { deps, sent } = setup({ states: [state()], deactivations: [deactivation(1)] }) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 8363ba93a6..19f927038b 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -10,6 +10,7 @@ export const BURST_MIN_DEACTIVATIONS = 3 export const BURST_MIN_SHARE = 0.25 export const CURSOR_SAFETY_MARGIN_MINUTES = 5 export const INITIAL_LOOKBACK_HOURS = 24 +export const MAX_ROWS_PER_SECTION = 25 export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' @@ -96,12 +97,32 @@ export function buildStaleAlert( text: [ `Last successful discovery: ${formatUtc(state.lastDiscoveredAt)} (${Math.floor(hoursSince(state.lastDiscoveredAt, state.checkedAt))}h ago)`, `Active mappings: ${state.activeMappings}`, + ...(state.activeMappings === 0 + ? ['No active mappings: every keyword is unmapped, so no mentions are being ingested.'] + : []), ].join('\n'), })), ], } } +// Capped because @crowd/slack silently prunes oversized messages while still reporting +// success, which would move the cursor past deactivations nobody saw. +function formatDeactivationLines(rows: IDeactivatedOctolensKeywordMapping[]): string[] { + const lines = rows + .slice(0, MAX_ROWS_PER_SECTION) + .map( + (row) => + `• ${escapeSlackText(row.keyword)} (${row.keywordId}) → ` + + `${escapeSlackText(row.segmentName)} (${escapeSlackText(row.segmentSlug)}), ` + + `deactivated ${formatUtc(row.deactivatedAt)}`, + ) + if (rows.length > MAX_ROWS_PER_SECTION) { + lines.push(`… and ${rows.length - MAX_ROWS_PER_SECTION} more`) + } + return lines +} + export function buildDeactivationAlert( deactivations: IDeactivatedOctolensKeywordMapping[], states: IOctolensMappingDiscoveryState[], @@ -124,14 +145,7 @@ export function buildDeactivationAlert( burst = burst || integrationBurst integrationSections.push({ title: `${integrationBurst ? '⚠️ ' : ''}Integration ${integrationId}: ${rows.length} of ${active + rows.length} mappings deactivated`, - text: rows - .map( - (row) => - `• ${escapeSlackText(row.keyword)} (${row.keywordId}) → ` + - `${escapeSlackText(row.segmentName)} (${escapeSlackText(row.segmentSlug)}), ` + - `deactivated ${formatUtc(row.deactivatedAt)}`, - ) - .join('\n'), + text: formatDeactivationLines(rows).join('\n'), }) } diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts index 6506ed89e2..f7df4f4103 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -5,7 +5,7 @@ import { listOctolensKeywordMappingsDeactivatedBetween, listOctolensMappingDiscoveryStates, } from '@crowd/data-access-layer' -import { READ_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { REDIS_CONFIG, getRedisClient } from '@crowd/redis' import { SlackChannel, sendSlackNotificationAsync } from '@crowd/slack' @@ -21,7 +21,8 @@ const job: IJobDefinition = { process: async (ctx) => { ctx.log.info('Running octolens-mapping-health-check job...') - const dbConnection = await getDbConnection(READ_DB_CONFIG(), 3, 0) + // Primary, not replica: replica lag beyond the cursor margin could skip deactivations. + const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 3, 0) const qx = pgpQx(dbConnection) const redis = await getRedisClient(REDIS_CONFIG()) From 65dba1d841dd6a5fdc8907837090192b3ab12cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Thu, 15 Oct 2026 15:20:45 +0200 Subject: [PATCH 03/10] fix: keep the octolens health cursor stable on retry and use the oldest active lastSeenAt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 22 +++++++++++++ .../src/jobs/octolensMappingHealth.ts | 21 ++++++++---- .../src/octolens/index.test.ts | 33 +++++++++++++++++-- .../data-access-layer/src/octolens/index.ts | 9 +++-- 4 files changed, 73 insertions(+), 12 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 09fb9cfdda..bf80ee63c7 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -188,6 +188,28 @@ describe('runOctolensMappingHealthCheck', () => { expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) + it('retries from the first lookback when the first send fails without a cursor', async () => { + const options = { + states: [state()], + deactivations: [deactivation(1)], + sendSucceeds: false, + } + const { deps, store, listDeactivatedBetween } = setup(options) + + await runOctolensMappingHealthCheck(deps) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-14T12:25:00.000Z') + + options.states = [state({ checkedAt: new Date('2026-10-15T14:30:00.000Z') })] + options.sendSucceeds = true + await runOctolensMappingHealthCheck(deps) + + expect(listDeactivatedBetween).toHaveBeenLastCalledWith( + new Date('2026-10-14T12:25:00.000Z'), + new Date('2026-10-15T14:25:00.000Z'), + ) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-15T14:25:00.000Z') + }) + it('caps listed deactivations per integration but keeps the full count', async () => { const { deps, sent } = setup({ states: [state()], diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 19f927038b..0be8d226de 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -64,14 +64,17 @@ export function isDeactivationBurst(deactivated: number, activeMappings: number) export function deactivationWindow( checkedAt: Date, cursor: string | null, -): { since: Date; until: Date } { +): { since: Date; until: Date; fromCursor: boolean } { const until = new Date(checkedAt.getTime() - CURSOR_SAFETY_MARGIN_MINUTES * MINUTE_MS) const cursorDate = cursor ? new Date(cursor) : null - const since = - cursorDate && !Number.isNaN(cursorDate.getTime()) - ? cursorDate - : new Date(until.getTime() - INITIAL_LOOKBACK_HOURS * HOUR_MS) - return { since, until } + if (cursorDate && !Number.isNaN(cursorDate.getTime())) { + return { since: cursorDate, until, fromCursor: true } + } + return { + since: new Date(until.getTime() - INITIAL_LOOKBACK_HOURS * HOUR_MS), + until, + fromCursor: false, + } } function formatUtc(date: Date): string { @@ -174,13 +177,17 @@ async function reportDeactivations( states: IOctolensMappingDiscoveryState[], result: IOctolensMappingHealthResult, ): Promise { - const { since, until } = deactivationWindow( + const { since, until, fromCursor } = deactivationWindow( states[0].checkedAt, await deps.getKey(DEACTIVATIONS_CURSOR_KEY), ) if (since >= until) { return } + // Pin the initial lookback so a retry after a failed send does not slide the window forward. + if (!fromCursor) { + await deps.setKey(DEACTIVATIONS_CURSOR_KEY, since.toISOString()) + } const deactivations = await deps.listDeactivatedBetween(since, until) if (deactivations.length > 0) { diff --git a/services/libs/data-access-layer/src/octolens/index.test.ts b/services/libs/data-access-layer/src/octolens/index.test.ts index 4da57d0e44..714ae16822 100644 --- a/services/libs/data-access-layer/src/octolens/index.test.ts +++ b/services/libs/data-access-layer/src/octolens/index.test.ts @@ -296,7 +296,7 @@ describe('octolens mapping discovery health', () => { expect(await listOctolensMappingDiscoveryStates(qx)).toEqual([]) }) - test('listOctolensMappingDiscoveryStates counts active mappings and takes lastDiscoveredAt over all rows', async ({ + test('listOctolensMappingDiscoveryStates counts active mappings and reports the oldest active lastSeenAt after a partial run', async ({ qx, }) => { const integrationId = await createIntegration(qx) @@ -320,10 +320,39 @@ describe('octolens mapping discovery health', () => { expect(states).toHaveLength(1) expect(states[0]).toMatchObject({ integrationId, activeMappings: 2 }) - expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T11:00:00.000Z') + expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T09:00:00.000Z') expect(states[0].checkedAt).toBeInstanceOf(Date) }) + test('listOctolensMappingDiscoveryStates falls back to the newest lastSeenAt without active mappings', async ({ + qx, + }) => { + const integrationId = await createIntegration(qx) + const segmentId = await createSegment(qx) + for (const keywordId of [1, 2]) { + await upsertOctolensKeywordMapping(qx, { + integrationId, + segmentId, + keywordId, + keyword: `k${keywordId}`, + }) + } + await setMappingTimestamps(qx, integrationId, 1, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T12:00:00Z', + }) + await setMappingTimestamps(qx, integrationId, 2, { + lastSeenAt: '2026-01-01T11:00:00Z', + deactivatedAt: '2026-01-01T12:00:00Z', + }) + + const states = await listOctolensMappingDiscoveryStates(qx) + + expect(states).toHaveLength(1) + expect(states[0]).toMatchObject({ integrationId, activeMappings: 0 }) + expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T11:00:00.000Z') + }) + test('listOctolensMappingDiscoveryStates skips deleted integrations', async ({ qx }) => { const integrationId = await createIntegration(qx) const deletedIntegrationId = await createIntegration(qx) diff --git a/services/libs/data-access-layer/src/octolens/index.ts b/services/libs/data-access-layer/src/octolens/index.ts index 8a1512db39..397007ccb4 100644 --- a/services/libs/data-access-layer/src/octolens/index.ts +++ b/services/libs/data-access-layer/src/octolens/index.ts @@ -106,8 +106,8 @@ export interface IOctolensMappingDiscoveryState { checkedAt: Date } -// A writing discovery run stamps lastSeenAt on every mapping it sees, so the max over active -// and deactivated rows is when discovery last wrote anything. +// A completed discovery run re-stamps every mapping it keeps active, but a run failing partway +// stamps only some, so the oldest active lastSeenAt is the last completed run. export async function listOctolensMappingDiscoveryStates( qx: QueryExecutor, ): Promise { @@ -117,7 +117,10 @@ export async function listOctolensMappingDiscoveryStates( SELECT m."integrationId", count(*) FILTER (WHERE m."deactivatedAt" IS NULL)::int AS "activeMappings", - max(m."lastSeenAt") AS "lastDiscoveredAt", + COALESCE( + min(m."lastSeenAt") FILTER (WHERE m."deactivatedAt" IS NULL), + max(m."lastSeenAt") + ) AS "lastDiscoveredAt", now() AS "checkedAt" FROM integration."octolensKeywordMappings" m JOIN public.integrations i ON i.id = m."integrationId" AND i."deletedAt" IS NULL From 69c787ab88e06d6c39e71051ad24daf7b414eba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Thu, 15 Oct 2026 15:32:09 +0200 Subject: [PATCH 04/10] fix: alert on octolens integrations whose discovery never completed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 43 ++++++++++++++++++- .../src/jobs/octolensMappingHealth.ts | 26 +++++++++-- .../src/octolens/index.test.ts | 25 ++++++++--- .../data-access-layer/src/octolens/index.ts | 37 ++++++++++------ 4 files changed, 109 insertions(+), 22 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index bf80ee63c7..97c3814d73 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -18,11 +18,14 @@ import { const CHECKED_AT = new Date('2026-10-15T12:30:00.000Z') const UNTIL = '2026-10-15T12:25:00.000Z' -function state(overrides: Partial = {}) { +function state( + overrides: Partial = {}, +): IOctolensMappingDiscoveryState { return { integrationId: 'int-1', activeMappings: 20, lastDiscoveredAt: new Date('2026-10-15T12:05:00.000Z'), + integrationCreatedAt: new Date('2026-09-01T00:00:00.000Z'), checkedAt: CHECKED_AT, ...overrides, } @@ -133,6 +136,44 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent).toHaveLength(2) }) + it('gives a never-discovered integration 3h from creation before alerting', async () => { + const { deps, sent } = setup({ + states: [ + state({ + activeMappings: 0, + lastDiscoveredAt: null, + integrationCreatedAt: new Date('2026-10-15T10:00:00.000Z'), + }), + ], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent).toEqual([]) + }) + + it('alerts once on an integration whose discovery never completed', async () => { + const { deps, store, sent } = setup({ + states: [ + state({ + activeMappings: 0, + lastDiscoveredAt: null, + integrationCreatedAt: new Date('2026-10-15T08:00:00.000Z'), + }), + ], + }) + + await runOctolensMappingHealthCheck(deps) + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(1) + expect(alertText(sent[0])).toContain( + 'Discovery has never completed a run since the integration was created at 2026-10-15 08:00 UTC (4h ago)', + ) + expect(alertText(sent[0])).not.toContain('Last successful discovery') + expect(store.get(staleAlertedKey('int-1'))).toBe('never:2026-10-15T08:00:00.000Z') + }) + it('alerts again when a new stale episode starts', async () => { const { deps, sent } = setup({ states: [state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') })], diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 0be8d226de..992af41b65 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -49,8 +49,26 @@ export function hoursSince(from: Date, now: Date): number { return (now.getTime() - from.getTime()) / HOUR_MS } +function stalenessBaseline(state: IOctolensMappingDiscoveryState): Date { + return state.lastDiscoveredAt ?? state.integrationCreatedAt +} + export function isStale(state: IOctolensMappingDiscoveryState): boolean { - return hoursSince(state.lastDiscoveredAt, state.checkedAt) > STALE_AFTER_HOURS + return hoursSince(stalenessBaseline(state), state.checkedAt) > STALE_AFTER_HOURS +} + +export function staleEpisodeId(state: IOctolensMappingDiscoveryState): string { + return state.lastDiscoveredAt + ? state.lastDiscoveredAt.toISOString() + : `never:${state.integrationCreatedAt.toISOString()}` +} + +function describeLastDiscovery(state: IOctolensMappingDiscoveryState): string { + const baseline = stalenessBaseline(state) + const ago = `${formatUtc(baseline)} (${Math.floor(hoursSince(baseline, state.checkedAt))}h ago)` + return state.lastDiscoveredAt + ? `Last successful discovery: ${ago}` + : `Discovery has never completed a run since the integration was created at ${ago}` } export function isDeactivationBurst(deactivated: number, activeMappings: number): boolean { @@ -98,7 +116,7 @@ export function buildStaleAlert( ...states.map((state) => ({ title: `Integration ${state.integrationId}`, text: [ - `Last successful discovery: ${formatUtc(state.lastDiscoveredAt)} (${Math.floor(hoursSince(state.lastDiscoveredAt, state.checkedAt))}h ago)`, + describeLastDiscovery(state), `Active mappings: ${state.activeMappings}`, ...(state.activeMappings === 0 ? ['No active mappings: every keyword is unmapped, so no mentions are being ingested.'] @@ -210,7 +228,7 @@ async function reportStaleIntegrations( const toAlert: IOctolensMappingDiscoveryState[] = [] for (const state of states.filter(isStale)) { const alertedFor = await deps.getKey(staleAlertedKey(state.integrationId)) - if (alertedFor !== state.lastDiscoveredAt.toISOString()) { + if (alertedFor !== staleEpisodeId(state)) { toAlert.push(state) } } @@ -225,7 +243,7 @@ async function reportStaleIntegrations( for (const state of toAlert) { await deps.setKey( staleAlertedKey(state.integrationId), - state.lastDiscoveredAt.toISOString(), + staleEpisodeId(state), STALE_REMINDER_HOURS * 60 * 60, ) } diff --git a/services/libs/data-access-layer/src/octolens/index.test.ts b/services/libs/data-access-layer/src/octolens/index.test.ts index 714ae16822..ad92b852d3 100644 --- a/services/libs/data-access-layer/src/octolens/index.test.ts +++ b/services/libs/data-access-layer/src/octolens/index.test.ts @@ -288,12 +288,27 @@ async function setMappingTimestamps( } describe('octolens mapping discovery health', () => { - test('listOctolensMappingDiscoveryStates returns nothing when there are no mappings', async ({ + test('listOctolensMappingDiscoveryStates returns live octolens integrations without mappings', async ({ qx, }) => { - await createIntegration(qx) + const integrationId = await createIntegration(qx) + const deletedIntegrationId = await createIntegration(qx) + await qx.result(`UPDATE public.integrations SET "deletedAt" = NOW() WHERE id = $(id)`, { + id: deletedIntegrationId, + }) + await qx.result( + ` + INSERT INTO public.integrations (id, platform, status, "tenantId", "createdAt", "updatedAt") + VALUES ($(id), 'github', 'done', $(tenantId), NOW(), NOW()) + `, + { id: generateUUIDv1(), tenantId: DEFAULT_TENANT_ID }, + ) - expect(await listOctolensMappingDiscoveryStates(qx)).toEqual([]) + const states = await listOctolensMappingDiscoveryStates(qx) + + expect(states).toHaveLength(1) + expect(states[0]).toMatchObject({ integrationId, activeMappings: 0, lastDiscoveredAt: null }) + expect(states[0].integrationCreatedAt).toBeInstanceOf(Date) }) test('listOctolensMappingDiscoveryStates counts active mappings and reports the oldest active lastSeenAt after a partial run', async ({ @@ -320,7 +335,7 @@ describe('octolens mapping discovery health', () => { expect(states).toHaveLength(1) expect(states[0]).toMatchObject({ integrationId, activeMappings: 2 }) - expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T09:00:00.000Z') + expect(states[0].lastDiscoveredAt?.toISOString()).toBe('2026-01-01T09:00:00.000Z') expect(states[0].checkedAt).toBeInstanceOf(Date) }) @@ -350,7 +365,7 @@ describe('octolens mapping discovery health', () => { expect(states).toHaveLength(1) expect(states[0]).toMatchObject({ integrationId, activeMappings: 0 }) - expect(states[0].lastDiscoveredAt.toISOString()).toBe('2026-01-01T11:00:00.000Z') + expect(states[0].lastDiscoveredAt?.toISOString()).toBe('2026-01-01T11:00:00.000Z') }) test('listOctolensMappingDiscoveryStates skips deleted integrations', async ({ qx }) => { diff --git a/services/libs/data-access-layer/src/octolens/index.ts b/services/libs/data-access-layer/src/octolens/index.ts index 397007ccb4..9e347b4b0c 100644 --- a/services/libs/data-access-layer/src/octolens/index.ts +++ b/services/libs/data-access-layer/src/octolens/index.ts @@ -1,3 +1,5 @@ +import { PlatformType } from '@crowd/types' + import { QueryExecutor } from '../queryExecutor' export interface IOctolensKeywordMapping { @@ -102,7 +104,8 @@ type Timestamped = Omit & Record export interface IOctolensMappingDiscoveryState { integrationId: string activeMappings: number - lastDiscoveredAt: Date + lastDiscoveredAt: Date | null + integrationCreatedAt: Date checkedAt: Date } @@ -111,26 +114,36 @@ export interface IOctolensMappingDiscoveryState { export async function listOctolensMappingDiscoveryStates( qx: QueryExecutor, ): Promise { - const rows: Timestamped[] = - await qx.select( - ` + const rows: (Omit< + IOctolensMappingDiscoveryState, + 'lastDiscoveredAt' | 'integrationCreatedAt' | 'checkedAt' + > & { + lastDiscoveredAt: string | null + integrationCreatedAt: string + checkedAt: string + })[] = await qx.select( + ` SELECT - m."integrationId", - count(*) FILTER (WHERE m."deactivatedAt" IS NULL)::int AS "activeMappings", + i.id AS "integrationId", + count(m.id) FILTER (WHERE m."deactivatedAt" IS NULL)::int AS "activeMappings", COALESCE( min(m."lastSeenAt") FILTER (WHERE m."deactivatedAt" IS NULL), max(m."lastSeenAt") ) AS "lastDiscoveredAt", + i."createdAt" AS "integrationCreatedAt", now() AS "checkedAt" - FROM integration."octolensKeywordMappings" m - JOIN public.integrations i ON i.id = m."integrationId" AND i."deletedAt" IS NULL - GROUP BY m."integrationId" - ORDER BY m."integrationId" + FROM public.integrations i + LEFT JOIN integration."octolensKeywordMappings" m ON m."integrationId" = i.id + WHERE i.platform = $(platform) AND i."deletedAt" IS NULL + GROUP BY i.id, i."createdAt" + ORDER BY i.id `, - ) + { platform: PlatformType.OCTOLENS }, + ) return rows.map((row) => ({ ...row, - lastDiscoveredAt: new Date(row.lastDiscoveredAt), + lastDiscoveredAt: row.lastDiscoveredAt ? new Date(row.lastDiscoveredAt) : null, + integrationCreatedAt: new Date(row.integrationCreatedAt), checkedAt: new Date(row.checkedAt), })) } From 7abff7ad7f62c6ec74942eb58f1be65736e8757e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 16 Oct 2026 14:26:07 +0200 Subject: [PATCH 05/10] feat: judge octolens mapping discovery health across all subproject integrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 260 ++++++++++-------- .../src/jobs/octolensMappingHealth.ts | 214 +++++++------- .../jobs/octolensMappingHealthCheck.job.ts | 10 +- .../src/octolens/health.test.ts | 222 +++++++++++++++ .../data-access-layer/src/octolens/health.ts | 95 +++++++ 5 files changed, 570 insertions(+), 231 deletions(-) create mode 100644 services/libs/data-access-layer/src/octolens/health.test.ts create mode 100644 services/libs/data-access-layer/src/octolens/health.ts diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 97c3814d73..5d4db72d5a 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -2,30 +2,31 @@ import { describe, expect, it, vi } from 'vitest' import type { IDeactivatedOctolensKeywordMapping, - IOctolensMappingDiscoveryState, -} from '@crowd/data-access-layer' + IOctolensMappingDiscoveryHealth, +} from '@crowd/data-access-layer/src/octolens/health' import { SlackPersona } from '@crowd/slack' import { DEACTIVATIONS_CURSOR_KEY, IOctolensMappingHealthAlert, IOctolensMappingHealthDeps, + STALE_ALERTED_KEY, + groupIntoDiscoveryRuns, isDeactivationBurst, runOctolensMappingHealthCheck, - staleAlertedKey, } from './octolensMappingHealth' const CHECKED_AT = new Date('2026-10-15T12:30:00.000Z') const UNTIL = '2026-10-15T12:25:00.000Z' +const RUN_AT = new Date('2026-10-15T12:00:00.000Z') -function state( - overrides: Partial = {}, -): IOctolensMappingDiscoveryState { +function health( + overrides: Partial = {}, +): IOctolensMappingDiscoveryHealth { return { - integrationId: 'int-1', activeMappings: 20, - lastDiscoveredAt: new Date('2026-10-15T12:05:00.000Z'), - integrationCreatedAt: new Date('2026-09-01T00:00:00.000Z'), + integrations: 4, + lastDiscoveredAt: RUN_AT, checkedAt: CHECKED_AT, ...overrides, } @@ -42,13 +43,14 @@ function deactivation( segmentId: 'seg-1', segmentName: 'Project One', segmentSlug: 'project-one', - deactivatedAt: new Date('2026-10-15T12:05:00.000Z'), + deactivatedAt: RUN_AT, + integrationDeleted: false, ...overrides, } } function setup(options: { - states: IOctolensMappingDiscoveryState[] + health: IOctolensMappingDiscoveryHealth deactivations?: IDeactivatedOctolensKeywordMapping[] store?: Record sendSucceeds?: boolean @@ -58,7 +60,7 @@ function setup(options: { const sent: IOctolensMappingHealthAlert[] = [] const listDeactivatedBetween = vi.fn(async () => options.deactivations ?? []) const deps: IOctolensMappingHealthDeps = { - listStates: async () => options.states, + getHealth: async () => options.health, listDeactivatedBetween, getKey: async (key) => store.get(key) ?? null, setKey: async (key, value, expireAfterSeconds) => { @@ -78,8 +80,10 @@ function alertText(alert: IOctolensMappingHealthAlert): string { } describe('runOctolensMappingHealthCheck', () => { - it('does nothing when no integration has mappings yet', async () => { - const { deps, store, sent, listDeactivatedBetween } = setup({ states: [] }) + it('stays quiet and keeps no state while there are no active mappings', async () => { + const { deps, store, sent, listDeactivatedBetween } = setup({ + health: health({ activeMappings: 0, integrations: 0, lastDiscoveredAt: null }), + }) await runOctolensMappingHealthCheck(deps) @@ -88,8 +92,8 @@ describe('runOctolensMappingHealthCheck', () => { expect(listDeactivatedBetween).not.toHaveBeenCalled() }) - it('sends nothing for a healthy integration and moves the cursor', async () => { - const { deps, store, sent } = setup({ states: [state()] }) + it('sends nothing when healthy and moves the cursor', async () => { + const { deps, store, sent } = setup({ health: health() }) await runOctolensMappingHealthCheck(deps) @@ -98,7 +102,7 @@ describe('runOctolensMappingHealthCheck', () => { }) it('looks back 24h on the first run and continues from the cursor afterwards', async () => { - const first = setup({ states: [state()] }) + const first = setup({ health: health() }) await runOctolensMappingHealthCheck(first.deps) expect(first.listDeactivatedBetween).toHaveBeenCalledWith( new Date('2026-10-14T12:25:00.000Z'), @@ -106,7 +110,7 @@ describe('runOctolensMappingHealthCheck', () => { ) const next = setup({ - states: [state()], + health: health(), store: { [DEACTIVATIONS_CURSOR_KEY]: '2026-10-15T11:25:00.000Z' }, }) await runOctolensMappingHealthCheck(next.deps) @@ -116,68 +120,43 @@ describe('runOctolensMappingHealthCheck', () => { ) }) - it('alerts on a stale integration once, then again only after the reminder expires', async () => { - const stale = state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }) - const { deps, store, ttls, sent } = setup({ states: [stale] }) - - await runOctolensMappingHealthCheck(deps) - await runOctolensMappingHealthCheck(deps) - - expect(sent).toHaveLength(1) - expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) - expect(alertText(sent[0])).toContain('2026-10-15 08:00 UTC (4h ago)') - expect(alertText(sent[0])).toContain('Active mappings: 20') - expect(alertText(sent[0])).not.toContain('No active mappings') - expect(ttls.get(staleAlertedKey('int-1'))).toBe(24 * 60 * 60) - - store.delete(staleAlertedKey('int-1')) - await runOctolensMappingHealthCheck(deps) - - expect(sent).toHaveLength(2) - }) - - it('gives a never-discovered integration 3h from creation before alerting', async () => { - const { deps, sent } = setup({ - states: [ - state({ - activeMappings: 0, - lastDiscoveredAt: null, - integrationCreatedAt: new Date('2026-10-15T10:00:00.000Z'), - }), - ], + it('does not query when the cursor is already at the window end', async () => { + const { deps, listDeactivatedBetween } = setup({ + health: health(), + store: { [DEACTIVATIONS_CURSOR_KEY]: UNTIL }, }) await runOctolensMappingHealthCheck(deps) - expect(sent).toEqual([]) + expect(listDeactivatedBetween).not.toHaveBeenCalled() }) - it('alerts once on an integration whose discovery never completed', async () => { - const { deps, store, sent } = setup({ - states: [ - state({ - activeMappings: 0, - lastDiscoveredAt: null, - integrationCreatedAt: new Date('2026-10-15T08:00:00.000Z'), - }), - ], - }) + it('alerts on staleness once per episode, then again after the reminder expires', async () => { + const stale = health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }) + const { deps, store, ttls, sent } = setup({ health: stale }) await runOctolensMappingHealthCheck(deps) await runOctolensMappingHealthCheck(deps) expect(sent).toHaveLength(1) - expect(alertText(sent[0])).toContain( - 'Discovery has never completed a run since the integration was created at 2026-10-15 08:00 UTC (4h ago)', - ) - expect(alertText(sent[0])).not.toContain('Last successful discovery') - expect(store.get(staleAlertedKey('int-1'))).toBe('never:2026-10-15T08:00:00.000Z') + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + const text = alertText(sent[0]) + expect(text).toContain('Last successful run: 2026-10-15 08:00 UTC (4h ago)') + expect(text).toContain('Active mappings: 20 across 4 projects') + expect(text).toContain('CROWD_OCTOLENS_API_TOKEN') + expect(store.get(STALE_ALERTED_KEY)).toBe('2026-10-15T08:00:00.000Z') + expect(ttls.get(STALE_ALERTED_KEY)).toBe(24 * 60 * 60) + + store.delete(STALE_ALERTED_KEY) + await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(2) }) it('alerts again when a new stale episode starts', async () => { const { deps, sent } = setup({ - states: [state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') })], - store: { [staleAlertedKey('int-1')]: '2026-10-14T08:00:00.000Z' }, + health: health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), + store: { [STALE_ALERTED_KEY]: '2026-10-14T08:00:00.000Z' }, }) await runOctolensMappingHealthCheck(deps) @@ -187,7 +166,7 @@ describe('runOctolensMappingHealthCheck', () => { it('does not treat discovery within the stale threshold as stale', async () => { const { deps, sent } = setup({ - states: [state({ lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z') })], + health: health({ lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z') }), }) await runOctolensMappingHealthCheck(deps) @@ -195,9 +174,9 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent).toEqual([]) }) - it('keeps alerting a stale integration while the Slack send fails', async () => { + it('keeps alerting staleness while the Slack send fails', async () => { const { deps, store, sent } = setup({ - states: [state({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') })], + health: health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), sendSucceeds: false, }) @@ -205,33 +184,63 @@ describe('runOctolensMappingHealthCheck', () => { await runOctolensMappingHealthCheck(deps) expect(sent).toHaveLength(2) - expect(store.has(staleAlertedKey('int-1'))).toBe(false) + expect(store.has(STALE_ALERTED_KEY)).toBe(false) }) - it('reports a burst of deactivations as a warning listing every keyword', async () => { + it('reports a burst in one run as a warning listing every keyword and project', async () => { const { deps, store, sent } = setup({ - states: [state()], - deactivations: [deactivation(1), deactivation(2), deactivation(3)], + health: health(), + deactivations: [ + deactivation(1), + deactivation(2), + deactivation(3, { + integrationId: 'int-2', + segmentName: 'Project Two', + segmentSlug: 'project-two', + integrationDeleted: true, + }), + ], }) await runOctolensMappingHealthCheck(deps) expect(sent).toHaveLength(1) expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + expect(sent[0].title).toBe('Octolens keyword mappings deactivated in bulk') const text = alertText(sent[0]) - expect(text).toContain('3 of 23 mappings deactivated') - for (const keywordId of [1, 2, 3]) { - expect(text).toContain( - `keyword-${keywordId} (${keywordId}) → Project One (project-one), deactivated 2026-10-15 12:05 UTC`, - ) - } - expect(text).toContain('re-fetches its whole history automatically') + expect(text).toContain('Deactivated mappings (3, 20 still active)') + expect(text).toContain('• ⚠️ 2026-10-15 12:00 UTC: 3 deactivated') + expect(text).toContain( + 'keyword-1 (1) → Project One (project-one), deactivated 2026-10-15 12:00 UTC', + ) + expect(text).toContain( + 'keyword-3 (3) → Project Two (project-two), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', + ) + expect(text).toContain('queue a manual backfill') + expect(text).toContain('Nothing is re-fetched automatically') expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) + it('reports deactivations spread over runs as info', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: [ + deactivation(1, { deactivatedAt: new Date('2026-10-15T10:00:00.000Z') }), + deactivation(2, { deactivatedAt: new Date('2026-10-15T11:00:00.000Z') }), + deactivation(3, { deactivatedAt: new Date('2026-10-15T12:00:00.000Z') }), + ], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) + expect(alertText(sent[0])).toContain('Discovery runs (3)') + expect(alertText(sent[0])).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') + }) + it('retries from the first lookback when the first send fails without a cursor', async () => { const options = { - states: [state()], + health: health(), deactivations: [deactivation(1)], sendSucceeds: false, } @@ -240,7 +249,7 @@ describe('runOctolensMappingHealthCheck', () => { await runOctolensMappingHealthCheck(deps) expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-14T12:25:00.000Z') - options.states = [state({ checkedAt: new Date('2026-10-15T14:30:00.000Z') })] + options.health = health({ checkedAt: new Date('2026-10-15T14:30:00.000Z') }) options.sendSucceeds = true await runOctolensMappingHealthCheck(deps) @@ -251,59 +260,51 @@ describe('runOctolensMappingHealthCheck', () => { expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-15T14:25:00.000Z') }) - it('caps listed deactivations per integration but keeps the full count', async () => { - const { deps, sent } = setup({ - states: [state()], - deactivations: Array.from({ length: 30 }, (_, i) => deactivation(i + 1)), + it('keeps the cursor when the deactivation alert fails so the next run retries', async () => { + const cursor = '2026-10-15T11:25:00.000Z' + const { deps, store, sent } = setup({ + health: health(), + deactivations: [deactivation(1)], + store: { [DEACTIVATIONS_CURSOR_KEY]: cursor }, + sendSucceeds: false, }) - await runOctolensMappingHealthCheck(deps) + const result = await runOctolensMappingHealthCheck(deps) - const text = alertText(sent[0]) - expect(text).toContain('30 of 50 mappings deactivated') - expect(text).toContain('keyword-25 (25)') - expect(text).not.toContain('keyword-26 (26)') - expect(text).toContain('… and 5 more') + expect(sent).toHaveLength(1) + expect(result.sendFailures).toBe(1) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(cursor) }) - it('flags a stale integration that has no active mappings left', async () => { + it('caps listed mappings and runs but keeps the full counts', async () => { const { deps, sent } = setup({ - states: [ - state({ activeMappings: 0, lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), - ], + health: health(), + deactivations: Array.from({ length: 30 }, (_, i) => + deactivation(i + 1, { deactivatedAt: new Date(RUN_AT.getTime() - i * 60 * 60 * 1000) }), + ), }) await runOctolensMappingHealthCheck(deps) - expect(alertText(sent[0])).toContain( - 'No active mappings: every keyword is unmapped, so no mentions are being ingested.', - ) - }) - - it('reports a single deactivation as info', async () => { - const { deps, sent } = setup({ states: [state()], deactivations: [deactivation(1)] }) - - await runOctolensMappingHealthCheck(deps) - - expect(sent).toHaveLength(1) - expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) - expect(alertText(sent[0])).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') + const text = alertText(sent[0]) + expect(text).toContain('Deactivated mappings (30, 20 still active)') + expect(text).toContain('Discovery runs (30)') + expect(text).toContain('keyword-25 (25)') + expect(text).not.toContain('keyword-26 (26)') + expect(text).toContain('… and 5 more mappings') + expect(text).toContain('… and 20 more runs') }) - it('keeps the cursor when the deactivation alert fails so the next run retries', async () => { - const cursor = '2026-10-15T11:25:00.000Z' - const { deps, store, sent } = setup({ - states: [state()], - deactivations: [deactivation(1)], - store: { [DEACTIVATIONS_CURSOR_KEY]: cursor }, - sendSucceeds: false, + it('truncates very long keyword and project names', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: [deactivation(1, { keyword: 'k'.repeat(500) })], }) - const result = await runOctolensMappingHealthCheck(deps) + await runOctolensMappingHealthCheck(deps) - expect(sent).toHaveLength(1) - expect(result.sendFailures).toBe(1) - expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(cursor) + expect(alertText(sent[0])).toContain(`${'k'.repeat(99)}…`) + expect(alertText(sent[0])).not.toContain('k'.repeat(100)) }) }) @@ -318,3 +319,24 @@ describe('isDeactivationBurst', () => { expect(isDeactivationBurst(2, 7)).toBe(false) }) }) + +describe('groupIntoDiscoveryRuns', () => { + it('groups rows by their shared deactivation timestamp in time order', () => { + const later = new Date('2026-10-15T11:00:00.000Z') + const earlier = new Date('2026-10-15T10:00:00.000Z') + + const runs = groupIntoDiscoveryRuns( + [ + deactivation(1, { deactivatedAt: later }), + deactivation(2, { deactivatedAt: earlier }), + deactivation(3, { deactivatedAt: later }), + ], + 100, + ) + + expect(runs).toEqual([ + { at: earlier, deactivated: 1, burst: false }, + { at: later, deactivated: 2, burst: false }, + ]) + }) +}) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 992af41b65..b28ce4a022 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -1,7 +1,7 @@ import type { IDeactivatedOctolensKeywordMapping, - IOctolensMappingDiscoveryState, -} from '@crowd/data-access-layer' + IOctolensMappingDiscoveryHealth, +} from '@crowd/data-access-layer/src/octolens/health' import { SlackMessageSection, SlackPersona, escapeSlackText } from '@crowd/slack' export const STALE_AFTER_HOURS = 3 @@ -10,13 +10,12 @@ export const BURST_MIN_DEACTIVATIONS = 3 export const BURST_MIN_SHARE = 0.25 export const CURSOR_SAFETY_MARGIN_MINUTES = 5 export const INITIAL_LOOKBACK_HOURS = 24 -export const MAX_ROWS_PER_SECTION = 25 +export const MAX_LISTED_KEYWORDS = 25 +export const MAX_LISTED_RUNS = 10 +const MAX_NAME_LENGTH = 100 export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' - -export function staleAlertedKey(integrationId: string): string { - return `octolens-mapping-health:stale-alerted:${integrationId}` -} +export const STALE_ALERTED_KEY = 'octolens-mapping-health:stale-alerted' const HOUR_MS = 60 * 60 * 1000 const MINUTE_MS = 60 * 1000 @@ -28,7 +27,7 @@ export interface IOctolensMappingHealthAlert { } export interface IOctolensMappingHealthDeps { - listStates: () => Promise + getHealth: () => Promise listDeactivatedBetween: ( since: Date, until: Date, @@ -39,36 +38,27 @@ export interface IOctolensMappingHealthDeps { } export interface IOctolensMappingHealthResult { - integrations: number - staleAlerted: number + activeMappings: number + staleAlerted: boolean deactivationsReported: number sendFailures: number } -export function hoursSince(from: Date, now: Date): number { - return (now.getTime() - from.getTime()) / HOUR_MS +export interface IDiscoveryRun { + at: Date + deactivated: number + burst: boolean } -function stalenessBaseline(state: IOctolensMappingDiscoveryState): Date { - return state.lastDiscoveredAt ?? state.integrationCreatedAt -} - -export function isStale(state: IOctolensMappingDiscoveryState): boolean { - return hoursSince(stalenessBaseline(state), state.checkedAt) > STALE_AFTER_HOURS -} - -export function staleEpisodeId(state: IOctolensMappingDiscoveryState): string { - return state.lastDiscoveredAt - ? state.lastDiscoveredAt.toISOString() - : `never:${state.integrationCreatedAt.toISOString()}` +export function hoursSince(from: Date, now: Date): number { + return (now.getTime() - from.getTime()) / HOUR_MS } -function describeLastDiscovery(state: IOctolensMappingDiscoveryState): string { - const baseline = stalenessBaseline(state) - const ago = `${formatUtc(baseline)} (${Math.floor(hoursSince(baseline, state.checkedAt))}h ago)` - return state.lastDiscoveredAt - ? `Last successful discovery: ${ago}` - : `Discovery has never completed a run since the integration was created at ${ago}` +export function isStale(health: IOctolensMappingDiscoveryHealth): boolean { + return ( + health.lastDiscoveredAt !== null && + hoursSince(health.lastDiscoveredAt, health.checkedAt) > STALE_AFTER_HOURS + ) } export function isDeactivationBurst(deactivated: number, activeMappings: number): boolean { @@ -79,6 +69,25 @@ export function isDeactivationBurst(deactivated: number, activeMappings: number) return total > 0 && deactivated / total >= BURST_MIN_SHARE } +// One discovery run deactivates in one transaction, so its rows share one now() timestamp. +export function groupIntoDiscoveryRuns( + deactivations: IDeactivatedOctolensKeywordMapping[], + activeMappings: number, +): IDiscoveryRun[] { + const counts = new Map() + for (const row of deactivations) { + const at = row.deactivatedAt.getTime() + counts.set(at, (counts.get(at) ?? 0) + 1) + } + return [...counts.entries()] + .sort(([a], [b]) => a - b) + .map(([at, deactivated]) => ({ + at: new Date(at), + deactivated, + burst: isDeactivationBurst(deactivated, activeMappings), + })) +} + export function deactivationWindow( checkedAt: Date, cursor: string | null, @@ -99,76 +108,65 @@ function formatUtc(date: Date): string { return `${date.toISOString().slice(0, 16).replace('T', ' ')} UTC` } +function formatName(name: string): string { + const short = name.length > MAX_NAME_LENGTH ? `${name.slice(0, MAX_NAME_LENGTH - 1)}…` : name + return escapeSlackText(short) +} + export function buildStaleAlert( - states: IOctolensMappingDiscoveryState[], + health: IOctolensMappingDiscoveryHealth, ): IOctolensMappingHealthAlert { + const lastDiscoveredAt = health.lastDiscoveredAt as Date return { persona: SlackPersona.WARNING_PROPAGATOR, title: 'Octolens mapping discovery looks stale', sections: [ { title: '', - text: - `Keyword-to-project mapping discovery has not completed a run in over ${STALE_AFTER_HOURS} hours. ` + - 'Existing mappings stay as they are, but keywords added, remapped or ' + - 'removed in Octolens are not picked up until discovery succeeds again.', - }, - ...states.map((state) => ({ - title: `Integration ${state.integrationId}`, text: [ - describeLastDiscovery(state), - `Active mappings: ${state.activeMappings}`, - ...(state.activeMappings === 0 - ? ['No active mappings: every keyword is unmapped, so no mentions are being ingested.'] - : []), + `Keyword-to-project mapping discovery has not completed a run in over ${STALE_AFTER_HOURS} hours.`, + `Last successful run: ${formatUtc(lastDiscoveredAt)} (${Math.floor(hoursSince(lastDiscoveredAt, health.checkedAt))}h ago)`, + `Active mappings: ${health.activeMappings} across ${health.integrations} projects`, + 'Existing mappings keep routing mentions, but keywords added, remapped or removed in ' + + 'Octolens are not picked up until discovery succeeds again. Check the ' + + '`connectors-octolens-discovery` schedule in connectors_worker and that ' + + '`CROWD_OCTOLENS_API_TOKEN` is set.', ].join('\n'), - })), + }, ], } } // Capped because @crowd/slack silently prunes oversized messages while still reporting // success, which would move the cursor past deactivations nobody saw. -function formatDeactivationLines(rows: IDeactivatedOctolensKeywordMapping[]): string[] { - const lines = rows - .slice(0, MAX_ROWS_PER_SECTION) - .map( - (row) => - `• ${escapeSlackText(row.keyword)} (${row.keywordId}) → ` + - `${escapeSlackText(row.segmentName)} (${escapeSlackText(row.segmentSlug)}), ` + - `deactivated ${formatUtc(row.deactivatedAt)}`, - ) - if (rows.length > MAX_ROWS_PER_SECTION) { - lines.push(`… and ${rows.length - MAX_ROWS_PER_SECTION} more`) +function capLines(lines: string[], max: number, noun: string): string[] { + if (lines.length <= max) { + return lines } - return lines + return [...lines.slice(0, max), `… and ${lines.length - max} more ${noun}`] +} + +function formatRun(run: IDiscoveryRun): string { + return `• ${run.burst ? '⚠️ ' : ''}${formatUtc(run.at)}: ${run.deactivated} deactivated` +} + +function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { + return ( + `• ${formatName(row.keyword)} (${row.keywordId}) → ` + + `${formatName(row.segmentName)} (${formatName(row.segmentSlug)}), ` + + `deactivated ${formatUtc(row.deactivatedAt)}` + + (row.integrationDeleted ? ', project has no Octolens keywords left' : '') + ) } export function buildDeactivationAlert( deactivations: IDeactivatedOctolensKeywordMapping[], - states: IOctolensMappingDiscoveryState[], + activeMappings: number, since: Date, until: Date, ): IOctolensMappingHealthAlert { - const activeByIntegration = new Map(states.map((s) => [s.integrationId, s.activeMappings])) - const byIntegration = new Map() - for (const row of deactivations) { - const bucket = byIntegration.get(row.integrationId) ?? [] - bucket.push(row) - byIntegration.set(row.integrationId, bucket) - } - - let burst = false - const integrationSections: SlackMessageSection[] = [] - for (const [integrationId, rows] of byIntegration) { - const active = activeByIntegration.get(integrationId) ?? 0 - const integrationBurst = isDeactivationBurst(rows.length, active) - burst = burst || integrationBurst - integrationSections.push({ - title: `${integrationBurst ? '⚠️ ' : ''}Integration ${integrationId}: ${rows.length} of ${active + rows.length} mappings deactivated`, - text: formatDeactivationLines(rows).join('\n'), - }) - } + const runs = groupIntoDiscoveryRuns(deactivations, activeMappings) + const burst = runs.some((run) => run.burst) return { persona: burst ? SlackPersona.WARNING_PROPAGATOR : SlackPersona.INFO_NOTIFIER, @@ -181,22 +179,31 @@ export function buildDeactivationAlert( text: [ `Window: ${formatUtc(since)} → ${formatUtc(until)}`, 'Discovery no longer saw these keywords mapped to a project, so their mentions stop ' + - 'reaching it. If a keyword gets mapped again, the mention poll re-fetches its whole ' + - 'history automatically.', + 'reaching it. Nothing is re-fetched automatically: if a keyword gets mapped again, ' + + 'queue a manual backfill from its deactivation time to recover the mentions in between.', ].join('\n'), }, - ...integrationSections, + { + title: `Discovery runs (${runs.length})`, + text: capLines(runs.map(formatRun), MAX_LISTED_RUNS, 'runs').join('\n'), + }, + { + title: `Deactivated mappings (${deactivations.length}, ${activeMappings} still active)`, + text: capLines(deactivations.map(formatDeactivation), MAX_LISTED_KEYWORDS, 'mappings').join( + '\n', + ), + }, ], } } async function reportDeactivations( deps: IOctolensMappingHealthDeps, - states: IOctolensMappingDiscoveryState[], + health: IOctolensMappingDiscoveryHealth, result: IOctolensMappingHealthResult, ): Promise { const { since, until, fromCursor } = deactivationWindow( - states[0].checkedAt, + health.checkedAt, await deps.getKey(DEACTIVATIONS_CURSOR_KEY), ) if (since >= until) { @@ -209,8 +216,8 @@ async function reportDeactivations( const deactivations = await deps.listDeactivatedBetween(since, until) if (deactivations.length > 0) { - const sent = await deps.sendAlert(buildDeactivationAlert(deactivations, states, since, until)) - if (!sent) { + const alert = buildDeactivationAlert(deactivations, health.activeMappings, since, until) + if (!(await deps.sendAlert(alert))) { result.sendFailures++ return } @@ -220,51 +227,44 @@ async function reportDeactivations( await deps.setKey(DEACTIVATIONS_CURSOR_KEY, until.toISOString()) } -async function reportStaleIntegrations( +async function reportStaleness( deps: IOctolensMappingHealthDeps, - states: IOctolensMappingDiscoveryState[], + health: IOctolensMappingDiscoveryHealth, result: IOctolensMappingHealthResult, ): Promise { - const toAlert: IOctolensMappingDiscoveryState[] = [] - for (const state of states.filter(isStale)) { - const alertedFor = await deps.getKey(staleAlertedKey(state.integrationId)) - if (alertedFor !== staleEpisodeId(state)) { - toAlert.push(state) - } + if (!isStale(health)) { + return } - if (toAlert.length === 0) { + const episode = (health.lastDiscoveredAt as Date).toISOString() + if ((await deps.getKey(STALE_ALERTED_KEY)) === episode) { return } - if (!(await deps.sendAlert(buildStaleAlert(toAlert)))) { + if (!(await deps.sendAlert(buildStaleAlert(health)))) { result.sendFailures++ return } - for (const state of toAlert) { - await deps.setKey( - staleAlertedKey(state.integrationId), - staleEpisodeId(state), - STALE_REMINDER_HOURS * 60 * 60, - ) - } - result.staleAlerted = toAlert.length + await deps.setKey(STALE_ALERTED_KEY, episode, STALE_REMINDER_HOURS * 60 * 60) + result.staleAlerted = true } +// No active mapping means discovery never completed a run (or the token was never set): a run +// that sees no mappings writes nothing, so it cannot empty a populated table. export async function runOctolensMappingHealthCheck( deps: IOctolensMappingHealthDeps, ): Promise { - const states = await deps.listStates() + const health = await deps.getHealth() const result: IOctolensMappingHealthResult = { - integrations: states.length, - staleAlerted: 0, + activeMappings: health.activeMappings, + staleAlerted: false, deactivationsReported: 0, sendFailures: 0, } - if (states.length === 0) { + if (health.activeMappings === 0) { return result } - await reportDeactivations(deps, states, result) - await reportStaleIntegrations(deps, states, result) + await reportDeactivations(deps, health, result) + await reportStaleness(deps, health, result) return result } diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts index f7df4f4103..e475f77c1e 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -1,11 +1,11 @@ import CronTime from 'cron-time-generator' import { IS_DEV_ENV, IS_PROD_ENV } from '@crowd/common' +import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' import { + getOctolensMappingDiscoveryHealth, listOctolensKeywordMappingsDeactivatedBetween, - listOctolensMappingDiscoveryStates, -} from '@crowd/data-access-layer' -import { WRITE_DB_CONFIG, getDbConnection } from '@crowd/data-access-layer/src/database' +} from '@crowd/data-access-layer/src/octolens/health' import { pgpQx } from '@crowd/data-access-layer/src/queryExecutor' import { REDIS_CONFIG, getRedisClient } from '@crowd/redis' import { SlackChannel, sendSlackNotificationAsync } from '@crowd/slack' @@ -27,7 +27,7 @@ const job: IJobDefinition = { const redis = await getRedisClient(REDIS_CONFIG()) const result = await runOctolensMappingHealthCheck({ - listStates: () => listOctolensMappingDiscoveryStates(qx), + getHealth: () => getOctolensMappingDiscoveryHealth(qx), listDeactivatedBetween: (since, until) => listOctolensKeywordMappingsDeactivatedBetween(qx, since, until), getKey: (key) => redis.get(key), @@ -48,7 +48,7 @@ const job: IJobDefinition = { } ctx.log.info( - `Octolens mapping health check done: integrations=${result.integrations}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, sendFailures=${result.sendFailures}`, + `Octolens mapping health check done: activeMappings=${result.activeMappings}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, sendFailures=${result.sendFailures}`, ) }, } diff --git a/services/libs/data-access-layer/src/octolens/health.test.ts b/services/libs/data-access-layer/src/octolens/health.test.ts new file mode 100644 index 0000000000..b862621daf --- /dev/null +++ b/services/libs/data-access-layer/src/octolens/health.test.ts @@ -0,0 +1,222 @@ +import { test as base, describe, expect } from 'vitest' + +import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' +import type { QueryExecutor } from '@crowd/database' +import { withQx } from '@crowd/test-kit/db' + +import { + getOctolensMappingDiscoveryHealth, + listOctolensKeywordMappingsDeactivatedBetween, +} from './health' +import { + findOrCreateOctolensAnchorIntegration, + reconcileOctolensKeywordMappings, + upsertOctolensKeywordMapping, +} from './index' + +const test = withQx(base) + +async function createSegment(qx: QueryExecutor, name: string): Promise { + const id = generateUUIDv1() + await qx.result( + ` + INSERT INTO public.segments (id, slug, name, "tenantId", "createdAt", "updatedAt") + VALUES ($(id), $(slug), $(name), $(tenantId), NOW(), NOW()) + `, + { id, slug: `${name}-slug`, name, tenantId: DEFAULT_TENANT_ID }, + ) + return id +} + +async function createIntegration( + qx: QueryExecutor, + options: { segmentId: string | null; platform?: string; deleted?: boolean }, +): Promise { + const id = generateUUIDv1() + await qx.result( + ` + INSERT INTO public.integrations + (id, platform, status, "segmentId", "tenantId", "createdAt", "updatedAt", "deletedAt") + VALUES ($(id), $(platform), 'done', $(segmentId), $(tenantId), NOW(), NOW(), + CASE WHEN $(deleted) THEN NOW() END) + `, + { + id, + platform: options.platform ?? 'octolens', + segmentId: options.segmentId, + tenantId: DEFAULT_TENANT_ID, + deleted: options.deleted ?? false, + }, + ) + return id +} + +async function createActiveSyncUnit(qx: QueryExecutor, integrationId: string): Promise { + await qx.result( + ` + INSERT INTO integration.sync_units + ("integrationId", platform, "channelId", "channelName", "syncName", status) + VALUES ($(integrationId), 'octolens', 'octolens', 'octolens', 'mentionPoll', 'active') + `, + { integrationId }, + ) +} + +async function setTimestamps( + qx: QueryExecutor, + keywordId: number, + timestamps: { lastSeenAt: string; deactivatedAt?: string }, +): Promise { + await qx.result( + ` + UPDATE integration."octolensKeywordMappings" + SET "lastSeenAt" = $(lastSeenAt), "deactivatedAt" = $(deactivatedAt) + WHERE "keywordId" = $(keywordId) + `, + { + keywordId, + lastSeenAt: timestamps.lastSeenAt, + deactivatedAt: timestamps.deactivatedAt ?? null, + }, + ) +} + +async function readDeactivatedAt(qx: QueryExecutor, keywordId: number): Promise { + const row: { deactivatedAt: string } = await qx.selectOne( + `SELECT "deactivatedAt" FROM integration."octolensKeywordMappings" WHERE "keywordId" = $(keywordId)`, + { keywordId }, + ) + return new Date(row.deactivatedAt) +} + +describe('getOctolensMappingDiscoveryHealth', () => { + test('reports nothing to judge without mappings, even with the anchor in place', async ({ + qx, + }) => { + const anchor = await findOrCreateOctolensAnchorIntegration(qx) + await createActiveSyncUnit(qx, anchor.id) + + const health = await getOctolensMappingDiscoveryHealth(qx) + + expect(health).toMatchObject({ activeMappings: 0, integrations: 0, lastDiscoveredAt: null }) + expect(health.checkedAt).toBeInstanceOf(Date) + }) + + test('takes the newest lastSeenAt across active mappings of every live subproject integration', async ({ + qx, + }) => { + const segmentA = await createSegment(qx, 'a') + const segmentB = await createSegment(qx, 'b') + await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: segmentA }, + { keywordId: 2, keyword: 'k2', segmentId: segmentA }, + { keywordId: 3, keyword: 'k3', segmentId: segmentB }, + ]) + await setTimestamps(qx, 1, { lastSeenAt: '2026-01-01T09:00:00Z' }) + await setTimestamps(qx, 2, { + lastSeenAt: '2026-01-01T12:00:00Z', + deactivatedAt: '2026-01-01T12:00:00Z', + }) + await setTimestamps(qx, 3, { lastSeenAt: '2026-01-01T10:00:00Z' }) + + const health = await getOctolensMappingDiscoveryHealth(qx) + + expect(health).toMatchObject({ activeMappings: 2, integrations: 2 }) + expect(health.lastDiscoveredAt?.toISOString()).toBe('2026-01-01T10:00:00.000Z') + }) + + test('ignores soft-deleted, sync-unit-owning and non-octolens integrations', async ({ qx }) => { + const segmentId = await createSegment(qx, 's') + const live = await createIntegration(qx, { segmentId }) + const deleted = await createIntegration(qx, { segmentId, deleted: true }) + const legacyOwner = await createIntegration(qx, { segmentId }) + await createActiveSyncUnit(qx, legacyOwner) + const github = await createIntegration(qx, { segmentId, platform: 'github' }) + const integrationByKeywordId = { 1: live, 2: deleted, 3: legacyOwner, 4: github } + for (const [keywordId, integrationId] of Object.entries(integrationByKeywordId)) { + await upsertOctolensKeywordMapping(qx, { + integrationId, + segmentId, + keywordId: Number(keywordId), + keyword: `k${keywordId}`, + }) + } + await setTimestamps(qx, 1, { lastSeenAt: '2026-01-01T09:00:00Z' }) + + const health = await getOctolensMappingDiscoveryHealth(qx) + + expect(health).toMatchObject({ activeMappings: 1, integrations: 1 }) + expect(health.lastDiscoveredAt?.toISOString()).toBe('2026-01-01T09:00:00.000Z') + }) +}) + +describe('listOctolensKeywordMappingsDeactivatedBetween', () => { + test('excludes since, includes until, and joins the project', async ({ qx }) => { + const segmentId = await createSegment(qx, 'project') + await reconcileOctolensKeywordMappings( + qx, + [1, 2, 3, 4, 5].map((keywordId) => ({ keywordId, keyword: `k${keywordId}`, segmentId })), + ) + await setTimestamps(qx, 1, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T10:00:00Z', + }) + await setTimestamps(qx, 2, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T10:30:00.000123Z', + }) + await setTimestamps(qx, 3, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T11:00:00Z', + }) + await setTimestamps(qx, 4, { + lastSeenAt: '2026-01-01T09:00:00Z', + deactivatedAt: '2026-01-01T11:00:00.000001Z', + }) + + const rows = await listOctolensKeywordMappingsDeactivatedBetween( + qx, + new Date('2026-01-01T10:00:00Z'), + new Date('2026-01-01T11:00:00Z'), + ) + + expect(rows.map((r) => r.keyword)).toEqual(['k2', 'k3']) + expect(rows[0]).toMatchObject({ + keywordId: 2, + segmentId, + segmentName: 'project', + segmentSlug: 'project-slug', + integrationDeleted: false, + }) + expect(rows[0].deactivatedAt.toISOString()).toBe('2026-01-01T10:30:00.000Z') + }) + + test('keeps mappings whose subproject integration discovery soft-deleted in the same run', async ({ + qx, + }) => { + const kept = await createSegment(qx, 'kept') + const emptied = await createSegment(qx, 'emptied') + await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: kept }, + { keywordId: 2, keyword: 'k2', segmentId: emptied }, + { keywordId: 3, keyword: 'k3', segmentId: emptied }, + ]) + const outcome = await reconcileOctolensKeywordMappings(qx, [ + { keywordId: 1, keyword: 'k1', segmentId: kept }, + ]) + expect(outcome.softDeleted).toHaveLength(1) + const deactivatedAt = await readDeactivatedAt(qx, 2) + + const rows = await listOctolensKeywordMappingsDeactivatedBetween( + qx, + new Date(deactivatedAt.getTime() - 1000), + new Date(deactivatedAt.getTime() + 1000), + ) + + expect(rows.map((r) => [r.keyword, r.segmentName, r.integrationDeleted])).toEqual([ + ['k2', 'emptied', true], + ['k3', 'emptied', true], + ]) + expect(rows[0].deactivatedAt.getTime()).toBe(rows[1].deactivatedAt.getTime()) + }) +}) diff --git a/services/libs/data-access-layer/src/octolens/health.ts b/services/libs/data-access-layer/src/octolens/health.ts new file mode 100644 index 0000000000..1a680495d8 --- /dev/null +++ b/services/libs/data-access-layer/src/octolens/health.ts @@ -0,0 +1,95 @@ +import { PlatformType } from '@crowd/types' + +import { QueryExecutor } from '../queryExecutor' + +const OCTOLENS_SUBPROJECT_INTEGRATION = ` + i.platform = $(platform) + AND i."segmentId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM integration.sync_units su WHERE su."integrationId" = i.id AND su.status = 'active' + ) +` + +export interface IOctolensMappingDiscoveryHealth { + activeMappings: number + integrations: number + lastDiscoveredAt: Date | null + checkedAt: Date +} + +// Discovery stamps every mapping it keeps in one transaction, so the newest lastSeenAt is its last +// complete run. The anchor has a null segment and no mappings, so it never counts. +export async function getOctolensMappingDiscoveryHealth( + qx: QueryExecutor, +): Promise { + const row: { + activeMappings: number + integrations: number + lastDiscoveredAt: string | null + checkedAt: string + } = await qx.selectOne( + ` + SELECT + count(*)::int AS "activeMappings", + count(DISTINCT m."integrationId")::int AS "integrations", + max(m."lastSeenAt") AS "lastDiscoveredAt", + now() AS "checkedAt" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" + WHERE m."deactivatedAt" IS NULL + AND i."deletedAt" IS NULL + AND ${OCTOLENS_SUBPROJECT_INTEGRATION} + `, + { platform: PlatformType.OCTOLENS }, + ) + return { + activeMappings: row.activeMappings, + integrations: row.integrations, + lastDiscoveredAt: row.lastDiscoveredAt ? new Date(row.lastDiscoveredAt) : null, + checkedAt: new Date(row.checkedAt), + } +} + +export interface IDeactivatedOctolensKeywordMapping { + integrationId: string + keywordId: number + keyword: string + segmentId: string + segmentName: string + segmentSlug: string + deactivatedAt: Date + integrationDeleted: boolean +} + +// Soft-deleted integrations stay in: discovery soft-deletes a subproject in the same run that +// deactivates its last mapping, and that is the deactivation most worth reporting. +export async function listOctolensKeywordMappingsDeactivatedBetween( + qx: QueryExecutor, + since: Date, + until: Date, +): Promise { + const rows: (Omit & { + deactivatedAt: string + })[] = await qx.select( + ` + SELECT + m."integrationId", + m."keywordId", + m.keyword, + m."segmentId", + s.name AS "segmentName", + s.slug AS "segmentSlug", + m."deactivatedAt", + i."deletedAt" IS NOT NULL AS "integrationDeleted" + FROM integration."octolensKeywordMappings" m + JOIN public.integrations i ON i.id = m."integrationId" + JOIN public.segments s ON s.id = m."segmentId" + WHERE m."deactivatedAt" > $(since) + AND m."deactivatedAt" <= $(until) + AND ${OCTOLENS_SUBPROJECT_INTEGRATION} + ORDER BY m."deactivatedAt", s.name, m.keyword, m."keywordId" + `, + { platform: PlatformType.OCTOLENS, since, until }, + ) + return rows.map((row) => ({ ...row, deactivatedAt: new Date(row.deactivatedAt) })) +} From 0b58493542983146dca52aad9bfc16fd3e742530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 16 Oct 2026 14:35:37 +0200 Subject: [PATCH 06/10] fix: isolate octolens health checks, bound alert size and reuse the subproject filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 297 ++++++++++-------- .../src/jobs/octolensMappingHealth.ts | 112 ++++--- .../jobs/octolensMappingHealthCheck.job.ts | 10 +- .../src/octolens/health.test.ts | 61 ++-- .../data-access-layer/src/octolens/health.ts | 16 +- .../data-access-layer/src/octolens/index.ts | 2 +- 6 files changed, 273 insertions(+), 225 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 5d4db72d5a..c2f59eed17 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -4,21 +4,26 @@ import type { IDeactivatedOctolensKeywordMapping, IOctolensMappingDiscoveryHealth, } from '@crowd/data-access-layer/src/octolens/health' +import type { Logger } from '@crowd/logging' import { SlackPersona } from '@crowd/slack' import { DEACTIVATIONS_CURSOR_KEY, IOctolensMappingHealthAlert, IOctolensMappingHealthDeps, + MAX_SECTION_CHARS, STALE_ALERTED_KEY, - groupIntoDiscoveryRuns, + capLines, + hasDeactivationBurst, isDeactivationBurst, runOctolensMappingHealthCheck, } from './octolensMappingHealth' const CHECKED_AT = new Date('2026-10-15T12:30:00.000Z') const UNTIL = '2026-10-15T12:25:00.000Z' +const FIRST_LOOKBACK = '2026-10-14T12:25:00.000Z' const RUN_AT = new Date('2026-10-15T12:00:00.000Z') +const STALE_AT = new Date('2026-10-15T08:00:00.000Z') function health( overrides: Partial = {}, @@ -50,7 +55,7 @@ function deactivation( } function setup(options: { - health: IOctolensMappingDiscoveryHealth + health: IOctolensMappingDiscoveryHealth | null deactivations?: IDeactivatedOctolensKeywordMapping[] store?: Record sendSucceeds?: boolean @@ -58,8 +63,10 @@ function setup(options: { const store = new Map(Object.entries(options.store ?? {})) const ttls = new Map() const sent: IOctolensMappingHealthAlert[] = [] + const log = { warn: vi.fn(), error: vi.fn() } const listDeactivatedBetween = vi.fn(async () => options.deactivations ?? []) const deps: IOctolensMappingHealthDeps = { + log: log as unknown as Logger, getHealth: async () => options.health, listDeactivatedBetween, getKey: async (key) => store.get(key) ?? null, @@ -72,7 +79,7 @@ function setup(options: { return options.sendSucceeds ?? true }, } - return { deps, store, ttls, sent, listDeactivatedBetween } + return { deps, store, ttls, sent, log, listDeactivatedBetween } } function alertText(alert: IOctolensMappingHealthAlert): string { @@ -81,9 +88,7 @@ function alertText(alert: IOctolensMappingHealthAlert): string { describe('runOctolensMappingHealthCheck', () => { it('stays quiet and keeps no state while there are no active mappings', async () => { - const { deps, store, sent, listDeactivatedBetween } = setup({ - health: health({ activeMappings: 0, integrations: 0, lastDiscoveredAt: null }), - }) + const { deps, store, sent, listDeactivatedBetween } = setup({ health: null }) await runOctolensMappingHealthCheck(deps) @@ -101,21 +106,19 @@ describe('runOctolensMappingHealthCheck', () => { expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) - it('looks back 24h on the first run and continues from the cursor afterwards', async () => { - const first = setup({ health: health() }) - await runOctolensMappingHealthCheck(first.deps) - expect(first.listDeactivatedBetween).toHaveBeenCalledWith( - new Date('2026-10-14T12:25:00.000Z'), - new Date(UNTIL), - ) - - const next = setup({ + it.each([ + { name: 'first run looks back 24h', cursor: undefined, since: FIRST_LOOKBACK }, + { name: 'later runs continue from the cursor', cursor: '2026-10-15T11:25:00.000Z' }, + ])('queries the window: $name', async ({ cursor, since }) => { + const { deps, listDeactivatedBetween } = setup({ health: health(), - store: { [DEACTIVATIONS_CURSOR_KEY]: '2026-10-15T11:25:00.000Z' }, + store: cursor ? { [DEACTIVATIONS_CURSOR_KEY]: cursor } : {}, }) - await runOctolensMappingHealthCheck(next.deps) - expect(next.listDeactivatedBetween).toHaveBeenCalledWith( - new Date('2026-10-15T11:25:00.000Z'), + + await runOctolensMappingHealthCheck(deps) + + expect(listDeactivatedBetween).toHaveBeenCalledWith( + new Date(since ?? (cursor as string)), new Date(UNTIL), ) }) @@ -131,60 +134,122 @@ describe('runOctolensMappingHealthCheck', () => { expect(listDeactivatedBetween).not.toHaveBeenCalled() }) - it('alerts on staleness once per episode, then again after the reminder expires', async () => { - const stale = health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }) - const { deps, store, ttls, sent } = setup({ health: stale }) + it.each([ + { name: 'first alert of an episode', lastDiscoveredAt: STALE_AT, alerted: undefined, sends: 1 }, + { + name: 'same episode already alerted', + lastDiscoveredAt: STALE_AT, + alerted: STALE_AT, + sends: 0, + }, + { + name: 'new episode after an older one', + lastDiscoveredAt: STALE_AT, + alerted: new Date('2026-10-14T08:00:00.000Z'), + sends: 1, + }, + { + name: 'within the threshold', + lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z'), + alerted: undefined, + sends: 0, + }, + ])('staleness: $name', async ({ lastDiscoveredAt, alerted, sends }) => { + const { deps, store, ttls, sent } = setup({ + health: health({ lastDiscoveredAt }), + store: alerted ? { [STALE_ALERTED_KEY]: alerted.toISOString() } : {}, + }) await runOctolensMappingHealthCheck(deps) + + expect(sent).toHaveLength(sends) + if (sends > 0) { + expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) + expect(store.get(STALE_ALERTED_KEY)).toBe(lastDiscoveredAt.toISOString()) + expect(ttls.get(STALE_ALERTED_KEY)).toBe(24 * 60 * 60) + } + }) + + it('describes the stale state and where to look', async () => { + const { deps, sent } = setup({ health: health({ lastDiscoveredAt: STALE_AT }) }) + await runOctolensMappingHealthCheck(deps) - expect(sent).toHaveLength(1) - expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) const text = alertText(sent[0]) expect(text).toContain('Last successful run: 2026-10-15 08:00 UTC (4h ago)') expect(text).toContain('Active mappings: 20 across 4 projects') expect(text).toContain('CROWD_OCTOLENS_API_TOKEN') - expect(store.get(STALE_ALERTED_KEY)).toBe('2026-10-15T08:00:00.000Z') - expect(ttls.get(STALE_ALERTED_KEY)).toBe(24 * 60 * 60) - - store.delete(STALE_ALERTED_KEY) - await runOctolensMappingHealthCheck(deps) - - expect(sent).toHaveLength(2) }) - it('alerts again when a new stale episode starts', async () => { - const { deps, sent } = setup({ - health: health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), - store: { [STALE_ALERTED_KEY]: '2026-10-14T08:00:00.000Z' }, + it('checks staleness before deactivations and runs both when one throws', async () => { + const { deps, sent, log, store } = setup({ + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [deactivation(1)], }) + const failing = { ...deps, listDeactivatedBetween: vi.fn().mockRejectedValue(new Error('db')) } - await runOctolensMappingHealthCheck(deps) + const result = await runOctolensMappingHealthCheck(failing) - expect(sent).toHaveLength(1) + expect(sent.map((a) => a.title)).toEqual(['Octolens mapping discovery looks stale']) + expect(result.failures).toBe(1) + expect(log.error).toHaveBeenCalledOnce() + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(FIRST_LOOKBACK) + + const staleThrows = { ...deps, getKey: vi.fn(deps.getKey) } + staleThrows.getKey.mockRejectedValueOnce(new Error('redis')) + await runOctolensMappingHealthCheck(staleThrows) + + expect(sent[sent.length - 1].title).toBe('Octolens keyword mappings deactivated') }) - it('does not treat discovery within the stale threshold as stale', async () => { - const { deps, sent } = setup({ - health: health({ lastDiscoveredAt: new Date('2026-10-15T09:30:00.000Z') }), - }) + it.each([ + { + name: 'stale alert stays unmarked', + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [], + store: {}, + key: STALE_ALERTED_KEY, + expected: undefined, + }, + { + name: 'cursor stays put', + health: health(), + deactivations: [deactivation(1)], + store: { [DEACTIVATIONS_CURSOR_KEY]: '2026-10-15T11:25:00.000Z' }, + key: DEACTIVATIONS_CURSOR_KEY, + expected: '2026-10-15T11:25:00.000Z', + }, + { + name: 'first lookback is pinned', + health: health(), + deactivations: [deactivation(1)], + store: {}, + key: DEACTIVATIONS_CURSOR_KEY, + expected: FIRST_LOOKBACK, + }, + ])('a failed send retries next run: $name', async (c) => { + const { deps, store, log } = setup({ ...c, sendSucceeds: false }) - await runOctolensMappingHealthCheck(deps) + const result = await runOctolensMappingHealthCheck(deps) - expect(sent).toEqual([]) + expect(result.failures).toBe(1) + expect(store.get(c.key)).toBe(c.expected) + expect(log.warn).toHaveBeenCalledOnce() }) - it('keeps alerting staleness while the Slack send fails', async () => { - const { deps, store, sent } = setup({ - health: health({ lastDiscoveredAt: new Date('2026-10-15T08:00:00.000Z') }), + it('logs the window start when the deactivation alert fails to send', async () => { + const { deps, log } = setup({ + health: health(), + deactivations: [deactivation(1)], sendSucceeds: false, }) - await runOctolensMappingHealthCheck(deps) await runOctolensMappingHealthCheck(deps) - expect(sent).toHaveLength(2) - expect(store.has(STALE_ALERTED_KEY)).toBe(false) + expect(log.warn).toHaveBeenCalledWith( + { windowStart: FIRST_LOOKBACK, deactivations: 1 }, + expect.any(String), + ) }) it('reports a burst in one run as a warning listing every keyword and project', async () => { @@ -208,135 +273,105 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent[0].persona).toBe(SlackPersona.WARNING_PROPAGATOR) expect(sent[0].title).toBe('Octolens keyword mappings deactivated in bulk') const text = alertText(sent[0]) + expect(text).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') expect(text).toContain('Deactivated mappings (3, 20 still active)') - expect(text).toContain('• ⚠️ 2026-10-15 12:00 UTC: 3 deactivated') expect(text).toContain( 'keyword-1 (1) → Project One (project-one), deactivated 2026-10-15 12:00 UTC', ) expect(text).toContain( 'keyword-3 (3) → Project Two (project-two), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', ) - expect(text).toContain('queue a manual backfill') expect(text).toContain('Nothing is re-fetched automatically') + expect(text).toContain('queue a manual backfill') expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) it('reports deactivations spread over runs as info', async () => { const { deps, sent } = setup({ health: health(), - deactivations: [ - deactivation(1, { deactivatedAt: new Date('2026-10-15T10:00:00.000Z') }), - deactivation(2, { deactivatedAt: new Date('2026-10-15T11:00:00.000Z') }), - deactivation(3, { deactivatedAt: new Date('2026-10-15T12:00:00.000Z') }), - ], + deactivations: [10, 11, 12].map((hour, i) => + deactivation(i + 1, { deactivatedAt: new Date(`2026-10-15T${hour}:00:00.000Z`) }), + ), }) await runOctolensMappingHealthCheck(deps) expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) - expect(alertText(sent[0])).toContain('Discovery runs (3)') - expect(alertText(sent[0])).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') - }) - - it('retries from the first lookback when the first send fails without a cursor', async () => { - const options = { - health: health(), - deactivations: [deactivation(1)], - sendSucceeds: false, - } - const { deps, store, listDeactivatedBetween } = setup(options) - - await runOctolensMappingHealthCheck(deps) - expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-14T12:25:00.000Z') - - options.health = health({ checkedAt: new Date('2026-10-15T14:30:00.000Z') }) - options.sendSucceeds = true - await runOctolensMappingHealthCheck(deps) - - expect(listDeactivatedBetween).toHaveBeenLastCalledWith( - new Date('2026-10-14T12:25:00.000Z'), - new Date('2026-10-15T14:25:00.000Z'), - ) - expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe('2026-10-15T14:25:00.000Z') }) - it('keeps the cursor when the deactivation alert fails so the next run retries', async () => { - const cursor = '2026-10-15T11:25:00.000Z' - const { deps, store, sent } = setup({ - health: health(), - deactivations: [deactivation(1)], - store: { [DEACTIVATIONS_CURSOR_KEY]: cursor }, - sendSucceeds: false, - }) - - const result = await runOctolensMappingHealthCheck(deps) - - expect(sent).toHaveLength(1) - expect(result.sendFailures).toBe(1) - expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(cursor) - }) - - it('caps listed mappings and runs but keeps the full counts', async () => { + it('caps the listed mappings by count and keeps the full count', async () => { const { deps, sent } = setup({ health: health(), - deactivations: Array.from({ length: 30 }, (_, i) => - deactivation(i + 1, { deactivatedAt: new Date(RUN_AT.getTime() - i * 60 * 60 * 1000) }), - ), + deactivations: Array.from({ length: 30 }, (_, i) => deactivation(i + 1)), }) await runOctolensMappingHealthCheck(deps) const text = alertText(sent[0]) expect(text).toContain('Deactivated mappings (30, 20 still active)') - expect(text).toContain('Discovery runs (30)') expect(text).toContain('keyword-25 (25)') expect(text).not.toContain('keyword-26 (26)') - expect(text).toContain('… and 5 more mappings') - expect(text).toContain('… and 20 more runs') + expect(text).toContain('… and 5 more') }) - it('truncates very long keyword and project names', async () => { + it('keeps every section under the Slack block size after escaping long names', async () => { const { deps, sent } = setup({ health: health(), - deactivations: [deactivation(1, { keyword: 'k'.repeat(500) })], + deactivations: Array.from({ length: 25 }, (_, i) => + deactivation(i + 1, { keyword: '&'.repeat(500), segmentName: '<'.repeat(500) }), + ), }) await runOctolensMappingHealthCheck(deps) - expect(alertText(sent[0])).toContain(`${'k'.repeat(99)}…`) - expect(alertText(sent[0])).not.toContain('k'.repeat(100)) + for (const section of sent[0].sections) { + expect(section.text.length).toBeLessThanOrEqual(MAX_SECTION_CHARS) + } + expect(JSON.stringify(sent[0]).length).toBeLessThan(20 * 1024) + expect(alertText(sent[0])).toMatch(/… and \d+ more$/) }) }) -describe('isDeactivationBurst', () => { - it('flags three or more deactivations regardless of size', () => { - expect(isDeactivationBurst(3, 1000)).toBe(true) +describe('capLines', () => { + it.each([ + { name: 'fits', lines: ['a', 'b'], maxLines: 5, maxChars: 100, expected: 'a\nb' }, + { + name: 'line cap', + lines: ['a', 'b', 'c'], + maxLines: 2, + maxChars: 100, + expected: 'a\nb\n… and 1 more', + }, + { + name: 'char cap', + lines: ['x'.repeat(30), 'y'.repeat(30)], + maxLines: 5, + maxChars: 60, + expected: `${'x'.repeat(30)}\n… and 1 more`, + }, + ])('$name', ({ lines, maxLines, maxChars, expected }) => { + expect(capLines(lines, maxLines, maxChars)).toBe(expected) }) +}) - it('flags a quarter or more of the mappings', () => { - expect(isDeactivationBurst(1, 3)).toBe(true) - expect(isDeactivationBurst(2, 6)).toBe(true) - expect(isDeactivationBurst(2, 7)).toBe(false) +describe('isDeactivationBurst', () => { + it.each([ + [3, 1000, true], + [1, 3, true], + [2, 6, true], + [2, 7, false], + ])('%i deactivated with %i still active is a burst: %s', (deactivated, active, expected) => { + expect(isDeactivationBurst(deactivated, active)).toBe(expected) }) }) -describe('groupIntoDiscoveryRuns', () => { - it('groups rows by their shared deactivation timestamp in time order', () => { - const later = new Date('2026-10-15T11:00:00.000Z') - const earlier = new Date('2026-10-15T10:00:00.000Z') - - const runs = groupIntoDiscoveryRuns( - [ - deactivation(1, { deactivatedAt: later }), - deactivation(2, { deactivatedAt: earlier }), - deactivation(3, { deactivatedAt: later }), - ], - 100, - ) +describe('hasDeactivationBurst', () => { + it('judges each run separately by its shared deactivation timestamp', () => { + const at = (hour: number) => new Date(`2026-10-15T${hour}:00:00.000Z`) + const spread = [10, 11, 12].map((hour, i) => deactivation(i + 1, { deactivatedAt: at(hour) })) + const oneRun = [1, 2, 3].map((id) => deactivation(id, { deactivatedAt: at(10) })) - expect(runs).toEqual([ - { at: earlier, deactivated: 1, burst: false }, - { at: later, deactivated: 2, burst: false }, - ]) + expect(hasDeactivationBurst(spread, 100)).toBe(false) + expect(hasDeactivationBurst(oneRun, 100)).toBe(true) }) }) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index b28ce4a022..cf412d0250 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -2,6 +2,7 @@ import type { IDeactivatedOctolensKeywordMapping, IOctolensMappingDiscoveryHealth, } from '@crowd/data-access-layer/src/octolens/health' +import { Logger } from '@crowd/logging' import { SlackMessageSection, SlackPersona, escapeSlackText } from '@crowd/slack' export const STALE_AFTER_HOURS = 3 @@ -11,8 +12,10 @@ export const BURST_MIN_SHARE = 0.25 export const CURSOR_SAFETY_MARGIN_MINUTES = 5 export const INITIAL_LOOKBACK_HOURS = 24 export const MAX_LISTED_KEYWORDS = 25 -export const MAX_LISTED_RUNS = 10 +// Under @crowd/slack's 2,900-char section block, so the list is never split or pruned. +export const MAX_SECTION_CHARS = 2800 const MAX_NAME_LENGTH = 100 +const OVERFLOW_NOTE_RESERVE = 20 export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' export const STALE_ALERTED_KEY = 'octolens-mapping-health:stale-alerted' @@ -27,7 +30,8 @@ export interface IOctolensMappingHealthAlert { } export interface IOctolensMappingHealthDeps { - getHealth: () => Promise + log: Logger + getHealth: () => Promise listDeactivatedBetween: ( since: Date, until: Date, @@ -41,13 +45,7 @@ export interface IOctolensMappingHealthResult { activeMappings: number staleAlerted: boolean deactivationsReported: number - sendFailures: number -} - -export interface IDiscoveryRun { - at: Date - deactivated: number - burst: boolean + failures: number } export function hoursSince(from: Date, now: Date): number { @@ -55,10 +53,7 @@ export function hoursSince(from: Date, now: Date): number { } export function isStale(health: IOctolensMappingDiscoveryHealth): boolean { - return ( - health.lastDiscoveredAt !== null && - hoursSince(health.lastDiscoveredAt, health.checkedAt) > STALE_AFTER_HOURS - ) + return hoursSince(health.lastDiscoveredAt, health.checkedAt) > STALE_AFTER_HOURS } export function isDeactivationBurst(deactivated: number, activeMappings: number): boolean { @@ -70,22 +65,16 @@ export function isDeactivationBurst(deactivated: number, activeMappings: number) } // One discovery run deactivates in one transaction, so its rows share one now() timestamp. -export function groupIntoDiscoveryRuns( +export function hasDeactivationBurst( deactivations: IDeactivatedOctolensKeywordMapping[], activeMappings: number, -): IDiscoveryRun[] { - const counts = new Map() +): boolean { + const perRun = new Map() for (const row of deactivations) { const at = row.deactivatedAt.getTime() - counts.set(at, (counts.get(at) ?? 0) + 1) + perRun.set(at, (perRun.get(at) ?? 0) + 1) } - return [...counts.entries()] - .sort(([a], [b]) => a - b) - .map(([at, deactivated]) => ({ - at: new Date(at), - deactivated, - burst: isDeactivationBurst(deactivated, activeMappings), - })) + return [...perRun.values()].some((count) => isDeactivationBurst(count, activeMappings)) } export function deactivationWindow( @@ -116,7 +105,6 @@ function formatName(name: string): string { export function buildStaleAlert( health: IOctolensMappingDiscoveryHealth, ): IOctolensMappingHealthAlert { - const lastDiscoveredAt = health.lastDiscoveredAt as Date return { persona: SlackPersona.WARNING_PROPAGATOR, title: 'Octolens mapping discovery looks stale', @@ -125,7 +113,7 @@ export function buildStaleAlert( title: '', text: [ `Keyword-to-project mapping discovery has not completed a run in over ${STALE_AFTER_HOURS} hours.`, - `Last successful run: ${formatUtc(lastDiscoveredAt)} (${Math.floor(hoursSince(lastDiscoveredAt, health.checkedAt))}h ago)`, + `Last successful run: ${formatUtc(health.lastDiscoveredAt)} (${Math.floor(hoursSince(health.lastDiscoveredAt, health.checkedAt))}h ago)`, `Active mappings: ${health.activeMappings} across ${health.integrations} projects`, 'Existing mappings keep routing mentions, but keywords added, remapped or removed in ' + 'Octolens are not picked up until discovery succeeds again. Check the ' + @@ -139,15 +127,19 @@ export function buildStaleAlert( // Capped because @crowd/slack silently prunes oversized messages while still reporting // success, which would move the cursor past deactivations nobody saw. -function capLines(lines: string[], max: number, noun: string): string[] { - if (lines.length <= max) { - return lines +export function capLines(lines: string[], maxLines: number, maxChars: number): string { + const budget = maxChars - OVERFLOW_NOTE_RESERVE + const kept: string[] = [] + let length = 0 + for (const line of lines) { + if (kept.length === maxLines || length + line.length + 1 > budget) { + break + } + kept.push(line) + length += line.length + 1 } - return [...lines.slice(0, max), `… and ${lines.length - max} more ${noun}`] -} - -function formatRun(run: IDiscoveryRun): string { - return `• ${run.burst ? '⚠️ ' : ''}${formatUtc(run.at)}: ${run.deactivated} deactivated` + const omitted = lines.length - kept.length + return [...kept, ...(omitted > 0 ? [`… and ${omitted} more`] : [])].join('\n') } function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { @@ -165,8 +157,7 @@ export function buildDeactivationAlert( since: Date, until: Date, ): IOctolensMappingHealthAlert { - const runs = groupIntoDiscoveryRuns(deactivations, activeMappings) - const burst = runs.some((run) => run.burst) + const burst = hasDeactivationBurst(deactivations, activeMappings) return { persona: burst ? SlackPersona.WARNING_PROPAGATOR : SlackPersona.INFO_NOTIFIER, @@ -183,14 +174,12 @@ export function buildDeactivationAlert( 'queue a manual backfill from its deactivation time to recover the mentions in between.', ].join('\n'), }, - { - title: `Discovery runs (${runs.length})`, - text: capLines(runs.map(formatRun), MAX_LISTED_RUNS, 'runs').join('\n'), - }, { title: `Deactivated mappings (${deactivations.length}, ${activeMappings} still active)`, - text: capLines(deactivations.map(formatDeactivation), MAX_LISTED_KEYWORDS, 'mappings').join( - '\n', + text: capLines( + deactivations.map(formatDeactivation), + MAX_LISTED_KEYWORDS, + MAX_SECTION_CHARS, ), }, ], @@ -218,7 +207,11 @@ async function reportDeactivations( if (deactivations.length > 0) { const alert = buildDeactivationAlert(deactivations, health.activeMappings, since, until) if (!(await deps.sendAlert(alert))) { - result.sendFailures++ + result.failures++ + deps.log.warn( + { windowStart: since.toISOString(), deactivations: deactivations.length }, + 'octolens deactivation alert failed to send, the next run retries from the same window start', + ) return } result.deactivationsReported = deactivations.length @@ -235,36 +228,51 @@ async function reportStaleness( if (!isStale(health)) { return } - const episode = (health.lastDiscoveredAt as Date).toISOString() + const episode = health.lastDiscoveredAt.toISOString() if ((await deps.getKey(STALE_ALERTED_KEY)) === episode) { return } if (!(await deps.sendAlert(buildStaleAlert(health)))) { - result.sendFailures++ + result.failures++ + deps.log.warn({ lastDiscoveredAt: episode }, 'octolens stale alert failed to send') return } await deps.setKey(STALE_ALERTED_KEY, episode, STALE_REMINDER_HOURS * 60 * 60) result.staleAlerted = true } -// No active mapping means discovery never completed a run (or the token was never set): a run -// that sees no mappings writes nothing, so it cannot empty a populated table. +async function runIsolated( + deps: IOctolensMappingHealthDeps, + result: IOctolensMappingHealthResult, + name: string, + check: () => Promise, +): Promise { + try { + await check() + } catch (err) { + result.failures++ + deps.log.error({ err }, `octolens mapping health: ${name} check failed`) + } +} + +// null means no active mapping, so discovery never completed a run (or the token was never set): +// a run that sees no mappings writes nothing, so it cannot empty a populated table. export async function runOctolensMappingHealthCheck( deps: IOctolensMappingHealthDeps, ): Promise { const health = await deps.getHealth() const result: IOctolensMappingHealthResult = { - activeMappings: health.activeMappings, + activeMappings: health?.activeMappings ?? 0, staleAlerted: false, deactivationsReported: 0, - sendFailures: 0, + failures: 0, } - if (health.activeMappings === 0) { + if (!health) { return result } - await reportDeactivations(deps, health, result) - await reportStaleness(deps, health, result) + await runIsolated(deps, result, 'staleness', () => reportStaleness(deps, health, result)) + await runIsolated(deps, result, 'deactivations', () => reportDeactivations(deps, health, result)) return result } diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts index e475f77c1e..c5812b9db1 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -21,12 +21,14 @@ const job: IJobDefinition = { process: async (ctx) => { ctx.log.info('Running octolens-mapping-health-check job...') - // Primary, not replica: replica lag beyond the cursor margin could skip deactivations. + // Primary, not replica: the cursor margin only covers the reconcile transaction's duration + // (deactivatedAt is its start time), so replica lag on top could skip deactivations. const dbConnection = await getDbConnection(WRITE_DB_CONFIG(), 3, 0) const qx = pgpQx(dbConnection) const redis = await getRedisClient(REDIS_CONFIG()) const result = await runOctolensMappingHealthCheck({ + log: ctx.log, getHealth: () => getOctolensMappingDiscoveryHealth(qx), listDeactivatedBetween: (since, until) => listOctolensKeywordMappingsDeactivatedBetween(qx, since, until), @@ -43,12 +45,8 @@ const job: IJobDefinition = { ), }) - if (result.sendFailures > 0) { - ctx.log.warn('octolens mapping health alert failed to send, will retry next run') - } - ctx.log.info( - `Octolens mapping health check done: activeMappings=${result.activeMappings}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, sendFailures=${result.sendFailures}`, + `Octolens mapping health check done: activeMappings=${result.activeMappings}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, failures=${result.failures}`, ) }, } diff --git a/services/libs/data-access-layer/src/octolens/health.test.ts b/services/libs/data-access-layer/src/octolens/health.test.ts index b862621daf..741046ef52 100644 --- a/services/libs/data-access-layer/src/octolens/health.test.ts +++ b/services/libs/data-access-layer/src/octolens/health.test.ts @@ -3,6 +3,8 @@ import { test as base, describe, expect } from 'vitest' import { DEFAULT_TENANT_ID, generateUUIDv1 } from '@crowd/common' import type { QueryExecutor } from '@crowd/database' import { withQx } from '@crowd/test-kit/db' +import { createSegments } from '@crowd/test-kit/factories' +import { PlatformType } from '@crowd/types' import { getOctolensMappingDiscoveryHealth, @@ -10,27 +12,33 @@ import { } from './health' import { findOrCreateOctolensAnchorIntegration, + findOrCreateOctolensSubprojectIntegration, reconcileOctolensKeywordMappings, upsertOctolensKeywordMapping, } from './index' const test = withQx(base) -async function createSegment(qx: QueryExecutor, name: string): Promise { - const id = generateUUIDv1() - await qx.result( - ` - INSERT INTO public.segments (id, slug, name, "tenantId", "createdAt", "updatedAt") - VALUES ($(id), $(slug), $(name), $(tenantId), NOW(), NOW()) - `, - { id, slug: `${name}-slug`, name, tenantId: DEFAULT_TENANT_ID }, - ) - return id +async function createSubproject(qx: QueryExecutor, name: string): Promise { + const { projectGroups } = await createSegments(qx, [ + { + name: `${name}-group`, + slug: `${name}-group`, + projects: [ + { + name: `${name}-project`, + slug: `${name}-project`, + subprojects: [{ name, slug: `${name}-slug` }], + }, + ], + }, + ]) + return projectGroups[0].projects[0].subprojects[0].id } async function createIntegration( qx: QueryExecutor, - options: { segmentId: string | null; platform?: string; deleted?: boolean }, + options: { segmentId: string | null; platform?: PlatformType; deleted?: boolean }, ): Promise { const id = generateUUIDv1() await qx.result( @@ -42,7 +50,7 @@ async function createIntegration( `, { id, - platform: options.platform ?? 'octolens', + platform: options.platform ?? PlatformType.OCTOLENS, segmentId: options.segmentId, tenantId: DEFAULT_TENANT_ID, deleted: options.deleted ?? false, @@ -56,9 +64,9 @@ async function createActiveSyncUnit(qx: QueryExecutor, integrationId: string): P ` INSERT INTO integration.sync_units ("integrationId", platform, "channelId", "channelName", "syncName", status) - VALUES ($(integrationId), 'octolens', 'octolens', 'octolens', 'mentionPoll', 'active') + VALUES ($(integrationId), $(platform), 'octolens', 'octolens', 'mentionPoll', 'active') `, - { integrationId }, + { integrationId, platform: PlatformType.OCTOLENS }, ) } @@ -96,17 +104,14 @@ describe('getOctolensMappingDiscoveryHealth', () => { const anchor = await findOrCreateOctolensAnchorIntegration(qx) await createActiveSyncUnit(qx, anchor.id) - const health = await getOctolensMappingDiscoveryHealth(qx) - - expect(health).toMatchObject({ activeMappings: 0, integrations: 0, lastDiscoveredAt: null }) - expect(health.checkedAt).toBeInstanceOf(Date) + expect(await getOctolensMappingDiscoveryHealth(qx)).toBeNull() }) test('takes the newest lastSeenAt across active mappings of every live subproject integration', async ({ qx, }) => { - const segmentA = await createSegment(qx, 'a') - const segmentB = await createSegment(qx, 'b') + const segmentA = await createSubproject(qx, 'a') + const segmentB = await createSubproject(qx, 'b') await reconcileOctolensKeywordMappings(qx, [ { keywordId: 1, keyword: 'k1', segmentId: segmentA }, { keywordId: 2, keyword: 'k2', segmentId: segmentA }, @@ -122,16 +127,16 @@ describe('getOctolensMappingDiscoveryHealth', () => { const health = await getOctolensMappingDiscoveryHealth(qx) expect(health).toMatchObject({ activeMappings: 2, integrations: 2 }) - expect(health.lastDiscoveredAt?.toISOString()).toBe('2026-01-01T10:00:00.000Z') + expect(health?.lastDiscoveredAt.toISOString()).toBe('2026-01-01T10:00:00.000Z') }) test('ignores soft-deleted, sync-unit-owning and non-octolens integrations', async ({ qx }) => { - const segmentId = await createSegment(qx, 's') - const live = await createIntegration(qx, { segmentId }) + const segmentId = await createSubproject(qx, 's') + const { id: live } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) const deleted = await createIntegration(qx, { segmentId, deleted: true }) const legacyOwner = await createIntegration(qx, { segmentId }) await createActiveSyncUnit(qx, legacyOwner) - const github = await createIntegration(qx, { segmentId, platform: 'github' }) + const github = await createIntegration(qx, { segmentId, platform: PlatformType.GITHUB }) const integrationByKeywordId = { 1: live, 2: deleted, 3: legacyOwner, 4: github } for (const [keywordId, integrationId] of Object.entries(integrationByKeywordId)) { await upsertOctolensKeywordMapping(qx, { @@ -146,13 +151,13 @@ describe('getOctolensMappingDiscoveryHealth', () => { const health = await getOctolensMappingDiscoveryHealth(qx) expect(health).toMatchObject({ activeMappings: 1, integrations: 1 }) - expect(health.lastDiscoveredAt?.toISOString()).toBe('2026-01-01T09:00:00.000Z') + expect(health?.lastDiscoveredAt.toISOString()).toBe('2026-01-01T09:00:00.000Z') }) }) describe('listOctolensKeywordMappingsDeactivatedBetween', () => { test('excludes since, includes until, and joins the project', async ({ qx }) => { - const segmentId = await createSegment(qx, 'project') + const segmentId = await createSubproject(qx, 'project') await reconcileOctolensKeywordMappings( qx, [1, 2, 3, 4, 5].map((keywordId) => ({ keywordId, keyword: `k${keywordId}`, segmentId })), @@ -194,8 +199,8 @@ describe('listOctolensKeywordMappingsDeactivatedBetween', () => { test('keeps mappings whose subproject integration discovery soft-deleted in the same run', async ({ qx, }) => { - const kept = await createSegment(qx, 'kept') - const emptied = await createSegment(qx, 'emptied') + const kept = await createSubproject(qx, 'kept') + const emptied = await createSubproject(qx, 'emptied') await reconcileOctolensKeywordMappings(qx, [ { keywordId: 1, keyword: 'k1', segmentId: kept }, { keywordId: 2, keyword: 'k2', segmentId: emptied }, diff --git a/services/libs/data-access-layer/src/octolens/health.ts b/services/libs/data-access-layer/src/octolens/health.ts index 1a680495d8..7690c0ff5a 100644 --- a/services/libs/data-access-layer/src/octolens/health.ts +++ b/services/libs/data-access-layer/src/octolens/health.ts @@ -1,27 +1,26 @@ import { PlatformType } from '@crowd/types' import { QueryExecutor } from '../queryExecutor' +import { OWNS_NO_ACTIVE_SYNC_UNITS } from './index' const OCTOLENS_SUBPROJECT_INTEGRATION = ` i.platform = $(platform) AND i."segmentId" IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM integration.sync_units su WHERE su."integrationId" = i.id AND su.status = 'active' - ) + AND ${OWNS_NO_ACTIVE_SYNC_UNITS} ` export interface IOctolensMappingDiscoveryHealth { activeMappings: number integrations: number - lastDiscoveredAt: Date | null + lastDiscoveredAt: Date checkedAt: Date } // Discovery stamps every mapping it keeps in one transaction, so the newest lastSeenAt is its last -// complete run. The anchor has a null segment and no mappings, so it never counts. +// complete run. export async function getOctolensMappingDiscoveryHealth( qx: QueryExecutor, -): Promise { +): Promise { const row: { activeMappings: number integrations: number @@ -42,10 +41,13 @@ export async function getOctolensMappingDiscoveryHealth( `, { platform: PlatformType.OCTOLENS }, ) + if (row.lastDiscoveredAt === null) { + return null + } return { activeMappings: row.activeMappings, integrations: row.integrations, - lastDiscoveredAt: row.lastDiscoveredAt ? new Date(row.lastDiscoveredAt) : null, + lastDiscoveredAt: new Date(row.lastDiscoveredAt), checkedAt: new Date(row.checkedAt), } } diff --git a/services/libs/data-access-layer/src/octolens/index.ts b/services/libs/data-access-layer/src/octolens/index.ts index 496d1a9284..3ee9080b03 100644 --- a/services/libs/data-access-layer/src/octolens/index.ts +++ b/services/libs/data-access-layer/src/octolens/index.ts @@ -56,7 +56,7 @@ export interface IOctolensKeywordMappingRef { } // An integration owning active sync units (the anchor, or a legacy one) is never a subproject integration. -const OWNS_NO_ACTIVE_SYNC_UNITS = `NOT EXISTS ( +export const OWNS_NO_ACTIVE_SYNC_UNITS = `NOT EXISTS ( SELECT 1 FROM integration.sync_units su WHERE su."integrationId" = i.id AND su.status = 'active' )` From 5b385ab1318b6e5457a12c2a7119e79470759748 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 16 Oct 2026 15:07:52 +0200 Subject: [PATCH 07/10] fix: give octolens health alerts accurate recovery steps and tell moved keywords apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 24 ++++++++++++-- .../src/jobs/octolensMappingHealth.ts | 31 ++++++++++++------- .../src/octolens/health.test.ts | 13 +++++--- .../data-access-layer/src/octolens/health.ts | 17 +++++++++- 4 files changed, 66 insertions(+), 19 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index c2f59eed17..d6ad824174 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -50,6 +50,7 @@ function deactivation( segmentSlug: 'project-one', deactivatedAt: RUN_AT, integrationDeleted: false, + nowMappedToSegmentName: null, ...overrides, } } @@ -176,9 +177,12 @@ describe('runOctolensMappingHealthCheck', () => { await runOctolensMappingHealthCheck(deps) const text = alertText(sent[0]) - expect(text).toContain('Last successful run: 2026-10-15 08:00 UTC (4h ago)') + expect(text).toContain('Last write: 2026-10-15 08:00 UTC (4h ago)') expect(text).toContain('Active mappings: 20 across 4 projects') expect(text).toContain('CROWD_OCTOLENS_API_TOKEN') + expect(text).toContain( + '`pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions -- --from 2026-10-15T08:00:00.000Z`', + ) }) it('checks staleness before deactivations and runs both when one throws', async () => { @@ -282,7 +286,7 @@ describe('runOctolensMappingHealthCheck', () => { 'keyword-3 (3) → Project Two (project-two), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', ) expect(text).toContain('Nothing is re-fetched automatically') - expect(text).toContain('queue a manual backfill') + expect(text).toContain('script:backfill-octolens-mentions -- --keywords KEYWORD_ID,KEYWORD_ID') expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) @@ -299,6 +303,22 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) }) + it('names the new project of a moved keyword and leaves moves out of the burst', async () => { + const { deps, sent } = setup({ + health: health(), + deactivations: [1, 2, 3].map((id) => + deactivation(id, { nowMappedToSegmentName: id === 1 ? 'Project Two' : null }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) + expect(alertText(sent[0])).toContain( + 'keyword-1 (1) → Project One (project-one), deactivated 2026-10-15 12:00 UTC, now mapped to Project Two', + ) + }) + it('caps the listed mappings by count and keeps the full count', async () => { const { deps, sent } = setup({ health: health(), diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index cf412d0250..beed616a77 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -20,6 +20,9 @@ const OVERFLOW_NOTE_RESERVE = 20 export const DEACTIVATIONS_CURSOR_KEY = 'octolens-mapping-health:deactivations-reported-until' export const STALE_ALERTED_KEY = 'octolens-mapping-health:stale-alerted' +const BACKFILL_COMMAND = + 'pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions --' + const HOUR_MS = 60 * 60 * 1000 const MINUTE_MS = 60 * 1000 @@ -65,12 +68,13 @@ export function isDeactivationBurst(deactivated: number, activeMappings: number) } // One discovery run deactivates in one transaction, so its rows share one now() timestamp. +// A keyword moved to another project is a deliberate change, not discovery losing it. export function hasDeactivationBurst( deactivations: IDeactivatedOctolensKeywordMapping[], activeMappings: number, ): boolean { const perRun = new Map() - for (const row of deactivations) { + for (const row of deactivations.filter((r) => r.nowMappedToSegmentName === null)) { const at = row.deactivatedAt.getTime() perRun.set(at, (perRun.get(at) ?? 0) + 1) } @@ -112,13 +116,15 @@ export function buildStaleAlert( { title: '', text: [ - `Keyword-to-project mapping discovery has not completed a run in over ${STALE_AFTER_HOURS} hours.`, - `Last successful run: ${formatUtc(health.lastDiscoveredAt)} (${Math.floor(hoursSince(health.lastDiscoveredAt, health.checkedAt))}h ago)`, + `Keyword-to-project mapping discovery has not written any mappings in over ${STALE_AFTER_HOURS} hours.`, + `Last write: ${formatUtc(health.lastDiscoveredAt)} (${Math.floor(hoursSince(health.lastDiscoveredAt, health.checkedAt))}h ago)`, `Active mappings: ${health.activeMappings} across ${health.integrations} projects`, - 'Existing mappings keep routing mentions, but keywords added, remapped or removed in ' + - 'Octolens are not picked up until discovery succeeds again. Check the ' + - '`connectors-octolens-discovery` schedule in connectors_worker and that ' + - '`CROWD_OCTOLENS_API_TOKEN` is set.', + 'Existing mappings keep routing mentions, but keywords added, moved or removed in ' + + 'Octolens are not picked up, and the mention poll skips mentions of new keywords.', + 'Check the `connectors-octolens-discovery` schedule and the connectors_worker logs: a ' + + 'failed run, an incomplete Octolens snapshot or a run that finds no mappings writes ' + + 'nothing, and so does a missing `CROWD_OCTOLENS_API_TOKEN`.', + `Once discovery writes again, recover the skipped mentions with \`${BACKFILL_COMMAND} --from ${health.lastDiscoveredAt.toISOString()}\``, ].join('\n'), }, ], @@ -147,7 +153,8 @@ function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { `• ${formatName(row.keyword)} (${row.keywordId}) → ` + `${formatName(row.segmentName)} (${formatName(row.segmentSlug)}), ` + `deactivated ${formatUtc(row.deactivatedAt)}` + - (row.integrationDeleted ? ', project has no Octolens keywords left' : '') + (row.integrationDeleted ? ', project has no Octolens keywords left' : '') + + (row.nowMappedToSegmentName ? `, now mapped to ${formatName(row.nowMappedToSegmentName)}` : '') ) } @@ -169,9 +176,11 @@ export function buildDeactivationAlert( title: '', text: [ `Window: ${formatUtc(since)} → ${formatUtc(until)}`, - 'Discovery no longer saw these keywords mapped to a project, so their mentions stop ' + - 'reaching it. Nothing is re-fetched automatically: if a keyword gets mapped again, ' + - 'queue a manual backfill from its deactivation time to recover the mentions in between.', + 'Discovery no longer saw these keywords mapped to the project, so their mentions stop ' + + 'reaching it. A keyword now mapped to another project goes there instead; any other ' + + 'keyword is skipped by the mention poll.', + 'Nothing is re-fetched automatically: once a skipped keyword is mapped again, re-fetch ' + + `its history with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`.`, ].join('\n'), }, { diff --git a/services/libs/data-access-layer/src/octolens/health.test.ts b/services/libs/data-access-layer/src/octolens/health.test.ts index 741046ef52..ab6bfd82dc 100644 --- a/services/libs/data-access-layer/src/octolens/health.test.ts +++ b/services/libs/data-access-layer/src/octolens/health.test.ts @@ -196,7 +196,7 @@ describe('listOctolensKeywordMappingsDeactivatedBetween', () => { expect(rows[0].deactivatedAt.toISOString()).toBe('2026-01-01T10:30:00.000Z') }) - test('keeps mappings whose subproject integration discovery soft-deleted in the same run', async ({ + test('keeps mappings of a subproject discovery soft-deleted in the same run and names moves', async ({ qx, }) => { const kept = await createSubproject(qx, 'kept') @@ -208,9 +208,10 @@ describe('listOctolensKeywordMappingsDeactivatedBetween', () => { ]) const outcome = await reconcileOctolensKeywordMappings(qx, [ { keywordId: 1, keyword: 'k1', segmentId: kept }, + { keywordId: 2, keyword: 'k2', segmentId: kept }, ]) expect(outcome.softDeleted).toHaveLength(1) - const deactivatedAt = await readDeactivatedAt(qx, 2) + const deactivatedAt = await readDeactivatedAt(qx, 3) const rows = await listOctolensKeywordMappingsDeactivatedBetween( qx, @@ -218,9 +219,11 @@ describe('listOctolensKeywordMappingsDeactivatedBetween', () => { new Date(deactivatedAt.getTime() + 1000), ) - expect(rows.map((r) => [r.keyword, r.segmentName, r.integrationDeleted])).toEqual([ - ['k2', 'emptied', true], - ['k3', 'emptied', true], + expect( + rows.map((r) => [r.keyword, r.segmentName, r.integrationDeleted, r.nowMappedToSegmentName]), + ).toEqual([ + ['k2', 'emptied', true, 'kept'], + ['k3', 'emptied', true, null], ]) expect(rows[0].deactivatedAt.getTime()).toBe(rows[1].deactivatedAt.getTime()) }) diff --git a/services/libs/data-access-layer/src/octolens/health.ts b/services/libs/data-access-layer/src/octolens/health.ts index 7690c0ff5a..105867d718 100644 --- a/services/libs/data-access-layer/src/octolens/health.ts +++ b/services/libs/data-access-layer/src/octolens/health.ts @@ -61,6 +61,7 @@ export interface IDeactivatedOctolensKeywordMapping { segmentSlug: string deactivatedAt: Date integrationDeleted: boolean + nowMappedToSegmentName: string | null } // Soft-deleted integrations stay in: discovery soft-deletes a subproject in the same run that @@ -82,10 +83,24 @@ export async function listOctolensKeywordMappingsDeactivatedBetween( s.name AS "segmentName", s.slug AS "segmentSlug", m."deactivatedAt", - i."deletedAt" IS NOT NULL AS "integrationDeleted" + i."deletedAt" IS NOT NULL AS "integrationDeleted", + moved.name AS "nowMappedToSegmentName" FROM integration."octolensKeywordMappings" m JOIN public.integrations i ON i.id = m."integrationId" JOIN public.segments s ON s.id = m."segmentId" + LEFT JOIN LATERAL ( + SELECT ms.name + FROM integration."octolensKeywordMappings" a + JOIN public.integrations ai ON ai.id = a."integrationId" + JOIN public.segments ms ON ms.id = a."segmentId" + WHERE a."keywordId" = m."keywordId" + AND a."deactivatedAt" IS NULL + AND ai."deletedAt" IS NULL + AND ai.platform = $(platform) + AND ai."segmentId" IS NOT NULL + ORDER BY ms.name + LIMIT 1 + ) moved ON true WHERE m."deactivatedAt" > $(since) AND m."deactivatedAt" <= $(until) AND ${OCTOLENS_SUBPROJECT_INTEGRATION} From c30e3c4042936642f877689397f8c15ce73ddb4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Fri, 16 Oct 2026 15:25:43 +0200 Subject: [PATCH 08/10] fix: ignore legacy sync-unit owners when naming a moved octolens keyword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 19 +++++- .../src/jobs/octolensMappingHealth.ts | 37 +++++------ .../jobs/octolensMappingHealthCheck.job.ts | 4 +- .../src/octolens/health.test.ts | 61 +++++++++++++++++++ .../data-access-layer/src/octolens/health.ts | 5 ++ 5 files changed, 104 insertions(+), 22 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index d6ad824174..08d0a5f576 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -180,12 +180,27 @@ describe('runOctolensMappingHealthCheck', () => { expect(text).toContain('Last write: 2026-10-15 08:00 UTC (4h ago)') expect(text).toContain('Active mappings: 20 across 4 projects') expect(text).toContain('CROWD_OCTOLENS_API_TOKEN') + expect(text).toContain('`--dry-run` first') expect(text).toContain( '`pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions -- --from 2026-10-15T08:00:00.000Z`', ) }) - it('checks staleness before deactivations and runs both when one throws', async () => { + it('sends the stale alert before the deactivation alert', async () => { + const { deps, sent } = setup({ + health: health({ lastDiscoveredAt: STALE_AT }), + deactivations: [deactivation(1)], + }) + + await runOctolensMappingHealthCheck(deps) + + expect(sent.map((a) => a.title)).toEqual([ + 'Octolens mapping discovery looks stale', + 'Octolens keyword mappings deactivated', + ]) + }) + + it('runs both checks when one throws', async () => { const { deps, sent, log, store } = setup({ health: health({ lastDiscoveredAt: STALE_AT }), deactivations: [deactivation(1)], @@ -231,7 +246,7 @@ describe('runOctolensMappingHealthCheck', () => { key: DEACTIVATIONS_CURSOR_KEY, expected: FIRST_LOOKBACK, }, - ])('a failed send retries next run: $name', async (c) => { + ])('a failed send leaves state for a retry: $name', async (c) => { const { deps, store, log } = setup({ ...c, sendSucceeds: false }) const result = await runOctolensMappingHealthCheck(deps) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index beed616a77..02a7093ba9 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -5,13 +5,13 @@ import type { import { Logger } from '@crowd/logging' import { SlackMessageSection, SlackPersona, escapeSlackText } from '@crowd/slack' -export const STALE_AFTER_HOURS = 3 -export const STALE_REMINDER_HOURS = 24 -export const BURST_MIN_DEACTIVATIONS = 3 -export const BURST_MIN_SHARE = 0.25 -export const CURSOR_SAFETY_MARGIN_MINUTES = 5 -export const INITIAL_LOOKBACK_HOURS = 24 -export const MAX_LISTED_KEYWORDS = 25 +const STALE_AFTER_HOURS = 3 +const STALE_REMINDER_HOURS = 24 +const BURST_MIN_DEACTIVATIONS = 3 +const BURST_MIN_SHARE = 0.25 +const CURSOR_SAFETY_MARGIN_MINUTES = 5 +const INITIAL_LOOKBACK_HOURS = 24 +const MAX_LISTED_KEYWORDS = 25 // Under @crowd/slack's 2,900-char section block, so the list is never split or pruned. export const MAX_SECTION_CHARS = 2800 const MAX_NAME_LENGTH = 100 @@ -51,11 +51,11 @@ export interface IOctolensMappingHealthResult { failures: number } -export function hoursSince(from: Date, now: Date): number { +function hoursSince(from: Date, now: Date): number { return (now.getTime() - from.getTime()) / HOUR_MS } -export function isStale(health: IOctolensMappingDiscoveryHealth): boolean { +function isStale(health: IOctolensMappingDiscoveryHealth): boolean { return hoursSince(health.lastDiscoveredAt, health.checkedAt) > STALE_AFTER_HOURS } @@ -81,7 +81,7 @@ export function hasDeactivationBurst( return [...perRun.values()].some((count) => isDeactivationBurst(count, activeMappings)) } -export function deactivationWindow( +function deactivationWindow( checkedAt: Date, cursor: string | null, ): { since: Date; until: Date; fromCursor: boolean } { @@ -102,13 +102,13 @@ function formatUtc(date: Date): string { } function formatName(name: string): string { - const short = name.length > MAX_NAME_LENGTH ? `${name.slice(0, MAX_NAME_LENGTH - 1)}…` : name + const chars = Array.from(name) + const short = + chars.length > MAX_NAME_LENGTH ? `${chars.slice(0, MAX_NAME_LENGTH - 1).join('')}…` : name return escapeSlackText(short) } -export function buildStaleAlert( - health: IOctolensMappingDiscoveryHealth, -): IOctolensMappingHealthAlert { +function buildStaleAlert(health: IOctolensMappingDiscoveryHealth): IOctolensMappingHealthAlert { return { persona: SlackPersona.WARNING_PROPAGATOR, title: 'Octolens mapping discovery looks stale', @@ -124,7 +124,9 @@ export function buildStaleAlert( 'Check the `connectors-octolens-discovery` schedule and the connectors_worker logs: a ' + 'failed run, an incomplete Octolens snapshot or a run that finds no mappings writes ' + 'nothing, and so does a missing `CROWD_OCTOLENS_API_TOKEN`.', - `Once discovery writes again, recover the skipped mentions with \`${BACKFILL_COMMAND} --from ${health.lastDiscoveredAt.toISOString()}\``, + `Once discovery writes again, recover the skipped mentions with \`${BACKFILL_COMMAND} --from ${health.lastDiscoveredAt.toISOString()}\`, ` + + 'running it with `--dry-run` first. If it refuses because the date is not earlier ' + + 'than the current start date, the poll already covers that range and nothing needs recovering.', ].join('\n'), }, ], @@ -158,7 +160,7 @@ function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { ) } -export function buildDeactivationAlert( +function buildDeactivationAlert( deactivations: IDeactivatedOctolensKeywordMapping[], activeMappings: number, since: Date, @@ -180,7 +182,8 @@ export function buildDeactivationAlert( 'reaching it. A keyword now mapped to another project goes there instead; any other ' + 'keyword is skipped by the mention poll.', 'Nothing is re-fetched automatically: once a skipped keyword is mapped again, re-fetch ' + - `its history with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`.`, + `its history with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`, running ` + + 'it with `--dry-run` first.', ].join('\n'), }, { diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts index c5812b9db1..beaf80e22f 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealthCheck.job.ts @@ -45,9 +45,7 @@ const job: IJobDefinition = { ), }) - ctx.log.info( - `Octolens mapping health check done: activeMappings=${result.activeMappings}, staleAlerted=${result.staleAlerted}, deactivationsReported=${result.deactivationsReported}, failures=${result.failures}`, - ) + ctx.log.info(result, 'Octolens mapping health check done') }, } diff --git a/services/libs/data-access-layer/src/octolens/health.test.ts b/services/libs/data-access-layer/src/octolens/health.test.ts index ab6bfd82dc..19b44f2520 100644 --- a/services/libs/data-access-layer/src/octolens/health.test.ts +++ b/services/libs/data-access-layer/src/octolens/health.test.ts @@ -89,6 +89,26 @@ async function setTimestamps( ) } +async function mapKeyword( + qx: QueryExecutor, + integrationId: string, + segmentId: string, + deactivatedAt: string | null = null, +): Promise { + await upsertOctolensKeywordMapping(qx, { integrationId, segmentId, keywordId: 7, keyword: 'k7' }) + await qx.result( + ` + UPDATE integration."octolensKeywordMappings" + SET "deactivatedAt" = $(deactivatedAt) + WHERE "integrationId" = $(integrationId) AND "keywordId" = 7 + `, + { integrationId, deactivatedAt }, + ) +} + +const IN_WINDOW = '2026-01-01T10:30:00Z' +const WINDOW: [Date, Date] = [new Date('2026-01-01T10:00:00Z'), new Date('2026-01-01T11:00:00Z')] + async function readDeactivatedAt(qx: QueryExecutor, keywordId: number): Promise { const row: { deactivatedAt: string } = await qx.selectOne( `SELECT "deactivatedAt" FROM integration."octolensKeywordMappings" WHERE "keywordId" = $(keywordId)`, @@ -227,4 +247,45 @@ describe('listOctolensKeywordMappingsDeactivatedBetween', () => { ]) expect(rows[0].deactivatedAt.getTime()).toBe(rows[1].deactivatedAt.getTime()) }) + + test('excludes the anchor and other platforms', async ({ qx }) => { + const segmentId = await createSubproject(qx, 'project') + const { id: subproject } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) + const anchor = await findOrCreateOctolensAnchorIntegration(qx) + const github = await createIntegration(qx, { segmentId, platform: PlatformType.GITHUB }) + for (const integrationId of [subproject, anchor.id, github]) { + await mapKeyword(qx, integrationId, segmentId, IN_WINDOW) + } + + const rows = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + + expect(rows.map((r) => r.integrationId)).toEqual([subproject]) + }) + + test('names a move only to a live subproject integration', async ({ qx }) => { + const segmentId = await createSubproject(qx, 'project') + const other = await createSubproject(qx, 'other') + const { id: subproject } = await findOrCreateOctolensSubprojectIntegration(qx, segmentId) + await mapKeyword(qx, subproject, segmentId, IN_WINDOW) + const legacyOwner = await createIntegration(qx, { segmentId: other }) + await createActiveSyncUnit(qx, legacyOwner) + const notSubprojects = [ + await createIntegration(qx, { segmentId: other, deleted: true }), + await createIntegration(qx, { segmentId: other, platform: PlatformType.GITHUB }), + legacyOwner, + ] + for (const integrationId of notSubprojects) { + await mapKeyword(qx, integrationId, other) + } + + const [ignored] = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + expect(ignored.nowMappedToSegmentName).toBeNull() + + const control = await createSubproject(qx, 'control') + const { id: controlIntegration } = await findOrCreateOctolensSubprojectIntegration(qx, control) + await mapKeyword(qx, controlIntegration, control) + + const [moved] = await listOctolensKeywordMappingsDeactivatedBetween(qx, ...WINDOW) + expect(moved.nowMappedToSegmentName).toBe('control') + }) }) diff --git a/services/libs/data-access-layer/src/octolens/health.ts b/services/libs/data-access-layer/src/octolens/health.ts index 105867d718..4edf178948 100644 --- a/services/libs/data-access-layer/src/octolens/health.ts +++ b/services/libs/data-access-layer/src/octolens/health.ts @@ -98,6 +98,11 @@ export async function listOctolensKeywordMappingsDeactivatedBetween( AND ai."deletedAt" IS NULL AND ai.platform = $(platform) AND ai."segmentId" IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM integration.sync_units asu + WHERE asu."integrationId" = ai.id AND asu.status = 'active' + ) ORDER BY ms.name LIMIT 1 ) moved ON true From 29e1410555cb45d5c77a7ceb5443842ac2eaf02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Sun, 18 Oct 2026 09:53:06 +0200 Subject: [PATCH 09/10] fix: list the newest octolens deactivations first and keep names from formatting slack text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../src/jobs/octolensMappingHealth.test.ts | 44 ++++++++++++++++--- .../src/jobs/octolensMappingHealth.ts | 21 ++++++--- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 08d0a5f576..09aa2d7fc8 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -14,6 +14,7 @@ import { MAX_SECTION_CHARS, STALE_ALERTED_KEY, capLines, + formatName, hasDeactivationBurst, isDeactivationBurst, runOctolensMappingHealthCheck, @@ -110,8 +111,9 @@ describe('runOctolensMappingHealthCheck', () => { it.each([ { name: 'first run looks back 24h', cursor: undefined, since: FIRST_LOOKBACK }, { name: 'later runs continue from the cursor', cursor: '2026-10-15T11:25:00.000Z' }, + { name: 'an unreadable cursor looks back 24h', cursor: 'not-a-date', since: FIRST_LOOKBACK }, ])('queries the window: $name', async ({ cursor, since }) => { - const { deps, listDeactivatedBetween } = setup({ + const { deps, store, listDeactivatedBetween } = setup({ health: health(), store: cursor ? { [DEACTIVATIONS_CURSOR_KEY]: cursor } : {}, }) @@ -122,6 +124,7 @@ describe('runOctolensMappingHealthCheck', () => { new Date(since ?? (cursor as string)), new Date(UNTIL), ) + expect(store.get(DEACTIVATIONS_CURSOR_KEY)).toBe(UNTIL) }) it('does not query when the cursor is already at the window end', async () => { @@ -295,10 +298,10 @@ describe('runOctolensMappingHealthCheck', () => { expect(text).toContain('Window: 2026-10-14 12:25 UTC → 2026-10-15 12:25 UTC') expect(text).toContain('Deactivated mappings (3, 20 still active)') expect(text).toContain( - 'keyword-1 (1) → Project One (project-one), deactivated 2026-10-15 12:00 UTC', + '`keyword-1` (1) → `Project One` (`project-one`), deactivated 2026-10-15 12:00 UTC', ) expect(text).toContain( - 'keyword-3 (3) → Project Two (project-two), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', + '`keyword-3` (3) → `Project Two` (`project-two`), deactivated 2026-10-15 12:00 UTC, project has no Octolens keywords left', ) expect(text).toContain('Nothing is re-fetched automatically') expect(text).toContain('script:backfill-octolens-mentions -- --keywords KEYWORD_ID,KEYWORD_ID') @@ -330,7 +333,7 @@ describe('runOctolensMappingHealthCheck', () => { expect(sent[0].persona).toBe(SlackPersona.INFO_NOTIFIER) expect(alertText(sent[0])).toContain( - 'keyword-1 (1) → Project One (project-one), deactivated 2026-10-15 12:00 UTC, now mapped to Project Two', + '`keyword-1` (1) → `Project One` (`project-one`), deactivated 2026-10-15 12:00 UTC, now mapped to `Project Two`', ) }) @@ -344,11 +347,28 @@ describe('runOctolensMappingHealthCheck', () => { const text = alertText(sent[0]) expect(text).toContain('Deactivated mappings (30, 20 still active)') - expect(text).toContain('keyword-25 (25)') - expect(text).not.toContain('keyword-26 (26)') + expect(text).toContain('`keyword-25` (25)') + expect(text).not.toContain('`keyword-26` (26)') expect(text).toContain('… and 5 more') }) + it('lists the newest deactivations first when the list is capped', async () => { + const at = (minute: number) => new Date(RUN_AT.getTime() + minute * 60 * 1000) + const { deps, sent } = setup({ + health: health(), + deactivations: Array.from({ length: 30 }, (_, i) => + deactivation(i + 1, { deactivatedAt: at(i) }), + ), + }) + + await runOctolensMappingHealthCheck(deps) + + const lines = sent[0].sections[1].text.split('\n') + expect(lines[0]).toContain('`keyword-30` (30)') + expect(lines[24]).toContain('`keyword-6` (6)') + expect(lines[25]).toBe('… and 5 more') + }) + it('keeps every section under the Slack block size after escaping long names', async () => { const { deps, sent } = setup({ health: health(), @@ -367,6 +387,18 @@ describe('runOctolensMappingHealthCheck', () => { }) }) +describe('formatName', () => { + it.each([ + { name: 'plain', input: 'kubernetes', expected: '`kubernetes`' }, + { name: 'mrkdwn markers', input: '*bold* _it_ ~st~', expected: '`*bold* _it_ ~st~`' }, + { name: 'backtick', input: 'a`b', expected: '`aˋb`' }, + { name: 'html entities', input: '', expected: '`<a&b>`' }, + { name: 'newlines', input: 'a\nb', expected: '`a b`' }, + ])('$name', ({ input, expected }) => { + expect(formatName(input)).toBe(expected) + }) +}) + describe('capLines', () => { it.each([ { name: 'fits', lines: ['a', 'b'], maxLines: 5, maxChars: 100, expected: 'a\nb' }, diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 02a7093ba9..5ae1fef861 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -101,11 +101,15 @@ function formatUtc(date: Date): string { return `${date.toISOString().slice(0, 16).replace('T', ' ')} UTC` } -function formatName(name: string): string { - const chars = Array.from(name) +// Names come from Octolens and project data. Inside a code span Slack applies no `*`, `_` or +// `~` formatting; a backtick would end the span, so it becomes a look-alike. +export function formatName(name: string): string { + const chars = Array.from(name.replace(/\s+/g, ' ')) const short = - chars.length > MAX_NAME_LENGTH ? `${chars.slice(0, MAX_NAME_LENGTH - 1).join('')}…` : name - return escapeSlackText(short) + chars.length > MAX_NAME_LENGTH + ? `${chars.slice(0, MAX_NAME_LENGTH - 1).join('')}…` + : chars.join('') + return `\`${escapeSlackText(short.replace(/`/g, 'ˋ'))}\`` } function buildStaleAlert(health: IOctolensMappingDiscoveryHealth): IOctolensMappingHealthAlert { @@ -160,6 +164,13 @@ function formatDeactivation(row: IDeactivatedOctolensKeywordMapping): string { ) } +// Stable, so rows of one run keep the query's project and keyword order. +function newestFirst( + deactivations: IDeactivatedOctolensKeywordMapping[], +): IDeactivatedOctolensKeywordMapping[] { + return [...deactivations].sort((a, b) => b.deactivatedAt.getTime() - a.deactivatedAt.getTime()) +} + function buildDeactivationAlert( deactivations: IDeactivatedOctolensKeywordMapping[], activeMappings: number, @@ -189,7 +200,7 @@ function buildDeactivationAlert( { title: `Deactivated mappings (${deactivations.length}, ${activeMappings} still active)`, text: capLines( - deactivations.map(formatDeactivation), + newestFirst(deactivations).map(formatDeactivation), MAX_LISTED_KEYWORDS, MAX_SECTION_CHARS, ), From 0ed0c15518f6b9985c643ab8ca90afc3fa0ee419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uro=C5=A1=20Marolt?= Date: Sun, 18 Oct 2026 14:12:09 +0200 Subject: [PATCH 10/10] fix: point a refused octolens rewind at the keyword backfill instead of calling it covered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs linuxfoundation/lfx-cdp-insights-ops#806 Signed-off-by: Uroš Marolt --- .../apps/cron_service/src/jobs/octolensMappingHealth.test.ts | 2 ++ services/apps/cron_service/src/jobs/octolensMappingHealth.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts index 09aa2d7fc8..06d2b92764 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.test.ts @@ -187,6 +187,8 @@ describe('runOctolensMappingHealthCheck', () => { expect(text).toContain( '`pnpm --filter @crowd/connectors-worker run script:backfill-octolens-mentions -- --from 2026-10-15T08:00:00.000Z`', ) + expect(text).toContain('script:backfill-octolens-mentions -- --keywords KEYWORD_ID,KEYWORD_ID') + expect(text).not.toContain('nothing needs recovering') }) it('sends the stale alert before the deactivation alert', async () => { diff --git a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts index 5ae1fef861..ce4b497802 100644 --- a/services/apps/cron_service/src/jobs/octolensMappingHealth.ts +++ b/services/apps/cron_service/src/jobs/octolensMappingHealth.ts @@ -130,7 +130,8 @@ function buildStaleAlert(health: IOctolensMappingDiscoveryHealth): IOctolensMapp 'nothing, and so does a missing `CROWD_OCTOLENS_API_TOKEN`.', `Once discovery writes again, recover the skipped mentions with \`${BACKFILL_COMMAND} --from ${health.lastDiscoveredAt.toISOString()}\`, ` + 'running it with `--dry-run` first. If it refuses because the date is not earlier ' + - 'than the current start date, the poll already covers that range and nothing needs recovering.', + 'than the current start date, the poll may already have read past some skipped mentions, ' + + `so re-fetch the keywords mapped since then with \`${BACKFILL_COMMAND} --keywords KEYWORD_ID,KEYWORD_ID\`.`, ].join('\n'), }, ],