fix(history): keep the published adjusted close within Yahoo float noise

scripts/update-data.test.ts

@@ -6,7 +6,7 @@ import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import {
   CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL,
-  annualizedToTotal, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa,
+  annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa,
   isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange,
   parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions,
   parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls,
@@ -470,6 +470,30 @@ describe('pipeline', () => {
     expect(s.meta('AAA').yields.dividendYieldBasis).toBe(row.metrics.dividendYieldBasis);
   });
 
+  test('a rerun whose live catalog is unreachable keeps the published category path and net assets source', async () => {
+    // Live DGRW: one run read the fund list, the next one fell back to the previous index and published
+    // "Domestic Equity" instead of "Domestic Equity / Large Cap Core" and an N-PORT net assets label
+    const s = new Scenario(['AAA']);
+    s.nportAsOf = '2026-06-30';
+    await s.run({ EDGAR_FALLBACK: 'true' });
+    const first = s.meta('AAA');
+    expect([first.categoryPath, first.aum.source]).toEqual(['Equity / Core', 'WisdomTree product table Assets Under Mgmt $(000)']);
+    const live = s.handler;
+    s.handler = (url, init) => url.includes('dataspanapi.wisdomtree.com') || url.endsWith('/us/products') ? new Response('unavailable', { status: 503 }) : live(url, init);
+    await s.run({ EDGAR_FALLBACK: 'true' });
+    const second = s.meta('AAA');
+    expect([second.categoryPath, second.aum.source, second.aum.value]).toEqual([first.categoryPath, first.aum.source, first.aum.value]);
+  });
+
+  test('aumSource: live catalog label, published label for a retained value, N-PORT only when the value is not retained', () => {
+    const nport = { netAssets: 1, repPdDate: '2026-06-30' };
+    const published = { aum: { source: 'WisdomTree product table Assets Under Mgmt $(000)' } };
+    expect(aumSource({ source: 'wisdomtree', netAssets: 5 }, nport, null)).toBe('WisdomTree product table Assets Under Mgmt $(000)');
+    expect(aumSource({ source: 'previous index', netAssets: 5 }, nport, published)).toBe(published.aum.source);
+    expect(aumSource({ source: 'previous index', netAssets: null }, nport, published)).toBe('SEC Form N-PORT-P net assets (2026-06-30)');
+    expect(aumSource({ source: 'previous index', netAssets: null }, null, null)).toBe('previous run');
+  });
+
   test('a one-ticker run keeps every row: unreadable catalog, a catalog listing only that fund, an unfiltered failing run, MAX_FETCHES', async () => {
     const published = async (tickers: string[], metaOnly: string[]): Promise<Scenario> => {
       const s = new Scenario([...tickers, ...metaOnly]);
@@ -523,6 +547,33 @@ describe('pipeline', () => {
     expect(s.files().some((path) => path.includes('.tmp-'))).toBe(false);
   });
 
+  test('two runs on the same source data write zero changes even when Yahoo adjusted closes jitter across a cent boundary', async () => {
+    // Yahoo float32 noise seen live: AAM SPDV 2024-11-12 came back as 31.544998 and 31.545001 a minute apart
+    const s = new Scenario(['AAA']);
+    const base = s.handler;
+    const jittered = (shift: number) => (url: string, init?: any): Response | Promise<Response> => {
+      if (!url.includes('finance.yahoo.com')) return base(url, init);
+      const payload = chartPayload();
+      const result = payload.chart.result[0];
+      result.indicators.adjclose = [{ adjclose: result.indicators.quote[0].close.map((_: number, i: number) => 90.005 + i * 0.01 + shift) }];
+      return new Response(JSON.stringify(payload));
+    };
+    const adj = () => s.files().filter((path) => path.includes('/history/')).flatMap((path) => JSON.parse(readFileSync(path, 'utf8')).rows.map((row: any) => row['Adj Close']));
+    s.handler = jittered(0.000001);
+    await s.run();
+    expect(adj().slice(0, 2)).toEqual(['90.01', '90.02']);
+    s.backdate();
+    const before = s.files().map((path) => readFileSync(path, 'utf8'));
+    s.handler = jittered(-0.000002);
+    await s.run();
+    expect(s.touched()).toEqual([]);
+    expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before);
+    // a real change beyond the noise is still written
+    s.handler = jittered(-0.015);
+    await s.run();
+    expect(adj().slice(0, 2)).toEqual(['89.99', '90']);
+  });
+
   test('a failed product page keeps the previous official data as one unit (returns, quarter end, distributions, NAV, price, premium)', async () => {
     const s = new Scenario();
     await s.run();

scripts/update-data.ts

@@ -1489,13 +1489,24 @@ export function deriveMetrics(derived: PriceReturns, fund: CatalogFund, dividend
   };
 }
 
-function historyRows(days: ChartDay[]): JsonRecord[] {
+// Yahoo's adjusted close is float32 and its last digits move with the request's period2, so a value
+// sitting on a cent boundary (31.544998 vs 31.545001) flips between runs. Within that noise the
+// published cent is kept: a rerun with the same source data changes nothing.
+export const ADJ_CLOSE_NOISE = 1e-4;
+export function keepPublishedAdjClose(adjClose: number, published: unknown): string {
+  const fresh = String(round(adjClose, 2));
+  const prior = numberOrNull(published);
+  return prior !== null && fresh !== String(published) && Math.abs(adjClose - prior) <= 0.005 + ADJ_CLOSE_NOISE ? String(published) : fresh;
+}
+
+export function historyRows(days: ChartDay[], published: JsonRecord[] = []): JsonRecord[] {
   // Yahoo recomputes the split/dividend-adjusted close on every request; storing
   // the raw float lets the last digit or two jitter between otherwise identical
   // requests, making every history row (and the fund) look "updated" on every
   // single run. Rounding to 2 decimals is well past any meaningful price
-  // precision and absorbs that jitter.
-  return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': String(round(day.adjClose, 2)), Volume: String(day.volume) }));
+  // precision and absorbs that jitter; keepPublishedAdjClose absorbs the cent-boundary flips.
+  const publishedAdj = new Map(published.map((row) => [toIsoDate(row.Date), row['Adj Close']]));
+  return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': keepPublishedAdjClose(day.adjClose, publishedAdj.get(day.date)), Volume: String(day.volume) }));
 }
 
 function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null): PriceReturns {
@@ -1740,6 +1751,17 @@ function distributionRowsFromTable(rows: unknown): DistributionRow[] {
 
 const isOfficialSource = (value: unknown): boolean => /^WisdomTree (official|product-page)/.test(String(value ?? ''));
 
+/**
+ * Where the published net assets came from. When this run's catalog row is the previous index (the live catalog
+ * was unreachable or did not list the fund), the value is the published one, so its published source label is kept
+ * too: a rerun with the same upstream data must not relabel it (product table -> N-PORT) and flip the fund to updated.
+ */
+export function aumSource(fund: Pick<CatalogFund, 'source' | 'netAssets'>, nport: { netAssets?: number | null; repPdDate?: string | null } | null | undefined, previousMeta: JsonRecord | null): string {
+  if (fund.source === 'wisdomtree') return 'WisdomTree product table Assets Under Mgmt $(000)';
+  if (fund.netAssets !== null && previousMeta?.aum?.source) return String(previousMeta.aum.source);
+  return nport ? `SEC Form N-PORT-P net assets (${nport.repPdDate || 'n/a'})` : 'previous run';
+}
+
 async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: JsonRecord = {}): Promise<JsonRecord> {
   const reasons = catalogFilterReasons(fund, config);
   if (reasons.length) {
@@ -1848,8 +1870,8 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
       outputNote(`[ ${'chart'.padEnd(9)}] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`);
     }
   }
+  const previousRows = await readPreviousSheet(fund.ticker, 'history');
   if (!days.length) {
-    const previousRows = await readPreviousSheet(fund.ticker, 'history');
     days = previousRows.map((row) => ({ date: toIsoDate(row.Date), close: numberOrNull(row.Close) || 0, adjClose: numberOrNull(row['Adj Close']) || numberOrNull(row.Close) || 0, volume: numberOrNull(row.Volume) || 0 })).filter((row) => row.date && row.close > 0);
     if (previousRows.length) historySource = previousMeta?.history?.source || 'previous run';
   }
@@ -1887,7 +1909,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
     return { __skipped: true, ticker: fund.ticker, __skipReasons: returnFilterReasons };
   }
 
-  const history = historyRows(days);
+  const history = historyRows(days, previousRows);
   const historyAsOf = derived.asOfDate || previousMeta?.history?.asOf || null;
   const holdingPages = await writePages(fundDir, fund.ticker, 'holdings', holdingsHeaders, holdingsRows, config.holdingsPageSize, holdingsAsOf, holdingsSource);
   const historyPages = await writePages(fundDir, fund.ticker, 'history', historyHeaders(), history, config.historyPageSize, historyAsOf, historySource);
@@ -1962,7 +1984,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
     ticker: fund.ticker,
     name: fund.name,
     category: fund.category,
-    categoryPath: fund.categoryPath,
+    categoryPath: fund.source === 'wisdomtree' ? fund.categoryPath : String(previousMeta?.categoryPath || fund.categoryPath),
     source: {
       fundPage: fund.fundPage,
       officialProductPage: productProxyUrl(fund.fundPage),
@@ -1981,7 +2003,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J
     nav: { display: nav === null ? '—' : `$${nav.toFixed(2)}`, value: nav, asOfDate: navAsOfLabel },
     marketPrice: { display: marketPrice === null ? '—' : `$${marketPrice.toFixed(2)}`, value: marketPrice, asOfDate: marketPriceAsOfLabel },
     premiumDiscount: { display: premiumDiscount === null ? '—' : `${premiumDiscount.toFixed(2)}%`, value: premiumDiscount },
-    aum: { display: formatAumDisplay(netAssets), value: netAssets, asOfDate: asOfDate ? formatDate(asOfDate) : (nport?.repPdDate ? formatDate(nport.repPdDate) : '—'), source: fund.source === 'wisdomtree' ? 'WisdomTree product table Assets Under Mgmt $(000)' : nport ? `SEC Form N-PORT-P net assets (${nport.repPdDate || 'n/a'})` : 'previous run' },
+    aum: { display: formatAumDisplay(netAssets), value: netAssets, asOfDate: asOfDate ? formatDate(asOfDate) : (nport?.repPdDate ? formatDate(nport.repPdDate) : '—'), source: aumSource(fund, nport, previousMeta) },
     yields: { dividendYield: metrics.dividendYield, dividendYieldText: metrics.dividendYieldText, dividendYieldBasis: metrics.dividendYieldBasis, dividendYieldKind: metrics.dividendYieldBasis === 'official-trailing-12m' ? YIELD_KIND_OFFICIAL : YIELD_KIND_INDICATED, distributionRate: productSummary?.distributionYield ?? numberOrNull(previousMeta?.yields?.distributionRate), secYield: metrics.secYield, secYieldText: metrics.secYieldText, secYieldKind: metrics.secYield !== null ? '30-day SEC yield published on the official WisdomTree product page' : 'not present in the current official product-page rendering' },
     returns,
     distributions: { frequency: distributionFrequency, paymentsPerYear: distributionPaymentsPerYear, source: distributionsSource, headers: distributionHeaders, rows: distributionTable },