diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index b36b306..f907881 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -6,7 +6,7 @@ import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { CONTROL_NAMES, HISTORY_RANGE_PATTERN, RETURNS_BASIS_DERIVED, RETURNS_BASIS_OFFICIAL, - annualizedToTotal, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa, + annualizedToTotal, aumSource, cleanHoldingTicker, deriveMetrics, dividendYieldBasisFromKind, formatDate, hasConfiguredFilters, inferDistributionFrequency, installSystemCa, isCertError, isinFromCusip, looksLikeProductPage, mergeDistributionRecords, normalizeHoldingName, nportUrlFor, numberOrNull, parseAumRange, parseCatalogMarkdown, parseChart, parseFundListJson, parseEdgarAtomFilings, parseFundTickerMap, parseNport, parseOfficialDistributions, parseProductPageSummary, parseRange, parseRanges, performanceAsOfDate, priceReturns, readConfig, readKnownRows, resolveControls, @@ -470,6 +470,30 @@ describe('pipeline', () => { expect(s.meta('AAA').yields.dividendYieldBasis).toBe(row.metrics.dividendYieldBasis); }); + test('a rerun whose live catalog is unreachable keeps the published category path and net assets source', async () => { + // Live DGRW: one run read the fund list, the next one fell back to the previous index and published + // "Domestic Equity" instead of "Domestic Equity / Large Cap Core" and an N-PORT net assets label + const s = new Scenario(['AAA']); + s.nportAsOf = '2026-06-30'; + await s.run({ EDGAR_FALLBACK: 'true' }); + const first = s.meta('AAA'); + expect([first.categoryPath, first.aum.source]).toEqual(['Equity / Core', 'WisdomTree product table Assets Under Mgmt $(000)']); + const live = s.handler; + s.handler = (url, init) => url.includes('dataspanapi.wisdomtree.com') || url.endsWith('/us/products') ? new Response('unavailable', { status: 503 }) : live(url, init); + await s.run({ EDGAR_FALLBACK: 'true' }); + const second = s.meta('AAA'); + expect([second.categoryPath, second.aum.source, second.aum.value]).toEqual([first.categoryPath, first.aum.source, first.aum.value]); + }); + + test('aumSource: live catalog label, published label for a retained value, N-PORT only when the value is not retained', () => { + const nport = { netAssets: 1, repPdDate: '2026-06-30' }; + const published = { aum: { source: 'WisdomTree product table Assets Under Mgmt $(000)' } }; + expect(aumSource({ source: 'wisdomtree', netAssets: 5 }, nport, null)).toBe('WisdomTree product table Assets Under Mgmt $(000)'); + expect(aumSource({ source: 'previous index', netAssets: 5 }, nport, published)).toBe(published.aum.source); + expect(aumSource({ source: 'previous index', netAssets: null }, nport, published)).toBe('SEC Form N-PORT-P net assets (2026-06-30)'); + expect(aumSource({ source: 'previous index', netAssets: null }, null, null)).toBe('previous run'); + }); + test('a one-ticker run keeps every row: unreadable catalog, a catalog listing only that fund, an unfiltered failing run, MAX_FETCHES', async () => { const published = async (tickers: string[], metaOnly: string[]): Promise => { const s = new Scenario([...tickers, ...metaOnly]); @@ -523,6 +547,33 @@ describe('pipeline', () => { expect(s.files().some((path) => path.includes('.tmp-'))).toBe(false); }); + test('two runs on the same source data write zero changes even when Yahoo adjusted closes jitter across a cent boundary', async () => { + // Yahoo float32 noise seen live: AAM SPDV 2024-11-12 came back as 31.544998 and 31.545001 a minute apart + const s = new Scenario(['AAA']); + const base = s.handler; + const jittered = (shift: number) => (url: string, init?: any): Response | Promise => { + if (!url.includes('finance.yahoo.com')) return base(url, init); + const payload = chartPayload(); + const result = payload.chart.result[0]; + result.indicators.adjclose = [{ adjclose: result.indicators.quote[0].close.map((_: number, i: number) => 90.005 + i * 0.01 + shift) }]; + return new Response(JSON.stringify(payload)); + }; + const adj = () => s.files().filter((path) => path.includes('/history/')).flatMap((path) => JSON.parse(readFileSync(path, 'utf8')).rows.map((row: any) => row['Adj Close'])); + s.handler = jittered(0.000001); + await s.run(); + expect(adj().slice(0, 2)).toEqual(['90.01', '90.02']); + s.backdate(); + const before = s.files().map((path) => readFileSync(path, 'utf8')); + s.handler = jittered(-0.000002); + await s.run(); + expect(s.touched()).toEqual([]); + expect(s.files().map((path) => readFileSync(path, 'utf8'))).toEqual(before); + // a real change beyond the noise is still written + s.handler = jittered(-0.015); + await s.run(); + expect(adj().slice(0, 2)).toEqual(['89.99', '90']); + }); + test('a failed product page keeps the previous official data as one unit (returns, quarter end, distributions, NAV, price, premium)', async () => { const s = new Scenario(); await s.run(); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index 43c5a34..7433379 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -1489,13 +1489,24 @@ export function deriveMetrics(derived: PriceReturns, fund: CatalogFund, dividend }; } -function historyRows(days: ChartDay[]): JsonRecord[] { +// Yahoo's adjusted close is float32 and its last digits move with the request's period2, so a value +// sitting on a cent boundary (31.544998 vs 31.545001) flips between runs. Within that noise the +// published cent is kept: a rerun with the same source data changes nothing. +export const ADJ_CLOSE_NOISE = 1e-4; +export function keepPublishedAdjClose(adjClose: number, published: unknown): string { + const fresh = String(round(adjClose, 2)); + const prior = numberOrNull(published); + return prior !== null && fresh !== String(published) && Math.abs(adjClose - prior) <= 0.005 + ADJ_CLOSE_NOISE ? String(published) : fresh; +} + +export function historyRows(days: ChartDay[], published: JsonRecord[] = []): JsonRecord[] { // Yahoo recomputes the split/dividend-adjusted close on every request; storing // the raw float lets the last digit or two jitter between otherwise identical // requests, making every history row (and the fund) look "updated" on every // single run. Rounding to 2 decimals is well past any meaningful price - // precision and absorbs that jitter. - return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': String(round(day.adjClose, 2)), Volume: String(day.volume) })); + // precision and absorbs that jitter; keepPublishedAdjClose absorbs the cent-boundary flips. + const publishedAdj = new Map(published.map((row) => [toIsoDate(row.Date), row['Adj Close']])); + return days.map((day) => ({ Date: formatDate(day.date), Close: String(day.close), 'Adj Close': keepPublishedAdjClose(day.adjClose, publishedAdj.get(day.date)), Volume: String(day.volume) })); } function mergeOfficialReturns(derived: PriceReturns, official: OfficialProductReturns | null): PriceReturns { @@ -1740,6 +1751,17 @@ function distributionRowsFromTable(rows: unknown): DistributionRow[] { const isOfficialSource = (value: unknown): boolean => /^WisdomTree (official|product-page)/.test(String(value ?? '')); +/** + * Where the published net assets came from. When this run's catalog row is the previous index (the live catalog + * was unreachable or did not list the fund), the value is the published one, so its published source label is kept + * too: a rerun with the same upstream data must not relabel it (product table -> N-PORT) and flip the fund to updated. + */ +export function aumSource(fund: Pick, nport: { netAssets?: number | null; repPdDate?: string | null } | null | undefined, previousMeta: JsonRecord | null): string { + if (fund.source === 'wisdomtree') return 'WisdomTree product table Assets Under Mgmt $(000)'; + if (fund.netAssets !== null && previousMeta?.aum?.source) return String(previousMeta.aum.source); + return nport ? `SEC Form N-PORT-P net assets (${nport.repPdDate || 'n/a'})` : 'previous run'; +} + async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: JsonRecord = {}): Promise { const reasons = catalogFilterReasons(fund, config); if (reasons.length) { @@ -1848,8 +1870,8 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J outputNote(`[ ${'chart'.padEnd(9)}] ${fund.ticker}: ${error instanceof Error ? error.message : String(error)}`); } } + const previousRows = await readPreviousSheet(fund.ticker, 'history'); if (!days.length) { - const previousRows = await readPreviousSheet(fund.ticker, 'history'); days = previousRows.map((row) => ({ date: toIsoDate(row.Date), close: numberOrNull(row.Close) || 0, adjClose: numberOrNull(row['Adj Close']) || numberOrNull(row.Close) || 0, volume: numberOrNull(row.Volume) || 0 })).filter((row) => row.date && row.close > 0); if (previousRows.length) historySource = previousMeta?.history?.source || 'previous run'; } @@ -1887,7 +1909,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J return { __skipped: true, ticker: fund.ticker, __skipReasons: returnFilterReasons }; } - const history = historyRows(days); + const history = historyRows(days, previousRows); const historyAsOf = derived.asOfDate || previousMeta?.history?.asOf || null; const holdingPages = await writePages(fundDir, fund.ticker, 'holdings', holdingsHeaders, holdingsRows, config.holdingsPageSize, holdingsAsOf, holdingsSource); const historyPages = await writePages(fundDir, fund.ticker, 'history', historyHeaders(), history, config.historyPageSize, historyAsOf, historySource); @@ -1962,7 +1984,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J ticker: fund.ticker, name: fund.name, category: fund.category, - categoryPath: fund.categoryPath, + categoryPath: fund.source === 'wisdomtree' ? fund.categoryPath : String(previousMeta?.categoryPath || fund.categoryPath), source: { fundPage: fund.fundPage, officialProductPage: productProxyUrl(fund.fundPage), @@ -1981,7 +2003,7 @@ async function processFund(fund: CatalogFund, config: UpdaterConfig, previous: J nav: { display: nav === null ? '—' : `$${nav.toFixed(2)}`, value: nav, asOfDate: navAsOfLabel }, marketPrice: { display: marketPrice === null ? '—' : `$${marketPrice.toFixed(2)}`, value: marketPrice, asOfDate: marketPriceAsOfLabel }, premiumDiscount: { display: premiumDiscount === null ? '—' : `${premiumDiscount.toFixed(2)}%`, value: premiumDiscount }, - aum: { display: formatAumDisplay(netAssets), value: netAssets, asOfDate: asOfDate ? formatDate(asOfDate) : (nport?.repPdDate ? formatDate(nport.repPdDate) : '—'), source: fund.source === 'wisdomtree' ? 'WisdomTree product table Assets Under Mgmt $(000)' : nport ? `SEC Form N-PORT-P net assets (${nport.repPdDate || 'n/a'})` : 'previous run' }, + aum: { display: formatAumDisplay(netAssets), value: netAssets, asOfDate: asOfDate ? formatDate(asOfDate) : (nport?.repPdDate ? formatDate(nport.repPdDate) : '—'), source: aumSource(fund, nport, previousMeta) }, yields: { dividendYield: metrics.dividendYield, dividendYieldText: metrics.dividendYieldText, dividendYieldBasis: metrics.dividendYieldBasis, dividendYieldKind: metrics.dividendYieldBasis === 'official-trailing-12m' ? YIELD_KIND_OFFICIAL : YIELD_KIND_INDICATED, distributionRate: productSummary?.distributionYield ?? numberOrNull(previousMeta?.yields?.distributionRate), secYield: metrics.secYield, secYieldText: metrics.secYieldText, secYieldKind: metrics.secYield !== null ? '30-day SEC yield published on the official WisdomTree product page' : 'not present in the current official product-page rendering' }, returns, distributions: { frequency: distributionFrequency, paymentsPerYear: distributionPaymentsPerYear, source: distributionsSource, headers: distributionHeaders, rows: distributionTable },