From fe39761cc8ccde300f23a71012dc63372ce4be56 Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Tue, 20 Oct 2026 19:15:50 +0000 Subject: [PATCH 01/10] Plot: show one point layer alone, from a row of buttons above the figure Each point layer is now its own group in the SVG, , with the plotted cohort as data-layer="base". A layer's cases that an earlier layer drew (the overlap rule) travel as `shared` and are drawn as a second group with display="none". The figure for MCP and exports looks exactly as before. On the page, PlotControls shows a "show" zone above the figure: one chip per drawn layer, in its colour and with its count. Clicking a chip sets data-solo on the figure's wrapper, and one CSS rule steps every other group back to 8% opacity while revealing that layer's shared group. So a layer shown alone is complete, e.g. KRAS G12C's 480 samples, not only the 414 it drew itself. Clicking the chip again or pressing Esc restores every layer. It is a reading gesture kept in browser state like a hover (invariant 9): it re-runs nothing and does not change the document. Verified on msk_chord_2024 TP53-mutant samples with [outliers, extremes x 20, KRAS G12C]: soloing KRAS shows its 414 drawn and 66 shared points at full opacity, the other groups at 0.08, and Esc restores them. A figure test checks the groups and the hidden shared group; an e2e check clicks a chip, checks the opacities, presses Esc and checks the document is untouched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/globals.css | 11 +++++++ app/lab/plot-controls.tsx | 37 +++++++++++++++++----- docs/plot-case-level.md | 17 +++++----- e2e.cjs | 19 +++++++++++ lib/figures/plot.ts | 66 +++++++++++++++++++++++++-------------- lib/plot.ts | 2 ++ lib/views/plot.ts | 2 ++ test/view-svg.test.ts | 12 +++++++ 8 files changed, 129 insertions(+), 37 deletions(-) diff --git a/app/globals.css b/app/globals.css index d6c5584..493a37b 100644 --- a/app/globals.css +++ b/app/globals.css @@ -1235,6 +1235,17 @@ details.cohortpick .gearmenu-items { min-width: 22rem; max-width: 30rem; gap: 0; } .menu-aside > .meta { display: block; } +/* a plot's point layers: one shown alone, the rest stepped back, and the cases it shares with an + earlier layer revealed (app/lab/plot-controls.tsx; the groups are lib/figures/plot.ts's) */ +.plot-figure[data-solo] g[data-layer] { opacity: 0.08; } +.plot-figure[data-solo="0"] g[data-layer="0"], +.plot-figure[data-solo="1"] g[data-layer="1"], +.plot-figure[data-solo="2"] g[data-layer="2"], +.plot-figure[data-solo="3"] g[data-layer="3"] { opacity: 1; } +.plot-figure[data-solo="0"] g[data-layer="0"][data-shared], +.plot-figure[data-solo="1"] g[data-layer="1"][data-shared], +.plot-figure[data-solo="2"] g[data-layer="2"][data-shared], +.plot-figure[data-solo="3"] g[data-layer="3"][data-shared] { display: inline; } /* a plot's axis controls: source, then whatever that source needs */ .axispick { display: inline-flex; align-items: center; gap: var(--s1); } diff --git a/app/lab/plot-controls.tsx b/app/lab/plot-controls.tsx index ec47ec6..408e84f 100644 --- a/app/lab/plot-controls.tsx +++ b/app/lab/plot-controls.tsx @@ -12,7 +12,7 @@ import { Checkbox } from "../_ui/checkbox"; import { Choice, OptionSelect } from "../_ui/choice.tsx"; import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; -import { FIELD } from "../_ui/toolbar.ts"; +import { CHIP, FIELD } from "../_ui/toolbar.ts"; import { DEFAULT_POINT_LAYERS, POINTS_CAP, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../../lib/plot-layers.ts"; type Update = (fn: (view: any) => any) => void; @@ -113,8 +113,9 @@ function PointLayers({ layers, plot, cohortOptions, axisName, busy, onChange }: export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohortOptions, update, children }: { /** the view as the document holds it */ spec: any; - /** the last result's plot: which shape is on screen, each axis's name and kind, how many cases it shows */ - plot?: { shape: string; n: number; x: { kind: string; label: string }; y: { kind: string; label: string }; coverage?: { total: number } }; + /** the last result's plot: which shape is on screen, each axis's name and kind, how many cases it shows, its layers */ + plot?: { shape: string; n: number; x: { kind: string; label: string }; y: { kind: string; label: string }; coverage?: { total: number }; + layers?: { label: string; color: string; n: number; shown: string }[] }; attrs: { attribute: string; kind: string }[]; profiles: string[]; loading?: boolean; busy: boolean; /** the cohorts a point layer may be: every one but the plotted one and a split */ cohortOptions: { name: string; label: string; depth: number }[]; @@ -123,6 +124,18 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor children: React.ReactNode; }) { const layers: PointLayerSpec[] = Array.isArray(spec.points) ? spec.points : DEFAULT_POINT_LAYERS; + // One layer shown alone, the rest stepped back: a reading gesture, kept in the browser like a + // hover (invariant 9), and dropped when the layers drawn change. + const drawn = (plot?.layers || []).map((l, i) => ({ ...l, i })).filter((l) => l.shown !== "none"); + const drawnKey = drawn.map((l) => l.i + ":" + l.label).join("|"); + const [solo, setSolo] = useState(null); + useEffect(() => setSolo(null), [drawnKey]); + useEffect(() => { + if (solo == null) return; + const off = (e: KeyboardEvent) => { if (e.key === "Escape") setSolo(null); }; + window.addEventListener("keydown", off); + return () => window.removeEventListener("keydown", off); + }, [solo]); const axisName = (side: "x" | "y") => plot?.[side].label || side; const unit = spec.unit || "sample"; // A box is drawn with its categories across, whichever axis the document holds them on, so a @@ -184,8 +197,8 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor

} {/* colour and ring split a box's violin and points; a scatter's density and a contingency have no per-case colour, so the controls appear only with a box on screen */} - {plot?.shape === "box" &&
-
+ {(plot?.shape === "box" || drawn.length > 0) &&
+ {plot?.shape === "box" && <>
colour samples by update((vv) => { const n = { ...vv }; if (a) n.colorBy = a; else delete n.colorBy; return n; })} /> @@ -193,9 +206,19 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor
update((vv) => { const n = { ...vv }; if (a) n.ringBy = a; else delete n.ringBy; return n; })} /> -
+
} + {drawn.length > 0 &&
+ show + {drawn.map((l) => ( + + ))} +
}
} - {children} +
{children}
diff --git a/docs/plot-case-level.md b/docs/plot-case-level.md index bc2f106..9a36024 100644 --- a/docs/plot-case-level.md +++ b/docs/plot-case-level.md @@ -75,23 +75,26 @@ One commit per item; `npm test` and a focused check per item; the gate before ea the figure beside it (`PlotControls`, shared by the lab card and the Results page). 3. **Done.** A point is a sample by default; patient mode combines samples by a stated rule. 4. **Done.** Point layers (above), replacing `highlight`, with the sidebar's Points section. -5. **Measure the cap.** SVG draw time and payload at 5k, 10k, 20k and 50k points on msk_chord; +5. **Done.** Show one layer alone: each layer is a group in the SVG tagged with its place in the + list (its cases an earlier layer drew are a hidden group of their own), and a row of buttons + above the figure steps the rest back with one CSS rule; Esc restores. Browser state only. +6. **Measure the cap.** SVG draw time and payload at 5k, 10k, 20k and 50k points on msk_chord; set `POINTS_CAP` from it. -6. **Multiple genes.** An alteration axis whose OQL has several statements gets one category per +7. **Multiple genes.** An alteration axis whose OQL has several statements gets one category per statement (overlaps as their own category, never assigned arbitrarily), and commas separate statements as well as `;` and newlines. -7. **Colour by gene per case** (phase 2). `colorBy: { source: "gene", gene }`: fill from the case's +8. **Colour by gene per case** (phase 2). `colorBy: { source: "gene", gene }`: fill from the case's highest-priority mutation (oncoprint priority, driver/VUS by the pinned OncoKB), ring from CNA/SV, "not profiled" its own class (panel coverage, reuse the oncoprint's). On scatters too; the rare classes become a point layer over the cap. Check per-class counts against the portal's Plots tab for one study and gene. -8. **Interaction** (phase 3): one overlay over the SVG maps the cursor to the nearest point (a grid +9. **Interaction** (phase 3): one overlay over the SVG maps the cursor to the nearest point (a grid index over the drawn points); hover tooltip (sample id linking to the patient view, values, - alterations), legend-click highlight, case-id / protein-change search whose matches come back + alterations), case-id / protein-change search whose matches come back from the server as a layer. -9. **Brush and drill-down** (phase 4): a region or a cell returns its cases as a layer; "N cases → +10. **Brush and drill-down** (phase 4): a region or a cell returns its cases as a layer; "N cases → study view" built from the full list on the server. -10. **Sidebar parity** (phase 5): "same gene", Driver vs VUS and VAF axes, box category filter and +11. **Sidebar parity** (phase 5): "same gene", Driver vs VUS and VAF axes, box category filter and sort-by-median, regression line toggle, R² and the equation, connect samples from the same patient. diff --git a/e2e.cjs b/e2e.cjs index f178f94..9da3c24 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -592,6 +592,25 @@ async function main() { } catch (e) { layerWhat = String(e.message).split("\n")[0]; } ok("a plot's point layers are listed in the sidebar, and adding one draws it with a key entry", layerOk, layerWhat); + // Showing one layer alone is instant and the browser's own: the others step back, Esc restores. + let soloWhat = "no layer buttons", soloOk = false; + try { + // every group, a layer's shared cases included: [layer, opacity, display] + const groups = () => page.evaluate(() => [...document.querySelectorAll(".plot-figure g[data-layer]")] + .map((g) => [g.getAttribute("data-layer"), getComputedStyle(g).opacity, getComputedStyle(g).display])); + const docBefore = await page.locator("#docjson").inputValue(); + await page.locator(".plot-solo button").last().click(); + const solo = await page.locator(".plot-figure").getAttribute("data-solo"); + const alone = await groups(); + await page.keyboard.press("Escape"); + const back = await groups(); + soloOk = solo != null && alone.some(([k]) => k === solo) && + alone.every(([k, o, d]) => k === solo ? o === "1" && d !== "none" : o !== "1") && + back.every(([, o]) => o === "1") && (await page.locator("#docjson").inputValue()) === docBefore; + soloWhat = "layer " + solo + " alone " + JSON.stringify(alone) + "; after Esc " + JSON.stringify(back) + "; document unchanged"; + } catch (e) { soloWhat = String(e.message).split("\n")[0]; } + ok("clicking a layer's button shows it alone, Esc shows every layer, and the document is untouched", soloOk, soloWhat); + // The oncoprint's gear keeps ONLY the link out to cBioPortal; genes moved // to a named button and a modal, so the gear is not where editing hides. let gearSplit = "no oncoprint rendered", gearOk = false; diff --git a/lib/figures/plot.ts b/lib/figures/plot.ts index e49fd3a..63e114e 100644 --- a/lib/figures/plot.ts +++ b/lib/figures/plot.ts @@ -5,7 +5,7 @@ // picture draws for a hundred cases or four hundred thousand. The only cases drawn one by one are // the points the engine chose (every case under the cap, the outliers over it), at most POINTS_CAP. import type { ViewResult } from "../types.ts"; -import type { PlotData } from "../plot.ts"; +import type { HighlightPoint, PlotData } from "../plot.ts"; import { isAbsent, jitterFor } from "../plot-draw.ts"; import { LAYER_BASE } from "../plot-layers.ts"; import { @@ -66,6 +66,13 @@ function layerItems(d: PlotData): LegendItem[] { } /** the layers drawn at all: with any, the plotted cohort steps back to a grey so their colours read */ const drawnLayers = (d: PlotData) => (d.layers || []).filter((l) => l.shown !== "none"); +/** + * One layer's marks as a group tagged with its place in the view's list, so a page can show that + * layer alone; `base` is the plotted cohort. A layer's cases an earlier layer drew are a second, + * hidden group the page reveals only when that layer stands alone. + */ +const layerGroup = (key: number | "base", marks: string[], shared = false) => + marks.length ? `${marks.join("")}` : ""; function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { const { x0, x1, y0, y1 } = d.extent!, [gx, gy] = d.grid!; @@ -100,17 +107,24 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { const shaded = d.pointSet?.rule === "all" && d.xyPoints?.length ? [] : cellMax ? d.bins!.filter((b) => b.n > cellMax) : d.bins!; // under point layers the plotted cohort is a floor-raised grey, so its faintest cells still read - for (const b of shaded) out.push(cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))); const own = d.xyPoints || []; - for (const pt of own) - out.push(``); - // a cohort layer too large to draw as points is cells on the same grid, shaded by its own densest cell - for (const h of hs) if (h.shown === "density" && h.bins) { - const hMax = Math.max(1, ...h.bins.map((b) => b.n)); - for (const b of h.bins) out.push(cell(b, h.color, 0.3 + 0.7 * Math.log1p(b.n) / Math.log1p(hMax))); + out.push(layerGroup("base", [ + ...shaded.map((b) => cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))), + ...own.map((pt) => ``)])); + const dot = (pt: { x: number; y: number }, color: string) => + ``; + for (const h of hs) { + const key = d.layers!.indexOf(h); + // a cohort layer too large to draw as points is cells on the same grid, shaded by its own densest cell + if (h.shown === "density" && h.bins) { + const hMax = Math.max(1, ...h.bins.map((b) => b.n)); + out.push(layerGroup(key, h.bins.map((b) => cell(b, h.color, 0.3 + 0.7 * Math.log1p(b.n) / Math.log1p(hMax))))); + } + if (h.shown === "points") { + out.push(layerGroup(key, h.points!.flatMap((pt) => "x" in pt ? [dot(pt, h.color)] : []))); + out.push(layerGroup(key, (h.shared || []).flatMap((pt) => "x" in pt ? [dot(pt, h.color)] : []), true)); + } } - for (const h of hs) if (h.shown === "points") for (const pt of h.points!) if ("x" in pt) - out.push(``); // the fitted line, over every case rather than over what is visible, clipped to the frame if (st && st.slope != null && st.intercept != null && x1 > x0) { const yAt = (xv: number) => st.intercept! + st.slope! * xv; @@ -209,6 +223,8 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { gs.forEach((g, i) => { const cx = L + bw * (i + 0.5), half = Math.min(24, bw * 0.32); + // the plotted cohort's violin and points, as one group a page can step back from + const baseMarks: string[] = []; if (d.overlay === "violin" && vio.has(g.g)) { const cells = vio.get(g.g)!; // a bin stands for a range of VALUES, mapped through the extent the bins were cut over @@ -224,7 +240,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const yTop = ys(Math.min(cHi, vHi)); const step = Math.max(1.2, ys(Math.max(cLo, vLo)) - yTop); if (!colorKey.length) { - out.push(``); + baseMarks.push(``); continue; } let x = cx - w; @@ -233,7 +249,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { for (const sg of segs) { if (!sg.n) continue; const seg = 2 * w * (sg.n / total); - out.push(``); + baseMarks.push(``); x += seg; } } @@ -245,14 +261,15 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const ring = pt.r != null && !isAbsent(pt.r) ? ringOf(pt.r) : null; if (pt.v > vhi || pt.v < vlo) { ptOffScale++; - out.push(edgeMark(cx + jitterFor(pt.id) * half * 0.9, pt.v > vhi, rBase, fill)); + baseMarks.push(edgeMark(cx + jitterFor(pt.id) * half * 0.9, pt.v > vhi, rBase, fill)); continue; } - out.push(``); } } + out.push(layerGroup("base", baseMarks)); out.push(``); out.push(`${esc(g.g)}: median ${fmtNum(g.median)}, ` + @@ -265,26 +282,29 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const hg = h.groups!.find((x) => x.g === g.g); if (!hg) return; const hw = Math.max(1.5, half * 0.42), hx = cx + (j - (boxed.length - 1) / 2) * hw * 1.25; - out.push(`<line x1="${r1(hx)}" y1="${r1(ys(inAxis(hg.loWhisker)))}" x2="${r1(hx)}" y2="${r1(ys(inAxis(hg.hiWhisker)))}" stroke="${h.color}"/>` + + out.push(layerGroup(d.layers!.indexOf(h), [`<line x1="${r1(hx)}" y1="${r1(ys(inAxis(hg.loWhisker)))}" x2="${r1(hx)}" y2="${r1(ys(inAxis(hg.hiWhisker)))}" stroke="${h.color}"/>` + `<rect x="${r1(hx - hw / 2)}" y="${r1(ys(hg.q3))}" width="${r1(hw)}" height="${r1(Math.max(1, ys(hg.q1) - ys(hg.q3)))}" fill="${h.color}" fill-opacity="0.85"/>` + - `<line x1="${r1(hx - hw / 2 - 1)}" y1="${r1(ys(hg.median))}" x2="${r1(hx + hw / 2 + 1)}" y2="${r1(ys(hg.median))}" stroke="${p.ink}" stroke-width="1.5"/>`); + `<line x1="${r1(hx - hw / 2 - 1)}" y1="${r1(ys(hg.median))}" x2="${r1(hx + hw / 2 + 1)}" y2="${r1(ys(hg.median))}" stroke="${p.ink}" stroke-width="1.5"/>`])); }); // A layer's value past the axis is pinned to the frame's edge as a triangle pointing off it, and // counted in the caption, rather than drawn outside the figure. With the cases coloured by a // category, a layer is a ring round the case, so fill and ring each keep one meaning. for (const h of hs) if (h.shown === "points") { const rH = PLOT_POINT_R; - for (const pt of h.points!) if ("g" in pt && pt.g === g.g) { + const marks = (pts: HighlightPoint[], counted: boolean) => pts.flatMap((pt) => { + if (!("g" in pt) || pt.g !== g.g) return []; const px = cx + jitterFor(pt.id) * half * 0.9; if (pt.v > vhi || pt.v < vlo) { - offScale++; - out.push(edgeMark(px, pt.v > vhi, rH, h.color)); - continue; + if (counted) offScale++; + return [edgeMark(px, pt.v > vhi, rH, h.color)]; } - out.push(categorised + return [categorised ? `<circle cx="${r1(px)}" cy="${r1(ys(pt.v))}" r="${r1(rH + 1.4)}" fill="none" stroke="${h.color}" stroke-width="1.4"/>` - : `<circle cx="${r1(px)}" cy="${r1(ys(pt.v))}" r="${rH}" fill="${h.color}" fill-opacity="0.9" stroke="${p.bg}" stroke-width="0.6"/>`); - } + : `<circle cx="${r1(px)}" cy="${r1(ys(pt.v))}" r="${rH}" fill="${h.color}" fill-opacity="0.9" stroke="${p.bg}" stroke-width="0.6"/>`]; + }); + const key = d.layers!.indexOf(h); + out.push(layerGroup(key, marks(h.points!, true))); + out.push(layerGroup(key, marks(h.shared || [], false), true)); } }); out.push(`<line x1="${L}" y1="${T + PLOT}" x2="${W - R}" y2="${T + PLOT}" stroke="${p.muted}" stroke-width="${SW.line}"/>`); diff --git a/lib/plot.ts b/lib/plot.ts index bf9e071..7bd7692 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -679,6 +679,8 @@ export interface PlotLayer { n: number; /** points drawn, once cases an earlier layer already drew are left to it */ drawn?: number; + /** the layer's cases an earlier layer drew: hidden on the figure, shown when the reader picks this layer alone */ + shared?: HighlightPoint[]; shown: "points" | "density" | "boxes" | "none"; /** why it is not drawn as points, when it is not */ why?: string; diff --git a/lib/views/plot.ts b/lib/views/plot.ts index e74e756..98e6c96 100644 --- a/lib/views/plot.ts +++ b/lib/views/plot.ts @@ -136,8 +136,10 @@ async function pointLayers(pv: PlotView, data: PlotData, c: ViewRun, cn: string, const { layer, keys } = picked; if (layer.shown === "points" && layer.points && keys) { const kept = layer.points.filter((_, j) => !seen.has(keys[j])); + const shared = layer.points.filter((_, j) => seen.has(keys[j])); for (const k of keys) seen.add(k); layer.drawn = kept.length; layer.points = kept; + if (shared.length) layer.shared = shared; budget -= kept.length; } out.push(layer); diff --git a/test/view-svg.test.ts b/test/view-svg.test.ts index c960905..72dae3b 100644 --- a/test/view-svg.test.ts +++ b/test/view-svg.test.ts @@ -404,6 +404,18 @@ describe("plot point layers: subsets drawn over the plotted cohort", () => { assert.match(svg, /1 value of a point layer lies more than a whisker span past the/); }); + test("each layer is a group tagged with its place in the list, its shared cases a hidden group of their own", () => { + const svg = viewToSvg(scatter([ + hl({ points: [{ id: "a", x: 10, y: 5 }] }), + hl({ name: "S", color: LAYER_COLORS[1], points: [{ id: "b", x: 20, y: 8 }], shared: [{ id: "a", x: 10, y: 5 }] }), + ]))!; + assert.match(svg, /<g data-layer="base">/); + assert.match(svg, /<g data-layer="0"><circle/); + assert.match(svg, /<g data-layer="1"><circle[^>]*fill="#0072B2"/); + assert.match(svg, /<g data-layer="1" data-shared="" display="none"><circle cx="[^"]*" cy="[^"]*" r="2.5" fill="#0072B2"/, + "the case layer 0 drew is drawn for layer 1 too, hidden until that layer is shown alone"); + }); + test("with the cases coloured by a category, a layer is a ring round the case, not a second fill", () => { const v: ViewResult = { kind: "plot", plot: { shape: "box", n: 3, x: { label: "G", kind: "cat" }, y: { label: "V", kind: "num" }, From d9447360804ce8265787e9088f142808107eb35b Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 19:24:36 +0000 Subject: [PATCH 02/10] Plot: the points cap is 10,000, set from a measurement The plot renderer drew a scatter and a box at 5k, 10k, 20k and 50k points, and headless Chrome on the dev server inserted and painted each SVG (median of 3): points paint scatter/box one hit-test JSON raw/gzip 5,000 84 / 117 ms 0.4 ms 220 / 35 KB 10,000 180 / 190 ms 1.7 ms 440 / 70 KB 20,000 352 / 337 ms 3.2 ms 880 / 140 KB 50,000 904 / 752 ms 7 ms 2.2 MB / 350 KB Painting, not building the string, is the cost: about 18 ms per thousand points, paid again on every redraw. 10,000 keeps a redraw near 200 ms on this CPU-only machine; 5,000 was inherited from the box plot's first version, not measured. POINTS_CAP moves to lib/plot-layers.ts (browser-safe) so the sidebar's labels read it, and the DSL reference and the controls print the constant instead of a typed "5,000". The measurement is recorded in docs/plot-case-level.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/plot-controls.tsx | 10 ++++++---- docs/dsl.md | 14 +++++++------- docs/plot-case-level.md | 19 ++++++++++++++++--- lib/dsl-reference.ts | 20 ++++++++++++-------- lib/plot-layers.ts | 5 ++++- lib/types.ts | 4 ++-- test/plot-layers.test.ts | 7 ++++--- test/plot-points.test.ts | 9 +++++---- 8 files changed, 56 insertions(+), 32 deletions(-) diff --git a/app/lab/plot-controls.tsx b/app/lab/plot-controls.tsx index 408e84f..7625114 100644 --- a/app/lab/plot-controls.tsx +++ b/app/lab/plot-controls.tsx @@ -16,6 +16,8 @@ import { CHIP, FIELD } from "../_ui/toolbar.ts"; import { DEFAULT_POINT_LAYERS, POINTS_CAP, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../../lib/plot-layers.ts"; type Update = (fn: (view: any) => any) => void; +/** the points cap as the controls print it */ +const CAP = POINTS_CAP.toLocaleString("en-US"); function LogToggle({ spec, side, numeric, busy, update }: { spec: any; side: "x" | "y"; numeric: boolean; busy: boolean; update: Update }) { const axis = spec[side]; @@ -71,7 +73,7 @@ function PointLayers({ layers, plot, cohortOptions, axisName, busy, onChange }: }) { const has = (f: (l: PointLayerSpec) => boolean) => layers.some(f); const offers: [string, string, PointLayerSpec][] = []; - if (!has((l) => "rule" in l && l.rule === "outliers")) offers.push(["outliers", "outliers (past 5,000 cases)", { rule: "outliers" }]); + if (!has((l) => "rule" in l && l.rule === "outliers")) offers.push(["outliers", "outliers (past " + CAP + " cases)", { rule: "outliers" }]); for (const side of ["x", "y"] as const) if (plot?.[side].kind === "num" && !has((l) => "rule" in l && l.rule === "extremes" && l.axis === side)) offers.push(["extremes:" + side, "top and bottom " + POINT_RULE_N + " by " + axisName(side), { rule: "extremes", axis: side }]); @@ -179,11 +181,11 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor <span className="axfield-name text-sm font-semibold text-foreground" title="subsets of the plotted cases drawn in their own colour over it; the first layer draws a case two share">Points</span> <label className="chk text-md text-foreground" title={overCap - ? "this plot has " + plot!.n.toLocaleString() + " " + unit + "s, more than 5,000, so it is drawn as density whatever this says" - : "every case as a point when there are at most 5,000 and the study may show cases; off draws the density alone"}> + ? "this plot has " + plot!.n.toLocaleString() + " " + unit + "s, more than " + CAP + ", so it is drawn as density whatever this says" + : "every case as a point when there are at most " + CAP + " and the study may show cases; off draws the density alone"}> <Checkbox checked={spec.allPoints !== false} disabled={busy || overCap} onCheckedChange={(c) => update((vv) => { const n = { ...vv }; if (c === true) delete n.allPoints; else n.allPoints = false; return n; })} /> - every {unit} (up to 5,000) + every {unit} (up to {CAP}) </label> <PointLayers layers={layers} plot={plot} cohortOptions={cohortOptions} axisName={axisName} busy={busy} onChange={(next) => update((vv) => ({ ...vv, points: next }))} /> diff --git a/docs/dsl.md b/docs/dsl.md index 9f3efc2..320666c 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -275,7 +275,7 @@ patient — so the arm counts summing to more than the base cohort is expected, | `logistic` | `outcome:axis, outcomeEvent?:"<level>", covariates:[axis, ...(max 8)], cohorts:[names]` | multivariate logistic regression: a BINARY outcome (not time-to-event — no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as `cox`'s. `outcome` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs `outcomeEvent` naming which observed value counts as the event (1) — every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given `outcomeEvent`. Covariates use the exact same dummy-coding and reference-level rules as `cox` (see above): a mutated-vs-not axis defaults to `"Not mutated"`, others default to the most frequent level, `reference` overrides either. Same patient-level-only boundary as `cox` | | `course` | `eventTypes:[≤6], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin — first recorded event to last contact (the later of their last event and their OS follow-up) — never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) `anchor` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED — read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (`Treatment`/`TREATMENT`) and the fold is named in the caveat; nothing else is merged | | `sequence` | `labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES — therapies less than `lineGap` days apart are one line, labelled by their whole combination (sorted, so `A + B` and `B + A` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one `lineGap` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days — so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, `lineClasses` `"all"` for every treatment record or a list of class names for your own; `labelKey` only names a line’s therapies (e.g. AGENT), it never decides which records form lines. `exclude` drops records by label value, matched ignoring case. More than two cohorts is refused | -| `plot` | `x:axis, y:axis, unit?:"sample"\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; `unit:"patient"` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 5,000 (`allPoints: false` turns that off), from a participant that may show cases; `points` lists up to four subsets drawn in colour over it — `{rule:"outliers"}` (the default), `{rule:"extremes",axis,n}`, `{rule:"trend",n}`, `{cohort}` — see **Points** below | +| `plot` | `x:axis, y:axis, unit?:"sample"\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; `unit:"patient"` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 10,000 (`allPoints: false` turns that off), from a participant that may show cases; `points` lists up to four subsets drawn in colour over it — `{rule:"outliers"}` (the default), `{rule:"extremes",axis,n}`, `{rule:"trend",n}`, `{cohort}` — see **Points** below | | `attrition` | `cohorts:[names] (≤6)` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its `all`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated | | `genome` | `binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ `threshold`) or lost (≤ −`threshold`) in every `binMb`-megabase bin (1–50) along chromosomes 1–22 (and X with `includeX` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out | | `pathways` | `pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"|"any", topGenes?:6, cohorts:[names] (≤4)` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own `sets`, each as the share of the cohort with at least one member altered. `measure` defaults to `"driver"` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or `"any"`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as `compare`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The `topGenes` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named | @@ -372,20 +372,20 @@ uses, not yet built for a box); its quartiles are a size-weighted approximation same one Explore's federated numeric histograms already disclose. **Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every -case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the +case as a point. So does this, up to 10,000 cases, on a scatter as on a box, and only when the study is one this participant may see cases in — the points carry case ids, so they come from the same entitlement that lets a patient-level participant list cohort members, and an aggregate-only participant has no method that could return them. The jitter across a box is fixed per case, so the figure does not move between the server SVG, the browser and the Python package. -Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a +Past 10,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a hover on it would land on an arbitrary case — and, by default, the **outliers** are a point layer over it (see **Points**): on a box, the values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at -most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of +most 3 cases (fewer, if that is what keeps the points under 10,000) is drawn as its cases instead of as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure -draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With +draws, and if a subset is itself past 10,000 none of it is drawn, never the first 10,000. With `allPoints: false` and `points: []`, or from a participant that may not disclose a case, the figure is density alone. The caption always says which it drew, how many points, and why there are none when there are @@ -416,7 +416,7 @@ or per sample following the plot's `unit`. Because it is a categorical axis, it `colorBy`/`ringBy` and as a Cox or logistic covariate, where "Unaltered" is the reference level. OQL is for alteration axes only: a molecular axis stays one gene's value. -**Points.** Every case is a point when there are at most 5,000 of them and every study may show +**Points.** Every case is a point when there are at most 10,000 of them and every study may show cases (`allPoints: false` draws the density alone at any size). Over that the plotted cohort is a density, and `points` lists the subsets worth seeing one by one, each drawn in its own colour with the plotted cohort turned grey beneath them (at most four layers, in order): @@ -428,7 +428,7 @@ grid cell holding at most 3 cases, left unshaded under its points); `{"rule": "e `{"cohort": "KRAS G12C"}` — the plotted cohort's cases that are also in that cohort (how many of its patients fall outside the plotted cohort is stated, and they are not drawn). The default is `[{"rule": "outliers"}]`; `[]` draws none. A case in two layers is drawn by the first, and the -caption counts what each layer left to an earlier one. Every layer shares one 5,000-point budget, +caption counts what each layer left to an earlier one. Every layer shares one 10,000-point budget, and a layer is never cut to its first cases: a cohort layer past the budget, or from a study that may not show cases, is drawn as an aggregate on the plot's own scale (cells on the scatter's grid, or a narrow box per group), and a rule layer is not drawn — the caption says which and why. diff --git a/docs/plot-case-level.md b/docs/plot-case-level.md index 9a36024..336c19d 100644 --- a/docs/plot-case-level.md +++ b/docs/plot-case-level.md @@ -35,7 +35,7 @@ a hover lands on an arbitrary one. So: combines the patient's samples by a stated rule: a number is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. -- **At or under `POINTS_CAP` (5,000), every case is a point** (`allPoints`, default true), on a +- **At or under `POINTS_CAP` (10,000), every case is a point** (`allPoints`, default true), on a scatter as on a box — the portal's picture, and what most single studies get. - **Over the cap, the crowd is density, and point layers draw chosen subsets in colour over it.** `points` is an ordered list of at most four layers (`lib/plot-layers.ts`): @@ -78,8 +78,21 @@ One commit per item; `npm test` and a focused check per item; the gate before ea 5. **Done.** Show one layer alone: each layer is a group in the SVG tagged with its place in the list (its cases an earlier layer drew are a hidden group of their own), and a row of buttons above the figure steps the rest back with one CSS rule; Esc restores. Browser state only. -6. **Measure the cap.** SVG draw time and payload at 5k, 10k, 20k and 50k points on msk_chord; - set `POINTS_CAP` from it. +6. **Done.** The cap, measured. The plot renderer drew a scatter and a box at n points + (synthetic values, real-shaped ids), and headless Chrome on the dev server (6 cores, no GPU) + inserted and painted each SVG, median of 3: + + | points | paint, scatter / box | one hover hit-test | JSON (raw / gzip) | + |---|---|---|---| + | 5,000 | 84 / 117 ms | 0.4 ms | 220 / 35 KB | + | 10,000 | 180 / 190 ms | 1.7 ms | 440 / 70 KB | + | 20,000 | 352 / 337 ms | 3.2 ms | 880 / 140 KB | + | 50,000 | 904 / 752 ms | 7 ms | 2.2 MB / 350 KB | + + Building the SVG string is 25–190 ms and not the limit; painting is, at about 18 ms per + thousand points, paid again on every redraw (a resize, a re-run). `POINTS_CAP` is 10,000, which + keeps a redraw near 200 ms on this machine (a laptop with a GPU is faster). Hover will go + through one overlay (item 9), so per-circle hit-testing does not set the cap. 7. **Multiple genes.** An alteration axis whose OQL has several statements gets one category per statement (overlaps as their own category, never assigned arbitrarily), and commas separate statements as well as `;` and newlines. diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index af3aa84..74d549e 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -17,12 +17,16 @@ // ============================================================================ import { DOC_VERSION } from "./doc-version.ts"; +import { POINTS_CAP } from "./plot-layers.ts"; import type { TemplateDoc } from "./types.ts"; import type { CustomDataset } from "./custom-data-names.ts"; import { LUAD_BIOPSY_SITE_DEMO } from "./custom-data-demo.ts"; import { SPECTRUM_SC_DEMO } from "./spectrum-sc-demo.ts"; import { HTA11_SC_DEMO } from "./hta11-sc-demo.ts"; +/** the plot's points cap as the docs print it */ +const POINTS_CAP_TEXT = POINTS_CAP.toLocaleString("en-US"); + export const DSL_VERSION = DOC_VERSION; export interface DslExample { @@ -1522,7 +1526,7 @@ patient — so the arm counts summing to more than the base cohort is expected, | \`logistic\` | \`outcome:axis, outcomeEvent?:"<level>", covariates:[axis, ...(max 8)], cohorts:[names]\` | multivariate logistic regression: a BINARY outcome (not time-to-event \u2014 no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as \`cox\`'s. \`outcome\` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs \`outcomeEvent\` naming which observed value counts as the event (1) \u2014 every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given \`outcomeEvent\`. Covariates use the exact same dummy-coding and reference-level rules as \`cox\` (see above): a mutated-vs-not axis defaults to \`\"Not mutated\"\`, others default to the most frequent level, \`reference\` overrides either. Same patient-level-only boundary as \`cox\` | | \`course\` | \`eventTypes:[\u22646], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]\` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin \u2014 first recorded event to last contact (the later of their last event and their OS follow-up) \u2014 never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) \`anchor\` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED \u2014 read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (\`Treatment\`/\`TREATMENT\`) and the fold is named in the caveat; nothing else is merged | | \`sequence\` | \`labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]\` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES \u2014 therapies less than \`lineGap\` days apart are one line, labelled by their whole combination (sorted, so \`A + B\` and \`B + A\` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one \`lineGap\` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days \u2014 so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, \`lineClasses\` \`"all"\` for every treatment record or a list of class names for your own; \`labelKey\` only names a line\u2019s therapies (e.g. AGENT), it never decides which records form lines. \`exclude\` drops records by label value, matched ignoring case. More than two cohorts is refused | -| \`plot\` | \`x:axis, y:axis, unit?:"sample"\\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; \`unit:"patient"\` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 5,000 (\`allPoints: false\` turns that off), from a participant that may show cases; \`points\` lists up to four subsets drawn in colour over it — \`{rule:"outliers"}\` (the default), \`{rule:"extremes",axis,n}\`, \`{rule:"trend",n}\`, \`{cohort}\` — see **Points** below | +| \`plot\` | \`x:axis, y:axis, unit?:"sample"\\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; \`unit:"patient"\` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to ${POINTS_CAP_TEXT} (\`allPoints: false\` turns that off), from a participant that may show cases; \`points\` lists up to four subsets drawn in colour over it — \`{rule:"outliers"}\` (the default), \`{rule:"extremes",axis,n}\`, \`{rule:"trend",n}\`, \`{cohort}\` — see **Points** below | | \`attrition\` | \`cohorts:[names] (≤6)\` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its \`all\`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated | | \`genome\` | \`binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)\` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ \`threshold\`) or lost (≤ −\`threshold\`) in every \`binMb\`-megabase bin (1–50) along chromosomes 1–22 (and X with \`includeX\` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out | | \`pathways\` | \`pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"\|"any", topGenes?:6, cohorts:[names] (≤4)\` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own \`sets\`, each as the share of the cohort with at least one member altered. \`measure\` defaults to \`"driver"\` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or \`"any"\`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as \`compare\`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The \`topGenes\` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named | @@ -1619,20 +1623,20 @@ uses, not yet built for a box); its quartiles are a size-weighted approximation same one Explore's federated numeric histograms already disclose. **Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every -case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the +case as a point. So does this, up to ${POINTS_CAP_TEXT} cases, on a scatter as on a box, and only when the study is one this participant may see cases in — the points carry case ids, so they come from the same entitlement that lets a patient-level participant list cohort members, and an aggregate-only participant has no method that could return them. The jitter across a box is fixed per case, so the figure does not move between the server SVG, the browser and the Python package. -Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a +Past ${POINTS_CAP_TEXT} the crowd is drawn as density — a quarter of a million dots is a solid block, and a hover on it would land on an arbitrary case — and, by default, the **outliers** are a point layer over it (see **Points**): on a box, the values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at -most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of +most 3 cases (fewer, if that is what keeps the points under ${POINTS_CAP_TEXT}) is drawn as its cases instead of as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure -draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With +draws, and if a subset is itself past ${POINTS_CAP_TEXT} none of it is drawn, never the first ${POINTS_CAP_TEXT}. With \`allPoints: false\` and \`points: []\`, or from a participant that may not disclose a case, the figure is density alone. The caption always says which it drew, how many points, and why there are none when there are @@ -1663,7 +1667,7 @@ or per sample following the plot's \`unit\`. Because it is a categorical axis, i \`colorBy\`/\`ringBy\` and as a Cox or logistic covariate, where "Unaltered" is the reference level. OQL is for alteration axes only: a molecular axis stays one gene's value. -**Points.** Every case is a point when there are at most 5,000 of them and every study may show +**Points.** Every case is a point when there are at most ${POINTS_CAP_TEXT} of them and every study may show cases (\`allPoints: false\` draws the density alone at any size). Over that the plotted cohort is a density, and \`points\` lists the subsets worth seeing one by one, each drawn in its own colour with the plotted cohort turned grey beneath them (at most four layers, in order): @@ -1675,7 +1679,7 @@ grid cell holding at most 3 cases, left unshaded under its points); \`{"rule": " \`{"cohort": "KRAS G12C"}\` — the plotted cohort's cases that are also in that cohort (how many of its patients fall outside the plotted cohort is stated, and they are not drawn). The default is \`[{"rule": "outliers"}]\`; \`[]\` draws none. A case in two layers is drawn by the first, and the -caption counts what each layer left to an earlier one. Every layer shares one 5,000-point budget, +caption counts what each layer left to an earlier one. Every layer shares one ${POINTS_CAP_TEXT}-point budget, and a layer is never cut to its first cases: a cohort layer past the budget, or from a study that may not show cases, is drawn as an aggregate on the plot's own scale (cells on the scatter's grid, or a narrow box per group), and a rule layer is not drawn — the caption says which and why. @@ -1902,7 +1906,7 @@ export const AGENT_GRAMMAR = `Document shape: // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|driver"?} // | {"source":"alteration","genes":[G,...(2-20)],"by":"mutated"?} // any-of-N as one category; only "mutated" // | {"source":"alteration","oql":"KRAS: G12C; STK11: MUT"} // Altered (matches any entry) vs Unaltered - // two numeric axes -> scatter (binned; every case a point up to 5,000, else sparse-cell outliers); one categorical -> box; + // two numeric axes -> scatter (binned; every case a point up to ${POINTS_CAP_TEXT}, else sparse-cell outliers); one categorical -> box; // two categorical -> a contingency table {"kind":"attrition","cohorts":[names?(max 6)]}, {"kind":"genome","binMb":2,"threshold":0.2,"includeX":false,"cohorts":[names?(max 4)]}, diff --git a/lib/plot-layers.ts b/lib/plot-layers.ts index 14913bf..c4bc096 100644 --- a/lib/plot-layers.ts +++ b/lib/plot-layers.ts @@ -4,8 +4,11 @@ /** * How many cases a plot draws as points before it stops being a drawing: at or under it every case * is a point; past it the plotted cohort is density, and the point layers share this many points. + * Set from a measurement (docs/plot-case-level.md): an SVG of n points paints in about 18 ms per + * thousand on the dev server's headless Chrome, so 10,000 keeps a redraw near 200 ms; the points + * travel as about 44 bytes each. */ -export const POINTS_CAP = 5000; +export const POINTS_CAP = 10000; /** layers one plot may draw: each takes a colour of its own, and past four they stop being told apart */ export const POINT_LAYER_CAP = 4; /** a layer's colour, by its place in the view's list */ diff --git a/lib/types.ts b/lib/types.ts index 4dc954f..acf1495 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -549,7 +549,7 @@ export interface PlotView { /** the grid a scatter is binned onto (default 240 x 160) */ bins?: [number, number]; /** - * Every case is a point when there are at most 5,000 of them and the reader may see cases + * Every case is a point when there are at most POINTS_CAP (10,000) of them and the reader may see cases * (default true); false draws the density alone at any size. */ allPoints?: boolean; @@ -558,7 +558,7 @@ export interface PlotView { * outliers (past the cap: a box's values beyond its whiskers, a scatter's sparse cells), the * extremes on an axis, the cases furthest from the fitted line, or a cohort's cases. Default * [{rule:"outliers"}]; [] draws none. A case in two layers is drawn by the first. Every layer - * shares one 5,000-point budget; a cohort layer past it is drawn as density cells or boxes. + * shares one POINTS_CAP budget; a cohort layer past it is drawn as density cells or boxes. */ points?: PointLayerSpec[]; /** diff --git a/test/plot-layers.test.ts b/test/plot-layers.test.ts index 68011bd..5fb0930 100644 --- a/test/plot-layers.test.ts +++ b/test/plot-layers.test.ts @@ -10,6 +10,7 @@ import type { ViewRun, EvaluatedCohort } from "../lib/views/run.ts"; import type { PlotData, ViewResult } from "../lib/types.ts"; type Calls = string[]; +const CAP = POINTS_CAP.toLocaleString("en-US"); const EXTENT = { x0: 0, x1: 100, y0: 0, y1: 50 }; const scatter = (): PlotData => ({ shape: "scatter", n: 1000, x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" }, extent: { ...EXTENT }, grid: [240, 160], bins: [{ bx: 1, by: 1, n: 1000 }], @@ -83,7 +84,7 @@ describe("a highlight on one study", () => { assert.match(r.note!, /statistics are over the whole plotted cohort/); }); - test("past the cap the same cases come back as cells on the plot's own grid and extent, never the first 5,000", async () => { + test("past the cap the same cases come back as cells on the plot's own grid and extent, never the first POINTS_CAP", async () => { const calls: Calls = []; const r = await run({ s: fake("patient-level", calls, { points: null }) }); const h = r.plot!.layers![0]; @@ -92,7 +93,7 @@ describe("a highlight on one study", () => { assert.deepEqual(h.bins, [{ bx: 2, by: 3, n: 7 }]); assert.equal(h.n, 7); assert.ok(calls.includes("bins [240,160] " + JSON.stringify(EXTENT)), "the highlight is binned on the scatter's grid: " + calls.join("; ")); - assert.match(r.note!, /drawn as shaded cells \(more than 5,000 cases/); + assert.match(r.note!, new RegExp("drawn as shaded cells \\(more than " + CAP + " cases")); }); test("an aggregate-only participant is never asked for points, and its highlight is an aggregate", async () => { @@ -226,7 +227,7 @@ describe("rule layers", () => { for (const extra of [{}, { allPoints: false }]) { const r = await run({ s: fake("patient-level", []) }, { points: [{ rule: "outliers" }], extra }); assert.deepEqual([r.plot!.layers![0].shown, r.plot!.layers![0].why], - ["none", "at or under 5,000 cases there is no density for them to stand out of"], JSON.stringify(extra)); + ["none", "at or under " + CAP + " cases there is no density for them to stand out of"], JSON.stringify(extra)); } }); diff --git a/test/plot-points.test.ts b/test/plot-points.test.ts index 82c38c8..cfcb271 100644 --- a/test/plot-points.test.ts +++ b/test/plot-points.test.ts @@ -9,6 +9,7 @@ import { POINTS_CAP, OUTLIER_CELL_MAX, sparseCellMax } from "../lib/plot.ts"; import type { ViewRun, EvaluatedCohort } from "../lib/views/run.ts"; import type { PlotData, ViewResult } from "../lib/types.ts"; +const CAP = POINTS_CAP.toLocaleString("en-US"); const EXTENT = { x0: 0, x1: 100, y0: 0, y1: 50 }; const scatter = (n: number, bins: { bx: number; by: number; n: number }[]): PlotData => ({ shape: "scatter", n, x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" }, @@ -83,7 +84,7 @@ describe("a scatter's points", () => { assert.equal(calls.length, 1, "over the cap the every-case query is not run"); assert.deepEqual(calls[0].extra, { grid: [240, 160], extent: EXTENT, cellMax: OUTLIER_CELL_MAX }); assert.equal(calls[0].cap, POINTS_CAP); - assert.deepEqual(r.plot!.pointSet, { rule: "none", drawn: 0, why: "more than 5,000 cases" }); + assert.deepEqual(r.plot!.pointSet, { rule: "none", drawn: 0, why: "more than " + CAP + " cases" }); const l = r.plot!.layers![0]; assert.deepEqual([l.kind, l.shown, l.n, l.drawn, l.cellMax], ["outliers", "points", 1, 1, OUTLIER_CELL_MAX]); assert.match(r.note!, /outliers: 1 sample in cells holding at most 3, left unshaded under them/); @@ -98,7 +99,7 @@ describe("a scatter's points", () => { assert.equal(r.plot!.pointSet!.rule, "none"); assert.equal(r.plot!.xyPoints, undefined); assert.equal(r.plot!.layers![0].shown, "none"); - assert.match(r.note!, /outliers: not drawn, the cases alone in a cell are more than the 5,000 points left to draw/); + assert.match(r.note!, new RegExp("outliers: not drawn, the cases alone in a cell are more than the " + CAP + " points left to draw")); }); test("an aggregate-only participant is never asked, and the caption names it", async () => { @@ -139,11 +140,11 @@ describe("a box's points", () => { assert.match(r.note!, /outliers: 1 value beyond the whiskers/); }); - test("more outliers than the cap: none, never the first 5,000", async () => { + test("more outliers than the cap: none, never the first POINTS_CAP", async () => { const calls: Call[] = []; const r = await run(fake("patient-level", calls, () => box(POINTS_CAP * 3), { outliers: null })); assert.equal(r.plot!.points, undefined); assert.equal(r.plot!.layers![0].shown, "none"); - assert.match(r.note!, /outliers: not drawn, more than 5,000 values lie beyond the whiskers/); + assert.match(r.note!, new RegExp("outliers: not drawn, more than " + CAP + " values lie beyond the whiskers")); }); }); From 995791f1b8dd63860f318e902008e3e7704a4dc7 Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 19:44:20 +0000 Subject: [PATCH 03/10] Plot: OQL axes by statement, commas between statements, and "Not profiled" never counted as unaltered An OQL axis's statements may now be separated by commas as well as ";" and new lines, everywhere OQL is read: a cohort's oql leaf, a plot axis, splits. The engine's parser and the gene query dialog share OQL_SEPARATOR / oqlStatements in lib/oql-parse.ts; a comma used to be an error, so nothing that worked changes meaning. `by: "statement"` gives each statement its own group, named as written with the gene in capitals. A case matching two statements is the pair ("KRAS: G12C + STK11: MUT"), three or more is "3 or more lines", and there are at most 6 statements. `by: "any"`, the default, stays Altered/Unaltered, which Cox and logistic covariates take with "Unaltered" as reference. In both, a case that matches no statement is "Unaltered" only when it is profiled for every statement's gene in the alteration types that statement asks about. Otherwise it is "Not profiled". Before, the axis counted a sample whose panel omits the gene as Unaltered, the wild-type error docs/PERFORMANCE.md's denominator rule forbids. The panel test is the `profiled` clause's own (sample_to_gene_panel_derived, WES counted), exported as cohort2's profiledCasesSql. EXP/PROT statements read a value, not a panel, so they need no profiling. Inner columns are named oql_case / profiled_id, because ClickHouse resolved a bare `id` to another axis's alias once the query was embedded in the plot's. The axis control offers "matches OQL (any line)" and "OQL, one group per line". The OQL gallery example gains a by-line view (STK11, KEAP1, KRAS G12C in lung cancer). The DSL reference and docs/plot-case-level.md say all of this. Verified on msk_chord_2024, all 25,040 samples, `KRAS: G12C, RRAGC: MUT` (RRAGC is on the panels of 20,048 samples): - by line: KRAS G12C 1,111, RRAGC 61, both 3, Unaltered 19,115, Not profiled 4,750 - any: Altered 1,175 Each equals a direct ClickHouse query over the panel and event tables. The OQL example's stated numbers (1,535 Altered, 6,274 Unaltered) still hold. docs:check runs every example and check:methods reproduces every view's numbers, Cox and logistic with OQL covariates included. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/view-chrome.tsx | 21 ++++++++----- docs/dsl.md | 38 ++++++++++++++++++----- docs/plot-case-level.md | 10 ++++-- lib/cohort2.ts | 18 +++++++++++ lib/dsl-reference.ts | 33 +++++++++++++------- lib/gene-query.ts | 4 +-- lib/oql-parse.ts | 9 +++++- lib/plot-oql.ts | 8 ++++- lib/plot.ts | 67 ++++++++++++++++++++++++++++++----------- test/gene-query.test.ts | 3 ++ test/plot.test.ts | 21 +++++++++++-- 11 files changed, 178 insertions(+), 54 deletions(-) diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index f48a5af..82ce335 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -134,28 +134,33 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang const isOql = a.oql != null; const genes: string[] = a.genes && a.genes.length ? a.genes : a.gene ? [a.gene] : []; const isSet = !isOql && genes.length > 1; - const groupBy = <OptionSelect className={w} value={isOql ? "oql" : a.by || (isSet ? "mutated" : "mutationType")} disabled={busy} + const groupBy = <OptionSelect className={w} value={isOql ? (a.by === "statement" ? "oqlEach" : "oql") : a.by || (isSet ? "mutated" : "mutationType")} disabled={busy} ariaLabel={side + " alteration grouping"} - title={isOql ? "Altered when a case matches any line of the query, Unaltered otherwise" + title={isOql ? (a.by === "statement" ? "a group for each line of the query; a case matching two is their pair, none Unaltered (or Not profiled)" + : "Altered when a case matches any line of the query; Unaltered when it matches none and is profiled for every gene, else Not profiled") : isSet ? "a gene set is \"mutated vs not\", or write it as an OQL query" : "how the gene's alterations are grouped"} onValueChange={(v) => { const by = v; // to OQL: the genes become a query (a bare gene is "any alteration"); from OQL: the // query's first gene, so a round trip keeps the gene - if (by === "oql") set({ source: "alteration", oql: genes.length ? genes.join("; ") : "TP53: MUT" }); + if (by === "oql" || by === "oqlEach") { + const oql = isOql ? a.oql : genes.length ? genes.join("; ") : "TP53: MUT"; + set(by === "oqlEach" ? { source: "alteration", oql, by: "statement" } : { source: "alteration", oql }); + } else if (isOql) set({ source: "alteration", gene: (/^\s*([A-Za-z0-9.-]+)/.exec(a.oql) || [])[1]?.toUpperCase() || "TP53", by }); else set({ ...a, by }); }}> <option value="mutationType" disabled={isSet}>mutation type</option> <option value="mutated">mutated vs not</option> <option value="cna" disabled={isSet}>copy number</option> - <option value="oql">matches OQL</option> + <option value="oql">matches OQL (any line)</option> + <option value="oqlEach">OQL, one group per line</option> </OptionSelect>; // stacked, the grouping comes first: it decides whether the box below takes genes or a query return <> {stacked && field("Group cases by", groupBy)} {field(isOql ? "OQL query" : "Gene(s)", isOql - ? <OqlInput query={a.oql} busy={busy} className={stacked ? w : undefined} onCommit={(oql) => set({ source: "alteration", oql })} /> + ? <OqlInput query={a.oql} busy={busy} className={stacked ? w : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} /> : <GeneSetInput genes={genes} busy={busy} className={stacked ? w : undefined} onCommit={(next) => { // A gene SET (2+) only supports "mutated vs not" — combining several genes' @@ -234,10 +239,10 @@ export function OqlInput({ query, busy, onCommit, className }: { query: string; useEffect(() => setDraft(query), [query]); const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); if (v.trim() && v !== query) onCommit(v.trim()); }; return ( - <Input className={cn(FIELD, "w-auto max-w-88 px-1.5", className)} value={draft} disabled={busy} placeholder="KRAS: G12C; STK11: MUT" spellCheck={false} + <Input className={cn(FIELD, "w-auto max-w-88 px-1.5", className)} value={draft} disabled={busy} placeholder="KRAS: G12C, STK11: MUT" spellCheck={false} size={Math.max(16, Math.min(44, draft.length + 2))} - title={"OQL, one gene per line or ';' — e.g. KRAS: G12C; STK11: MUT; EGFR: AMP; TP53: DRIVER; ERBB2: EXP>2. " + - "A case is Altered when it matches any of them."} + title={"OQL, one statement per line, ';' or ',' — e.g. KRAS: G12C, STK11: MUT, EGFR: AMP, TP53: DRIVER, ERBB2: EXP>2. " + + "Grouped by any line (Altered or not) or one group per line, as chosen beside it."} onChange={(e) => { const v = e.target.value; setDraft(v); diff --git a/docs/dsl.md b/docs/dsl.md index 320666c..c396343 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -407,13 +407,19 @@ bin's width is shared out between the classes in it. That stays an aggregate, so may not disclose a case can still colour its plot. The key is capped at 12 classes. **OQL on an axis.** An alteration axis can be an OQL query instead of genes: -`{"source":"alteration","oql":"KRAS: G12C; STK11: MUT"}`. Each case is **Altered** when it matches -any entry — the portal's own reading of a multi-gene query — and **Unaltered** otherwise; every -case of the cohort is one or the other, so the unaltered are counted rather than missing. It is -the same OQL a cohort's `oql` leaf takes (mutation classes and protein changes, AMP/HOMDEL, -FUSION, DRIVER against the pinned OncoKB version, EXP/PROT thresholds), and it matches per patient -or per sample following the plot's `unit`. Because it is a categorical axis, it also works as -`colorBy`/`ringBy` and as a Cox or logistic covariate, where "Unaltered" is the reference level. +`{"source":"alteration","oql":"KRAS: G12C, STK11: MUT"}`, its statements separated by a comma, a +semicolon or a new line (as in a cohort's `oql` leaf). Each case is **Altered** when it matches any +statement — the portal's own reading of a multi-gene query — and **Unaltered** when it matches none +and is profiled for every statement's gene in the alteration types that statement asks about; a +case whose gene panel omits one of them is **Not profiled**, never counted as wild type (the +panel-aware rule of `docs/PERFORMANCE.md`). Every case of the cohort is one of the three, so none is +missing. With `"by": "statement"` each statement is its own group, named as written: a case matching +two is the pair ("KRAS: G12C + STK11: MUT"), three or more "3 or more lines", none Unaltered or Not +profiled by the same rule (at most 6 statements). It is the same OQL a cohort's `oql` leaf takes +(mutation classes and protein changes, AMP/HOMDEL, FUSION, DRIVER against the pinned OncoKB version, +EXP/PROT thresholds), and it matches per patient or per sample following the plot's `unit`. +Because it is a categorical axis, it also works as `colorBy`/`ringBy` and as a Cox or logistic +covariate, where "Unaltered" is the reference level. OQL is for alteration axes only: a molecular axis stays one gene's value. **Points.** Every case is a point when there are at most 10,000 of them and every study may show @@ -2189,7 +2195,7 @@ TCGA lung adenocarcinoma and squamous cell carcinoma pooled (1,053 patients). KR ### Example 20: OQL on a plot axis: STK11 or KEAP1 mutant lung cancer -Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients. +Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients. The third gives each statement of `STK11: MUT, KEAP1: MUT, KRAS: G12C` its own group, over samples: 620 KRAS G12C, 459 KEAP1, 377 STK11, 403 with both STK11 and KEAP1 (median TMB 8.6 against 4.1 for the 5,720 with none), the other pairs as their own groups and 109 matching all three. None of these lung samples is on a panel without the three genes, so no group is Not profiled. ```json { @@ -2244,6 +2250,22 @@ Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups "source": "alteration", "oql": "STK11: MUT; KEAP1: MUT" } + }, + { + "kind": "plot", + "unit": "sample", + "cohorts": [ + "NSCLC" + ], + "x": { + "source": "alteration", + "oql": "STK11: MUT, KEAP1: MUT, KRAS: G12C", + "by": "statement" + }, + "y": { + "source": "clinical", + "attribute": "TMB_NONSYNONYMOUS" + } } ] } diff --git a/docs/plot-case-level.md b/docs/plot-case-level.md index 336c19d..e33db78 100644 --- a/docs/plot-case-level.md +++ b/docs/plot-case-level.md @@ -93,9 +93,13 @@ One commit per item; `npm test` and a focused check per item; the gate before ea thousand points, paid again on every redraw (a resize, a re-run). `POINTS_CAP` is 10,000, which keeps a redraw near 200 ms on this machine (a laptop with a GPU is faster). Hover will go through one overlay (item 9), so per-circle hit-testing does not set the cap. -7. **Multiple genes.** An alteration axis whose OQL has several statements gets one category per - statement (overlaps as their own category, never assigned arbitrarily), and commas separate - statements as well as `;` and newlines. +7. **Done.** Multiple genes. An OQL axis's statements are separated by ",", ";" or a new line + (everywhere OQL is read). `by: "statement"` gives each statement its own group, a case matching + two the pair, three or more "3 or more lines". Either way a case matching nothing is + "Unaltered" only when it is profiled for every statement's gene (panel-aware), otherwise "Not + profiled". On msk_chord, KRAS G12C and RRAGC (on 20,048 of 25,040 samples' panels): 1,111 / + 61 / 3 both / 19,115 Unaltered / 4,750 Not profiled, equal to a direct query; the 4,750 were + counted as Unaltered before. 8. **Colour by gene per case** (phase 2). `colorBy: { source: "gene", gene }`: fill from the case's highest-priority mutation (oncoprint priority, driver/VUS by the pinned OncoKB), ring from CNA/SV, "not profiled" its own class (panel coverage, reuse the oncoprint's). On scatters too; diff --git a/lib/cohort2.ts b/lib/cohort2.ts index 94d0024..d6af877 100644 --- a/lib/cohort2.ts +++ b/lib/cohort2.ts @@ -232,6 +232,24 @@ function derivedLeafSql(leaf: any, ctx: Ctx): string { " WHERE " + conds.join(" AND "); } +/** + * The cases (samples, or patients with a profiled sample) profiled for `gene` in one alteration + * type, as bare ids in the column `profiled_id`: the same panel tables, and the same whole-exome rule, as the `profiled` clause + * and the gene-table denominator. What an OQL plot axis needs to tell "not altered" from "not + * profiled", per sample or per patient. + */ +export function profiledCasesSql(gene: string, type: "MUTATION" | "CNA" | "SV", o: { study: string; db: string; unit: "patient" | "sample" }): string { + const g = String(gene).toUpperCase().trim(); + if (!/^[A-Z0-9][A-Z0-9.-]{0,19}$/.test(g)) throw new Error("profiled: invalid gene symbol"); + const col = o.unit === "sample" ? "sd.sample_unique_id" : "sd.patient_unique_id"; + return "SELECT DISTINCT " + PID(col, o.study) + " AS profiled_id\n" + + " FROM " + o.db + ".sample_to_gene_panel_derived sp\n" + + " JOIN " + o.db + ".sample_derived sd ON sd.sample_unique_id = sp.sample_unique_id\n" + + " WHERE sp.cancer_study_identifier = " + sq(o.study) + " AND sp.alteration_type = " + sq(PROFILED_TYPE[type]) + + "\n AND (sp.gene_panel_id = 'WES' OR sp.gene_panel_id IN (\n" + + " SELECT gene_panel_id FROM " + o.db + ".gene_panel_to_gene_derived WHERE gene = " + sq(g) + "))"; +} + function profiledLeafSql(leaf: ProfiledLeaf, ctx: Ctx): string { const t = PROFILED_TYPE[String(leaf.type || "").toUpperCase()]; if (!t) throw new Error("profiled clause: type must be MUTATION, CNA, or SV"); diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index 74d549e..0fdabab 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -18,6 +18,7 @@ import { DOC_VERSION } from "./doc-version.ts"; import { POINTS_CAP } from "./plot-layers.ts"; +import { OQL_STATEMENT_CAP } from "./plot-oql.ts"; import type { TemplateDoc } from "./types.ts"; import type { CustomDataset } from "./custom-data-names.ts"; import { LUAD_BIOPSY_SITE_DEMO } from "./custom-data-demo.ts"; @@ -463,9 +464,9 @@ export const EXAMPLES: DslExample[] = [ }, { title: "OQL on a plot axis: STK11 or KEAP1 mutant lung cancer", - demonstrates: "An alteration axis written as an OQL query — several genes, Altered when a case matches any of them — used as a box plot's groups and as a colour.", + demonstrates: "An alteration axis written as an OQL query — several genes, Altered when a case matches any of them, or one group per statement — used as a box plot's groups and as a colour.", tags: ["OQL"], - note: "Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients.", + note: "Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients. The third gives each statement of `STK11: MUT, KEAP1: MUT, KRAS: G12C` its own group, over samples: 620 KRAS G12C, 459 KEAP1, 377 STK11, 403 with both STK11 and KEAP1 (median TMB 8.6 against 4.1 for the 5,720 with none), the other pairs as their own groups and 109 matching all three. None of these lung samples is on a panel without the three genes, so no group is Not profiled.", doc: { study: "msk_chord_2024", cohorts: { @@ -478,7 +479,10 @@ export const EXAMPLES: DslExample[] = [ { kind: "plot", unit: "sample", cohorts: ["NSCLC"], allPoints: false, points: [], x: { source: "clinical", attribute: "SAMPLE_TYPE" }, y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" }, - colorBy: { source: "alteration", oql: "STK11: MUT; KEAP1: MUT" } } + colorBy: { source: "alteration", oql: "STK11: MUT; KEAP1: MUT" } }, + { kind: "plot", unit: "sample", cohorts: ["NSCLC"], + x: { source: "alteration", oql: "STK11: MUT, KEAP1: MUT, KRAS: G12C", by: "statement" }, + y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" } } ] } }, @@ -1658,13 +1662,19 @@ bin's width is shared out between the classes in it. That stays an aggregate, so may not disclose a case can still colour its plot. The key is capped at 12 classes. **OQL on an axis.** An alteration axis can be an OQL query instead of genes: -\`{"source":"alteration","oql":"KRAS: G12C; STK11: MUT"}\`. Each case is **Altered** when it matches -any entry — the portal's own reading of a multi-gene query — and **Unaltered** otherwise; every -case of the cohort is one or the other, so the unaltered are counted rather than missing. It is -the same OQL a cohort's \`oql\` leaf takes (mutation classes and protein changes, AMP/HOMDEL, -FUSION, DRIVER against the pinned OncoKB version, EXP/PROT thresholds), and it matches per patient -or per sample following the plot's \`unit\`. Because it is a categorical axis, it also works as -\`colorBy\`/\`ringBy\` and as a Cox or logistic covariate, where "Unaltered" is the reference level. +\`{"source":"alteration","oql":"KRAS: G12C, STK11: MUT"}\`, its statements separated by a comma, a +semicolon or a new line (as in a cohort's \`oql\` leaf). Each case is **Altered** when it matches any +statement — the portal's own reading of a multi-gene query — and **Unaltered** when it matches none +and is profiled for every statement's gene in the alteration types that statement asks about; a +case whose gene panel omits one of them is **Not profiled**, never counted as wild type (the +panel-aware rule of \`docs/PERFORMANCE.md\`). Every case of the cohort is one of the three, so none is +missing. With \`"by": "statement"\` each statement is its own group, named as written: a case matching +two is the pair ("KRAS: G12C + STK11: MUT"), three or more "3 or more lines", none Unaltered or Not +profiled by the same rule (at most ${OQL_STATEMENT_CAP} statements). It is the same OQL a cohort's \`oql\` leaf takes +(mutation classes and protein changes, AMP/HOMDEL, FUSION, DRIVER against the pinned OncoKB version, +EXP/PROT thresholds), and it matches per patient or per sample following the plot's \`unit\`. +Because it is a categorical axis, it also works as \`colorBy\`/\`ringBy\` and as a Cox or logistic +covariate, where "Unaltered" is the reference level. OQL is for alteration axes only: a molecular axis stays one gene's value. **Points.** Every case is a point when there are at most ${POINTS_CAP_TEXT} of them and every study may show @@ -1905,7 +1915,8 @@ export const AGENT_GRAMMAR = `Document shape: // | {"source":"molecular","profileType":P,"gene":G,"log":?} // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|driver"?} // | {"source":"alteration","genes":[G,...(2-20)],"by":"mutated"?} // any-of-N as one category; only "mutated" - // | {"source":"alteration","oql":"KRAS: G12C; STK11: MUT"} // Altered (matches any entry) vs Unaltered + // | {"source":"alteration","oql":"KRAS: G12C, STK11: MUT","by":"any|statement"?} + // // any: Altered / Unaltered / Not profiled; statement: one group per statement (max 6) // two numeric axes -> scatter (binned; every case a point up to ${POINTS_CAP_TEXT}, else sparse-cell outliers); one categorical -> box; // two categorical -> a contingency table {"kind":"attrition","cohorts":[names?(max 6)]}, diff --git a/lib/gene-query.ts b/lib/gene-query.ts index 5236296..512ca7d 100644 --- a/lib/gene-query.ts +++ b/lib/gene-query.ts @@ -2,7 +2,7 @@ // (each line or ";"-separated entry a gene and what counts as altered in it) or a plain gene list // (cBioPortal's, where `[A B]` is one merged row). Pure and browser-safe; the symbols it finds are // checked against the warehouse's gene table separately (POST /api/genes). -import { parseOql, type OqlSpec } from "./oql-parse.ts"; +import { parseOql, OQL_SEPARATOR, type OqlSpec } from "./oql-parse.ts"; import { parseGeneList, geneSymbols, normalizeGeneTracks, MAX_GENE_TRACKS, type GeneTrackSpec } from "./gene-tracks.ts"; export type GeneQueryMode = "oql" | "genes"; @@ -64,7 +64,7 @@ export function checkGeneQuery(text: string, mode: GeneQueryMode): QueryCheck { return { entries: [], genes: [], error: String((e as Error).message || e) }; } } - const parts = text.split(/[;\n]/).flatMap(splitBareGenes); + const parts = text.split(OQL_SEPARATOR).flatMap(splitBareGenes); const entries: QueryEntry[] = parts.map((p) => { const head = (p.match(/^([A-Za-z0-9.-]+)/)?.[1] || "").toUpperCase(); try { diff --git a/lib/oql-parse.ts b/lib/oql-parse.ts index 1fa8340..dc99b8e 100644 --- a/lib/oql-parse.ts +++ b/lib/oql-parse.ts @@ -27,11 +27,18 @@ const REJECT: Record<string, string> = { SOMATIC: "SOMATIC modifier is not supported (all stored mutations are somatic/unknown)" }; +/** what separates one OQL statement from the next: a semicolon, a new line or a comma */ +export const OQL_SEPARATOR = /[;\n,]/; +/** a query's statements as typed, trimmed, empty ones dropped */ +export function oqlStatements(q: string): string[] { + return q.split(OQL_SEPARATOR).map((s) => s.trim()).filter(Boolean); +} + // -> [{gene, specs: [{type:'MUT'|'CNA'|'SV'|'ANY', pchange?, cls?, cna?, driver}]}] export function parseOql(q: string): OqlEntry[] { if (typeof q !== "string" || !q.trim()) throw new Error("empty OQL"); if (q.length > 1000) throw new Error("OQL too long"); - const entries = q.split(/[;\n]/).map(s => s.trim()).filter(Boolean); + const entries = oqlStatements(q); // Each entry compiles to its own UNION-DISTINCT'd subquery (see below), so cost scales // roughly linearly rather than sitting at a wall near any particular number — 30 covers a // full pathway gene set (e.g. a 20-30 gene DDR panel) with room to spare, not a measured diff --git a/lib/plot-oql.ts b/lib/plot-oql.ts index 51b7358..93fe52a 100644 --- a/lib/plot-oql.ts +++ b/lib/plot-oql.ts @@ -1,4 +1,10 @@ -// The two categories an OQL plot axis takes. Browser-safe, so the method panel names an OQL +// The categories an OQL plot axis takes. Browser-safe, so the method panel names an OQL // covariate's reference level the way the engine chose it. export const OQL_ALTERED = "Altered"; export const OQL_UNALTERED = "Unaltered"; +/** a case not profiled for a statement it did not match: not altered as far as anyone knows, but never "Unaltered" */ +export const OQL_NOT_PROFILED = "Not profiled"; +/** one group per statement: a case matching three or more statements */ +export const OQL_SEVERAL = "3 or more lines"; +/** one group per statement: at most this many statements, so the groups (and their pairs) stay readable */ +export const OQL_STATEMENT_CAP = 6; diff --git a/lib/plot.ts b/lib/plot.ts index 7bd7692..653c02e 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -17,7 +17,14 @@ import { DB } from "./ch.ts"; import { STYLE } from "./oncoprint-style.ts"; import { clinicalTable, ATTR_NAME_RE } from "./custom-data.ts"; import { parseOql, oqlCasesSql } from "./oql.ts"; -import { OQL_ALTERED, OQL_UNALTERED } from "./plot-oql.ts"; +import { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP } from "./plot-oql.ts"; +import { oqlStatements } from "./oql-parse.ts"; +import { profiledCasesSql } from "./cohort2.ts"; + +/** the alteration types a statement must be profiled in before "not matched" can mean "not altered" */ +const OQL_PROFILED_TYPES: Partial<Record<string, ("MUTATION" | "CNA" | "SV")[]>> = { + MUT: ["MUTATION"], CNA: ["CNA"], SV: ["SV"], ANY: ["MUTATION", "CNA", "SV"] +}; /** what goes on one axis */ export type PlotAxis = @@ -32,19 +39,22 @@ export type PlotAxis = | { source: "alteration"; gene: string; by?: AlterationBy } | { source: "alteration"; genes: string[]; by?: "mutated" } /** - * An OQL query as one category: "Altered" when the case matches any entry, the portal's own - * reading of a multi-gene query, "Unaltered" otherwise. Per patient (any of its samples) or per - * sample, following the plot's unit. + * An OQL query, its statements separated by ";", a new line or ",". `by: "any"` (the default) + * is one category: "Altered" when the case matches any statement, the portal's own reading of a + * multi-gene query, "Unaltered" when it matches none and is profiled for every statement's gene, + * "Not profiled" otherwise. `by: "statement"` gives each statement its own group, a case + * matching two a group of the pair, three or more "3 or more lines", none "Unaltered" or "Not + * profiled" by the same rule. Per patient (any of its samples) or per sample, following the unit. */ - | { source: "alteration"; oql: string } + | { source: "alteration"; oql: string; by?: "any" | "statement" } | { source: "none" }; export type AlterationBy = "mutated" | "mutationType" | "cna"; const AXIS_SOURCES: readonly string[] = ["clinical", "molecular", "alteration", "none"] satisfies PlotAxis["source"][]; /** an alteration axis given as an OQL query, rather than as genes */ -function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string } { +function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string; by?: "any" | "statement" } { return a.source === "alteration" && "oql" in a && a.oql != null; } -export { OQL_ALTERED, OQL_UNALTERED }; +export { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP }; /** every gene named on an alteration axis, singular or set, uppercased */ function alterationGenes(a: { gene?: string; genes?: string[] }): string[] { @@ -119,8 +129,8 @@ export function axisLabel(a: PlotAxis): string { if (a.source === "clinical") return a.log ? "log10(" + a.attribute + " + 1)" : a.attribute; if (a.source === "molecular") return a.log ? "log10(" + a.gene + " " + a.profileType + " + 1)" : a.gene + " " + a.profileType; if (isOqlAxis(a)) { - const q = a.oql.trim().replace(/\s*[;\n]\s*/g, "; "); - return (q.length > 48 ? q.slice(0, 47) + "…" : q) + " (OQL)"; + const q = oqlStatements(a.oql).join("; "); + return (q.length > 48 ? q.slice(0, 47) + "…" : q) + (a.by === "statement" ? " (OQL, by line)" : " (OQL)"); } if (a.source === "alteration") { const genes = alterationGenes(a); @@ -158,10 +168,15 @@ export function validateAxis(a: PlotAxis, which: string): void { throw new Error(which + " axis: a molecular axis needs one gene. Plotting a whole profile would read every gene of every case; name the gene you mean."); } if (isOqlAxis(a)) { - if ("gene" in a || "genes" in a || "by" in a) - throw new Error(which + " axis: an OQL axis is the query alone; drop gene, genes and by"); + if ("gene" in a || "genes" in a) + throw new Error(which + " axis: an OQL axis is the query alone; drop gene and genes"); + if (a.by != null && a.by !== "any" && a.by !== "statement") + throw new Error(which + " axis: an OQL axis groups by \"any\" (Altered or not) or \"statement\" (one group per line); not " + JSON.stringify(a.by)); try { parseOql(String(a.oql)); } catch (e) { throw new Error(which + " axis: " + (e as Error).message, { cause: e }); } + const n = oqlStatements(String(a.oql)).length; + if (a.by === "statement" && n > OQL_STATEMENT_CAP) + throw new Error(which + " axis: one group per line takes at most " + OQL_STATEMENT_CAP + " lines; this query has " + n); return; } if (a.source === "alteration") { @@ -245,14 +260,30 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s } if (isOqlAxis(a)) { - // every case in the cohort, so "unaltered" is counted rather than missing - const matched = oqlCasesSql(a.oql, { oncokbPin: ctx.oncokbPin, profiles: ctx.profiles, study, db, unit: ctx.unit }); + // Every case in the cohort, so "unaltered" is counted rather than missing. Each statement is + // matched on its own, and a case it did not match is "Unaltered" only when it is profiled for + // that statement's gene in every alteration type the statement asks about (panel-aware, as a + // cohort split decides it); a sample whose panel omits the gene is never counted as wild type. + // each statement names its group as typed, its gene in capitals + const statements = oqlStatements(a.oql).map((st) => st.replace(/^[A-Za-z0-9.-]+/, (g) => g.toUpperCase()).replace(/\s+/g, " ")); const idCol = strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study); - return "SELECT " + idCol + " AS id, if(" + idCol + " IN (SELECT pid FROM (" + matched + ")), " + - sq(OQL_ALTERED) + ", " + sq(OQL_UNALTERED) + ") AS v\n" + - "FROM " + db + ".sample_derived sd\n" + - "WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "\n" + - "GROUP BY id"; + const o = { oncokbPin: ctx.oncokbPin, profiles: ctx.profiles, study, db, unit: ctx.unit }; + // the case column is `oql_case`, not `id`: ClickHouse would otherwise resolve `id` to another axis's alias + const matched = statements.map((st) => "oql_case IN (SELECT pid FROM (" + oqlCasesSql(st, o) + "))"); + const profiled = statements.map((st) => { + const types = [...new Set(parseOql(st)[0].specs.flatMap((sp) => OQL_PROFILED_TYPES[sp.type] || []))]; + const gene = parseOql(st)[0].gene; + // expression and protein thresholds read a value, not a panel: a case with no value is not matched, and not "not profiled" + return types.length ? types.map((t) => "oql_case IN (SELECT profiled_id FROM (" + profiledCasesSql(gene, t, { study, db, unit: ctx.unit }) + "))").join(" AND ") : "1"; + }); + const none = "if(" + profiled.map((p) => "(" + p + ")").join(" AND ") + ", " + sq(OQL_UNALTERED) + ", " + sq(OQL_NOT_PROFILED) + ")"; + const value = a.by === "statement" + ? "multiIf(length(hit) = 0, " + none + ", length(hit) = 1, hit[1], length(hit) = 2, concat(hit[1], ' + ', hit[2]), " + sq(OQL_SEVERAL) + ")" + : "if(length(hit) > 0, " + sq(OQL_ALTERED) + ", " + none + ")"; + return "SELECT oql_case AS id, " + value + " AS v FROM (\n" + + "SELECT oql_case, arrayFilter((l, m) -> m, [" + statements.map(sq).join(", ") + "], [" + matched.join(", ") + "]) AS hit FROM (\n" + + "SELECT DISTINCT " + idCol + " AS oql_case FROM " + db + ".sample_derived sd\n" + + "WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "))"; } if (a.source === "alteration") { const genes = alterationGenes(a), by = a.by || "mutated"; diff --git a/test/gene-query.test.ts b/test/gene-query.test.ts index 44285b4..32f7be4 100644 --- a/test/gene-query.test.ts +++ b/test/gene-query.test.ts @@ -35,6 +35,9 @@ describe("gene query: OQL", () => { test("the browser-safe parser is the one the engine uses", () => { assert.equal(parseOql, parsePure); }); + test("the engine reads a comma between statements as the dialog does: a cohort's or an axis's OQL alike", () => { + assert.deepEqual(parseOql("KRAS: G12C, STK11: MUT;EGFR: AMP\nTP53").map((e) => e.gene), ["KRAS", "STK11", "EGFR", "TP53"]); + }); }); describe("gene query: gene list", () => { diff --git a/test/plot.test.ts b/test/plot.test.ts index 8bd7d4e..c06dbef 100644 --- a/test/plot.test.ts +++ b/test/plot.test.ts @@ -212,7 +212,7 @@ test("log: true reads log10(v + 1), leaves out what has no such log, and says so assert.equal(axisLabel({ source: "molecular", profileType: "mrna", gene: "ESR1", log: true }), "log10(ESR1 mrna + 1)"); }); -test("an OQL axis: each case is Altered when it matches any entry, Unaltered otherwise", () => { +test("an OQL axis: Altered when a case matches any statement, Unaltered only when it is profiled for every one", () => { const oql = { source: "alteration" as const, oql: "KRAS: G12C; STK11: MUT" }; validateAxis(oql, "x"); assert.equal(axisKind(oql), "cat"); @@ -220,9 +220,26 @@ test("an OQL axis: each case is Altered when it matches any entry, Unaltered oth const sql = axisSelect(oql, ctx, "cat"); // every case of the cohort is a row, so the unaltered are counted, not missing assert.match(sql, /FROM db\.sample_derived sd/); - assert.match(sql, /'Altered', 'Unaltered'\) AS v/); + assert.match(sql, /if\(length\(hit\) > 0, 'Altered', if\(/); + assert.match(sql, /, 'Unaltered', 'Not profiled'\)\) AS v/); assert.match(sql, /hugo_gene_symbol = 'KRAS'/); assert.match(sql, /hugo_gene_symbol = 'STK11'/); + // panel-aware: a sample whose panel omits the gene is not profiled for it, never wild type + assert.match(sql, /sample_to_gene_panel_derived sp[\s\S]*alteration_type = 'MUTATION_EXTENDED'[\s\S]*WHERE gene = 'KRAS'/); + assert.match(sql, /WHERE gene = 'STK11'/); +}); + +test("an OQL axis by statement: a group per line, a pair for two, '3 or more lines' past that; commas separate lines", () => { + const oql = { source: "alteration" as const, oql: "kras: G12C, STK11: MUT\nKEAP1", by: "statement" as const }; + validateAxis(oql, "x"); + assert.equal(axisLabel(oql), "kras: G12C; STK11: MUT; KEAP1 (OQL, by line)"); + const sql = axisSelect(oql, ctx, "cat"); + assert.match(sql, /\['KRAS: G12C', 'STK11: MUT', 'KEAP1'\]/, "each group is named by its statement, the gene in capitals"); + assert.match(sql, /length\(hit\) = 2, concat\(hit\[1\], ' \+ ', hit\[2\]\), '3 or more lines'\)/); + // a bare gene is any alteration, so it must be profiled for all three kinds before "Unaltered" + assert.match(sql, /'COPY_NUMBER_ALTERATION'[\s\S]*WHERE gene = 'KEAP1'/); + assert.throws(() => validateAxis({ source: "alteration", oql: "A;B;C;D;E;F;G", by: "statement" }, "x"), /at most 6 lines; this query has 7/); + assert.throws(() => validateAxis({ source: "alteration", oql: "KRAS", by: "cna" } as never, "x"), /groups by "any" .* or "statement"/); }); test("an OQL axis matches samples in sample mode and patients in patient mode", () => { From bec0a90743606497e856fe5d54966b6a75869b3a Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 20:27:41 +0000 Subject: [PATCH 04/10] Plot axes: genes and OQL entered through the app's gene query dialog Every axis field that takes genes or OQL gets a button beside it that opens GeneQueryDialog, the app's one way of entering genes, with search over genes, aliases and gene sets, a check of every entry as it is typed, alias fixes, and recent and saved lists. That covers an alteration axis's OQL query or gene(s), a molecular axis's gene, and the colour/ring pickers, which share AxisPicker. The fields stay editable as text. - OQL opens one line per statement and applies them comma-separated. The limit is 6 statements when grouping by line, 30 otherwise, and the description says how lines are read for the chosen grouping. - A gene set opens in gene-list mode, capped at ALTERATION_GENE_SET_CAP (20). A molecular axis takes exactly one gene. ALTERATION_GENE_SET_CAP moves to the browser-safe lib/plot-oql.ts so the dialog can enforce it. The dialog's help says commas separate entries too. Verified in the browser: the OQL dialog opens with "KRAS: G12C\nRRAGC: MUT", and adding a line applies "KRAS: G12C, RRAGC: MUT, STK11: MUT" with by: "statement" kept; the gene dialog turns a TP53 axis into the set [TP53, KRAS]; no page errors. A new e2e check opens, edits and applies the OQL dialog. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/_genes/GeneQueryDialog.tsx | 2 +- app/lab/view-chrome.tsx | 52 +++++++++++++++++++++++++++++----- e2e.cjs | 23 +++++++++++++++ lib/plot-oql.ts | 7 +++-- lib/plot.ts | 6 ++-- 5 files changed, 76 insertions(+), 14 deletions(-) diff --git a/app/_genes/GeneQueryDialog.tsx b/app/_genes/GeneQueryDialog.tsx index a1108ee..41222bf 100644 --- a/app/_genes/GeneQueryDialog.tsx +++ b/app/_genes/GeneQueryDialog.tsx @@ -245,7 +245,7 @@ function GeneQueryBody({ onOpenChange, title, description, mode, value, onApply, <span className="block text-xs text-muted-foreground">{h.means}</span> </button> ))} - <p className="mt-1.5 mb-0 text-xs text-muted-foreground">Entries on separate lines (or split by “;”) are alternatives: a patient is altered if any of them matches. {OQL_REFUSED}</p> + <p className="mt-1.5 mb-0 text-xs text-muted-foreground">Entries on separate lines (or split by “;” or “,”) are alternatives: a patient is altered if any of them matches. {OQL_REFUSED}</p> </div> ) : ( <div className="flex flex-col gap-2 text-sm text-muted-foreground"> diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index 82ce335..d6a646e 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -8,7 +8,11 @@ import type { AnalysisSource, ViewParticipation } from "../../lib/types.ts"; import { VIEW_HELP } from "../../lib/view-help.ts"; import { SERIES, seriesStyle } from "../../lib/chart/style.ts"; import { Waiting } from "../_oncoprint/OncoprintCard.tsx"; -import { ArrowDown, ArrowUp, Ellipsis, ExternalLink, Lock, LockOpen, Pencil, Plus, Save, Split, StickyNote, Trash2 } from "lucide-react"; +import { ArrowDown, ArrowUp, Ellipsis, ExternalLink, ListChecks, Lock, LockOpen, Pencil, Plus, Save, Split, StickyNote, Trash2 } from "lucide-react"; +import { GeneQueryDialog } from "../_genes/GeneQueryDialog.tsx"; +import type { GeneQueryMode, QueryCheck } from "../../lib/gene-query.ts"; +import { oqlStatements } from "../../lib/oql-parse.ts"; +import { ALTERATION_GENE_SET_CAP, OQL_STATEMENT_CAP } from "../../lib/plot-oql.ts"; import { Button } from "../_ui/button"; import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; @@ -67,6 +71,23 @@ export function Participation({ p }: { p?: ViewParticipation }) { ); } +/** + * Beside a field that takes genes or OQL, the app's gene query dialog: search, a check of every + * entry as it is typed, alias fixes and remembered lists. The field stays editable as text; the + * dialog hands back what it applied. + */ +function QueryButton({ mode, value, max, title, description, busy, onApply }: { + mode: GeneQueryMode; value: string; max: number; title: string; description: string; busy?: boolean; + onApply: (text: string, check: QueryCheck) => void; +}) { + const [open, setOpen] = useState(false); + return <> + <Button type="button" variant="outline" size="icon-sm" className="size-ctl flex-none" disabled={busy} + aria-label={title} title={title + ": search genes and gene sets, and check every entry"} onClick={() => setOpen(true)}><ListChecks /></Button> + <GeneQueryDialog open={open} onOpenChange={setOpen} title={title} description={description} mode={mode} value={value} max={max} onApply={onApply} /> + </>; +} + /** * One axis of a plot. The source decides what else is asked for: a clinical attribute, a gene * in a molecular profile, or a gene's alterations. It is the same vocabulary the engine uses, @@ -90,8 +111,10 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang const set = (next: any) => onChange(next); const w = stacked ? "w-full max-w-none" : "max-w-44"; /** a control with its name above it when stacked; inline, the toolbar's side label names the run */ - const field = (name: string, control: React.ReactNode) => - stacked ? <div className="axfield flex flex-col gap-1"><span className="axfield-name text-sm font-semibold text-foreground">{name}</span>{control}</div> : control; + const field = (name: string, control: React.ReactNode, extra?: React.ReactNode) => + stacked ? <div className="axfield flex flex-col gap-1"><span className="axfield-name text-sm font-semibold text-foreground">{name}</span> + {extra ? <div className="flex min-w-0 gap-1">{control}{extra}</div> : control}</div> + : <>{control}{extra}</>; const srcName: Record<string, string> = { clinical: "Clinical attribute", molecular: "Molecular profile", alteration: "Genes" }; return ( <span className={stacked ? "axispick-stack flex flex-col gap-2.5" : "axispick"}> @@ -128,7 +151,9 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang {!profiles.includes(a.profileType || "") && <option value={a.profileType || ""}>{a.profileType || "(none)"}</option>} {profiles.map((pt) => <option key={pt} value={pt}>{pt}</option>)} </OptionSelect>)} - {field("Gene", <GeneInput gene={a.gene || ""} busy={busy} className={stacked ? w : undefined} onCommit={(gene) => set({ ...a, gene })} />)} + {field("Gene", <GeneInput gene={a.gene || ""} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(gene) => set({ ...a, gene })} />, + <QueryButton mode="genes" value={a.gene || ""} max={1} busy={busy} title="Gene for this axis" + description="The gene whose value in the profile this axis reads." onApply={(_t, c) => set({ ...a, gene: c.genes[0] || "" })} />)} </>} {src === "alteration" && (() => { const isOql = a.oql != null; @@ -160,8 +185,8 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang return <> {stacked && field("Group cases by", groupBy)} {field(isOql ? "OQL query" : "Gene(s)", isOql - ? <OqlInput query={a.oql} busy={busy} className={stacked ? w : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} /> - : <GeneSetInput genes={genes} busy={busy} className={stacked ? w : undefined} + ? <OqlInput query={a.oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} /> + : <GeneSetInput genes={genes} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(next) => { // A gene SET (2+) only supports "mutated vs not" — combining several genes' // mutation types or copy-number states into one category has no honest reading, @@ -170,7 +195,20 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang const by = next.length > 1 && a.by && a.by !== "mutated" ? "mutated" : a.by; set(next.length > 1 ? { ...a, gene: undefined, genes: next, by } : { ...a, genes: undefined, gene: next[0] || "", by }); - }} />)} + }} />, + isOql + ? <QueryButton mode="oql" value={oqlStatements(a.oql).join("\n")} busy={busy} title="OQL for this axis" + max={a.by === "statement" ? OQL_STATEMENT_CAP : 30} + description={a.by === "statement" ? "Each line is its own group; a case matching two is their pair. A bare gene means any alteration in it." + : "A case is Altered if any line matches. A bare gene means any alteration in it; add OQL to be specific."} + onApply={(text) => set({ ...a, source: "alteration", oql: oqlStatements(text).join(", ") })} /> + : <QueryButton mode="genes" value={genes.join(" ")} max={ALTERATION_GENE_SET_CAP} busy={busy} title="Genes for this axis" + description="One gene, or several: a case counts as mutated when any of them is. For anything finer, group cases by OQL." + onApply={(_t, c) => { + const next = c.genes; + const by = next.length > 1 && a.by && a.by !== "mutated" ? "mutated" : a.by; + set(next.length > 1 ? { ...a, gene: undefined, genes: next, by } : { ...a, genes: undefined, gene: next[0] || "", by }); + }} />)} {!stacked && groupBy} </>; })()} diff --git a/e2e.cjs b/e2e.cjs index 9da3c24..dbe8551 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -611,6 +611,29 @@ async function main() { } catch (e) { soloWhat = String(e.message).split("\n")[0]; } ok("clicking a layer's button shows it alone, Esc shows every layer, and the document is untouched", soloOk, soloWhat); + // An OQL axis is edited through the app's gene query dialog: it opens with one line per + // statement, and what it applies is the axis's query. + let oqlWhat = "no OQL axis", oqlOk = false; + try { + await page.goto(BASE + "/lab?tab=analysis&doc=" + encodeURIComponent(Buffer.from(JSON.stringify({ + study: "msk_chord_2024", cohorts: { All: { all: [] } }, + views: [{ kind: "plot", cohorts: ["All"], x: { source: "alteration", oql: "KRAS: G12C, STK11: MUT", by: "statement" }, + y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" } }] + })).toString("base64"))); + await page.waitForSelector(".plot-side", { timeout: 300000 }); + await page.getByRole("button", { name: "OQL for this axis" }).click(); + const dlg = page.locator("[role=dialog]"); + await dlg.waitFor({ timeout: 30000 }); + const opened = await dlg.locator("textarea").inputValue(); + await dlg.locator("textarea").fill("KRAS: G12C\nSTK11: MUT\nKEAP1: MUT"); + await dlg.getByRole("button", { name: /^Apply/ }).click({ timeout: 30000 }); + await page.waitForFunction(() => /KEAP1/.test(JSON.parse(document.querySelector("#docjson").value).views[0].x.oql), null, { timeout: 30000 }); + const x = await page.evaluate(() => JSON.parse(document.querySelector("#docjson").value).views[0].x); + oqlOk = opened === "KRAS: G12C\nSTK11: MUT" && x.oql === "KRAS: G12C, STK11: MUT, KEAP1: MUT" && x.by === "statement"; + oqlWhat = "opened " + JSON.stringify(opened) + " -> axis " + JSON.stringify(x); + } catch (e) { oqlWhat = String(e.message).split("\n")[0]; } + ok("a plot's OQL axis is edited in the gene query dialog, and its lines become the axis's statements", oqlOk, oqlWhat); + // The oncoprint's gear keeps ONLY the link out to cBioPortal; genes moved // to a named button and a modal, so the gear is not where editing hides. let gearSplit = "no oncoprint rendered", gearOk = false; diff --git a/lib/plot-oql.ts b/lib/plot-oql.ts index 93fe52a..3eecde3 100644 --- a/lib/plot-oql.ts +++ b/lib/plot-oql.ts @@ -1,5 +1,6 @@ -// The categories an OQL plot axis takes. Browser-safe, so the method panel names an OQL -// covariate's reference level the way the engine chose it. +// The categories an alteration plot axis takes, and its limits. Browser-safe, so the method panel +// names an OQL covariate's reference level the way the engine chose it, and the axis's gene dialog +// holds the reader to the limits the engine enforces. export const OQL_ALTERED = "Altered"; export const OQL_UNALTERED = "Unaltered"; /** a case not profiled for a statement it did not match: not altered as far as anyone knows, but never "Unaltered" */ @@ -8,3 +9,5 @@ export const OQL_NOT_PROFILED = "Not profiled"; export const OQL_SEVERAL = "3 or more lines"; /** one group per statement: at most this many statements, so the groups (and their pairs) stay readable */ export const OQL_STATEMENT_CAP = 6; +/** a gene-set alteration axis takes at most this many genes */ +export const ALTERATION_GENE_SET_CAP = 20; diff --git a/lib/plot.ts b/lib/plot.ts index 653c02e..6ab959e 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -17,7 +17,7 @@ import { DB } from "./ch.ts"; import { STYLE } from "./oncoprint-style.ts"; import { clinicalTable, ATTR_NAME_RE } from "./custom-data.ts"; import { parseOql, oqlCasesSql } from "./oql.ts"; -import { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP } from "./plot-oql.ts"; +import { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP, ALTERATION_GENE_SET_CAP } from "./plot-oql.ts"; import { oqlStatements } from "./oql-parse.ts"; import { profiledCasesSql } from "./cohort2.ts"; @@ -54,7 +54,7 @@ const AXIS_SOURCES: readonly string[] = ["clinical", "molecular", "alteration", function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string; by?: "any" | "statement" } { return a.source === "alteration" && "oql" in a && a.oql != null; } -export { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP }; +export { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP, ALTERATION_GENE_SET_CAP }; /** every gene named on an alteration axis, singular or set, uppercased */ function alterationGenes(a: { gene?: string; genes?: string[] }): string[] { @@ -62,8 +62,6 @@ function alterationGenes(a: { gene?: string; genes?: string[] }): string[] { } /** a patient-mode category for a patient whose samples carry different values */ export const MULTIPLE = "Multiple"; -/** a gene-set alteration axis takes at most this many genes */ -export const ALTERATION_GENE_SET_CAP = 20; export type AxisKind = "num" | "cat" | "none"; export type PlotShape = "scatter" | "box" | "table" | "waterfall"; From d14e036d71e3284f63a48fe0d8bf763e3d3b26b1 Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 21:45:02 +0000 Subject: [PATCH 05/10] Plot axes read the study's own profiles: one Data choice, and what an axis is follows the profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An axis is now a clinical attribute, one of the study's own molecular profiles, or a query across genes. A profile axis, {source: "profile", profile, gene}, is read as the study describes the profile in genetic_profile (alteration type and datatype), never from its name: - a mutation profile (MAF) by mutation type, or `by: "mutated"`; - a structural-variant profile by whether the case has one; - a discrete copy-number profile by its own calls (Deep deletion, Shallow deletion, Diploid, Gain, Amplification, drawn in that order), not only the ±2 events; - a continuous one (mRNA, z-scores, log2 copy number, protein) as a number. lib/profile-catalog.ts (browser-safe) holds that reading, and lib/plot-profile.ts resolves a profile axis against every study a view draws over into the forms the engine already runs (molecular with `discrete`, alteration with `by`). A profile a study lacks is refused with the list it has, and one two pooled studies describe differently is refused. Generic-assay profiles are offered but disabled until they get an entity picker. Plots, Cox and logistic all resolve their axes this way. Panel-aware single-gene axes: a case with no mutation, high-level copy-number call or SV in the gene is "Not mutated" / "Neutral" / "No structural variant" only when it is profiled for that gene in that alteration type; otherwise it is "Not profiled", as the OQL axis already is. In TCGA BRCA, 18 samples outside the mutation profile were counted as not mutated before. The page: studyProfiles() also returns each study's catalog; Explore returns it (profileCatalog), and the lab and portal catalogs carry it. AxisPicker's Data is one grouped list: Clinical, then each alteration type with the study's own profile names under it, then Across genes (OQL, gene set). The fields that follow come from the choice: attribute; gene (with the gene query dialog) plus "Plot mutations by" for a mutation profile; or the query. The Results page picks its expression and discrete copy-number profiles by type and datatype, no longer by matching profile names, and its Plots tab uses profile axes. The co-expression pair plot uses profile axes too. The DSL reference documents the profile axis, and a gallery example shows it. Verified on brca_tcga_pan_can_atlas_2018: - ERBB2 GISTIC x mRNA z-scores: Diploid 480, Shallow deletion 260, Gain 206, Amplification 122. - TP53 mutation type: Missense 209, Truncating 102, Splice 24, Inframe 5, Multiple 4, Not mutated 706, Not profiled 18. - ERBB2 SV: 15 / 1,037 / 18 not profiled. - msk_chord's `cna` profile shows its five levels, plus one sample whose stored value is -1.5, shown as written. - The quoted APC Cox and logistic results in the gallery are unchanged. In the browser, picking Mutations in the Data list rewrites the axis and redraws, and the Cox covariates use the same picker; no page errors. Unit tests cover the reading, resolution, refusals and SQL; the new example's cohort is pinned. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/_oncoprint/OncoprintCard.tsx | 5 +- app/api/explore/route.ts | 9 +- app/lab/page.tsx | 4 +- app/lab/plot-controls.tsx | 15 ++- app/lab/portal-bodies.tsx | 2 +- app/lab/use-catalogs.ts | 9 +- app/lab/use-view-options.ts | 5 +- app/lab/view-cards.tsx | 23 ++-- app/lab/view-chrome.tsx | 198 ++++++++++++++++-------------- app/portal/PortalTabs.tsx | 2 +- app/portal/comparison/page.tsx | 4 +- app/portal/results/page.tsx | 9 +- app/portal/study-known.ts | 16 ++- docs/dsl.md | 173 ++++++++++++++++++-------- e2e.cjs | 2 +- lib/analysis.ts | 22 +++- lib/dsl-reference.ts | 49 ++++++-- lib/plot-draw.ts | 2 +- lib/plot-profile.ts | 42 +++++++ lib/plot.ts | 68 ++++++++-- lib/profile-catalog.ts | 62 ++++++++++ lib/types.ts | 3 + lib/views/cox.ts | 12 +- lib/views/logistic.ts | 14 ++- lib/views/plot.ts | 23 +++- lib/views/run.ts | 3 +- lib/views/shared.ts | 6 + test/fixtures/example-counts.json | 5 +- test/plot-profile.test.ts | 84 +++++++++++++ test/plot.test.ts | 2 +- 30 files changed, 657 insertions(+), 216 deletions(-) create mode 100644 lib/plot-profile.ts create mode 100644 lib/profile-catalog.ts create mode 100644 test/plot-profile.test.ts diff --git a/app/_oncoprint/OncoprintCard.tsx b/app/_oncoprint/OncoprintCard.tsx index 3b8e5da..eac249a 100644 --- a/app/_oncoprint/OncoprintCard.tsx +++ b/app/_oncoprint/OncoprintCard.tsx @@ -6,6 +6,7 @@ // document) and hands back the next view; whether that re-runs is the page's to decide, with // the rule from view-display.ts: the figure's own display options and sort never re-run. import React, { useState } from "react"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; import OncoprintFigure, { type ColumnSelection } from "./OncoprintFigure.tsx"; import { figureDisplayOf, withFigureDisplay, sortKeysOf, withSortKeys, sortKeyText } from "./view-display.ts"; import { normalizeGeneTracks, type GeneTrackSpec } from "../../lib/gene-tracks.ts"; @@ -199,7 +200,9 @@ export function TracksMenu({ attrs, profiles, clinical, heatmap, cohorts, derive // ---- the card ---------------------------------------------------------------------------------- -export interface TrackCatalog { attrs: { attribute: string; kind: string; patients: number }[]; profiles: string[]; loading: boolean } +export interface TrackCatalog { attrs: { attribute: string; kind: string; patients: number }[]; profiles: string[]; loading: boolean; + /** every profile as the studies describe them, for a plot axis's Data picker */ + profileCatalog?: ProfileInfo[] } export function OncoprintCard({ view, spec, busy, cohortNames, catalog, onView, onGenes, onCreateCohort, onSaveFeature, onContract, contract, note, studies, uploads, onUndo }: { /** the view as the document holds it */ diff --git a/app/api/explore/route.ts b/app/api/explore/route.ts index bb48ff0..a054154 100644 --- a/app/api/explore/route.ts +++ b/app/api/explore/route.ts @@ -8,6 +8,7 @@ import { cacheLife } from "next/cache"; import { pack, throughCache, dataVersion } from "../../../lib/next-cache.ts"; import { studyProfiles, resolveStudy } from "../../../lib/analysis.ts"; import { chQuery } from "../../../lib/ch.ts"; +import type { ProfileInfo } from "../../../lib/profile-catalog.ts"; import { withRequestCustomData } from "../../../lib/custom-data-server.ts"; import { assertCustomDataPresent } from "../../../lib/custom-data.ts"; import { idSetRefsIn } from "../../../lib/id-sets.ts"; @@ -112,9 +113,13 @@ async function exploreFor(body: any) { if (!local.length) { return remoteParts.length === 1 ? remoteParts[0] : mergeExplore(remoteParts); } + const { catalog: _c, ...oneProfiles } = profilesByStudy.get(local[0]) || {}; const localResult = local.length === 1 ? await buildExplore({ study: local[0], draft, oncokbPin: pin, geneSearch, attributes, geneCharts, derived, - ...profilesByStudy.get(local[0]), conn: connByStudy.get(local[0]) }) + ...oneProfiles, conn: connByStudy.get(local[0]) }) : await buildExploreFederated({ studies: local, draft, oncokbPin: pin, profilesByStudy, connByStudy, geneSearch, attributes, geneCharts, derived }); - return remoteParts.length ? mergeExplore([localResult, ...remoteParts]) : localResult; + // every local study's profiles as it describes them, once per profile id, for a plot axis's Data picker + const profileCatalog = [...new Map(local.flatMap((st) => (profilesByStudy.get(st)?.catalog || []).map((p: ProfileInfo) => [p.profile, p] as const))).values()]; + const result = remoteParts.length ? mergeExplore([localResult, ...remoteParts]) : localResult; + return { ...result, profileCatalog }; } diff --git a/app/lab/page.tsx b/app/lab/page.tsx index 321dd3f..c48e743 100644 --- a/app/lab/page.tsx +++ b/app/lab/page.tsx @@ -353,7 +353,7 @@ export default function AnalysisPage() { const { setAddChartOpen, addClause, addClauses, addDraftToAnalysis, addMode, analyzeDraft, barDef, clearDraft, closeAddCohort, exMsg, exSel, exStudies, explore, exploreBusy, fetchExplore, geneCharts, openAddCohort, openCustomManager, parseDraft, removeDraftClause, setExStudies, setExploreStudies } = exploreState; const openTab = (t: typeof TABS[number]) => { goTab(t); if (t === "explore" && !explore) fetchExplore(); }; const viewOptions = useViewOptions({ cat, catLoading, doc, explore, exploreBusy, result, trackAttrs, trackAttrsLoading, studies }); - const { VIEW_KINDS, assayOptions, attrsLoading, cantDrawCohort, catalogAttrs, contProfile, drawsOwnHead, geneChoices, numericAttrs, profileOptions, viewAttrs, viewEvents, viewGenes } = viewOptions; + const { VIEW_KINDS, assayOptions, attrsLoading, cantDrawCohort, catalogAttrs, contProfile, drawsOwnHead, geneChoices, numericAttrs, profileOptions, profileCatalog, viewAttrs, viewEvents, viewGenes } = viewOptions; const viewEdits = useViewEdits({ VIEW_KINDS, assayOptions, busy, cantDrawCohort, catalogAttrs, contProfile, doc, docs, numericAttrs, profileOptions, result, setChooser, setToast, study, viewAttrs, viewEvents, viewGenes, viewIds }); const { addCohortToView, addView, cohortChoice, docView, emptyViewHint, maxCohortsOf, moveView, ocView, openCohortChooser, patchView, removeView, removeViews, setViewAttrs, setViewCohorts, setViewGenes, updateView, updateViewLocal, viewTitle } = viewEdits; @@ -361,7 +361,7 @@ export default function AnalysisPage() { /** what Explore's sections read and the edits they can make (app/lab/explore-sections.tsx) */ const exploreCtx: ExploreCtx = { ...exploreState, menuStore: menuOpenRef.current, showClauseData, studies }; /** what every view card reads and the edits it can make (app/lab/view-cards.tsx) */ - const viewCtx = { VIEW_KINDS, docHistory, study, assayOptions, attrsLoading, busy, cat, catLoading, catalogAttrs, cohortChoice, cohortFromSelection, contracts, doc, docView, drawsOwnHead, explore, featureFromSelection, maxCohortsOf, menuOpenRef, moveView, noteEdit, ocView, openCohortChooser, openGenes, patchView, pendingKeys, profileOptions, removeView, setContracts, setViewAttrs, setViewCohorts, trackAttrs, trackAttrsLoading, updateView, updateViewLocal, viewEvents, viewIds }; + const viewCtx = { VIEW_KINDS, docHistory, study, assayOptions, attrsLoading, busy, cat, catLoading, catalogAttrs, cohortChoice, cohortFromSelection, contracts, doc, docView, drawsOwnHead, explore, featureFromSelection, maxCohortsOf, menuOpenRef, moveView, noteEdit, ocView, openCohortChooser, openGenes, patchView, pendingKeys, profileOptions, profileCatalog, removeView, setContracts, setViewAttrs, setViewCohorts, trackAttrs, trackAttrsLoading, updateView, updateViewLocal, viewEvents, viewIds }; /** what the Analysis tab's sections read and the actions they take (app/lab/analysis-sections.tsx) */ const analysisCtx: AnalysisCtx = { lockCohorts, unlockCohort, unlockAll, canSave, analysisStudies, requestLogin, signIn: (!customState.enabled ? "off" : !loggedIn ? "locked" : "ready") as SignInState, VIEW_KINDS, addView, docHistory, applyText, busy, catalogAttrs, clearAnalysis, doc, docDialog, docText, docs, explore, fetchExplore, cohortCells, cohortSaveState, cohortSource, openCohortCells, emptyViewHint, genesFor, goTab, loggedIn, menuOpenRef, moveView, openCohortChooser, openCohortEdit, openCustomManager, openWorkspaceSave, removeCohort, removePartitionNamed, removeView, removeViews, renamePartitionRow, result, saveCohortToLibrary, saved, setCdFocus, setGenesFor, setPartitionDef, setUpgradeNote, setViewGenes, showClauseData, spaces, splitCohortBy, splitsFlat, studyNames, tab, updateFromSaved, upgradeNote, viewCtx, viewIds, viewTitle }; // what the page offers the researcher's AI assistant (app/lab/agent-tools.ts), current every render diff --git a/app/lab/plot-controls.tsx b/app/lab/plot-controls.tsx index 7625114..4b9ef44 100644 --- a/app/lab/plot-controls.tsx +++ b/app/lab/plot-controls.tsx @@ -13,6 +13,7 @@ import { Choice, OptionSelect } from "../_ui/choice.tsx"; import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; import { CHIP, FIELD } from "../_ui/toolbar.ts"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; import { DEFAULT_POINT_LAYERS, POINTS_CAP, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../../lib/plot-layers.ts"; type Update = (fn: (view: any) => any) => void; @@ -21,7 +22,7 @@ const CAP = POINTS_CAP.toLocaleString("en-US"); function LogToggle({ spec, side, numeric, busy, update }: { spec: any; side: "x" | "y"; numeric: boolean; busy: boolean; update: Update }) { const axis = spec[side]; - if (!numeric || !axis || !("source" in axis) || (axis.source !== "clinical" && axis.source !== "molecular")) return null; + if (!numeric || !axis || !("source" in axis) || (axis.source !== "clinical" && axis.source !== "molecular" && axis.source !== "profile")) return null; return ( <label className="chk text-md text-foreground" title="draw this axis on a log scale — log10(value + 1), so zero stays on the axis and a negative value is left out"> <Checkbox checked={!!axis.log} disabled={busy} @@ -112,13 +113,15 @@ function PointLayers({ layers, plot, cohortOptions, axisName, busy, onChange }: ); } -export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohortOptions, update, children }: { +export function PlotControls({ spec, plot, attrs, catalog, loading, busy, cohortOptions, update, children }: { /** the view as the document holds it */ spec: any; /** the last result's plot: which shape is on screen, each axis's name and kind, how many cases it shows, its layers */ plot?: { shape: string; n: number; x: { kind: string; label: string }; y: { kind: string; label: string }; coverage?: { total: number }; layers?: { label: string; color: string; n: number; shown: string }[] }; - attrs: { attribute: string; kind: string }[]; profiles: string[]; loading?: boolean; busy: boolean; + attrs: { attribute: string; kind: string }[]; + /** the studies' profiles as they describe them, for each axis's Data picker */ + catalog: ProfileInfo[]; loading?: boolean; busy: boolean; /** the cohorts a point layer may be: every one but the plotted one and a split */ cohortOptions: { name: string; label: string; depth: number }[]; update: Update; @@ -145,7 +148,7 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor const boxTransposed = plot?.shape === "box" && plot.x.kind === "num"; const boxDrawn = plot?.shape === "box" && !boxTransposed; const axisFields = (side: "x" | "y") => <> - <AxisPicker side={side} stacked axis={spec[side]} attrs={attrs} profiles={profiles} loading={loading} busy={busy} + <AxisPicker side={side} stacked axis={spec[side]} attrs={attrs} catalog={catalog} loading={loading} busy={busy} onChange={(a) => update((vv) => ({ ...vv, [side]: a }))} /> <LogToggle spec={spec} side={side} numeric={plot?.[side].kind === "num"} busy={busy} update={update} /> </>; @@ -202,11 +205,11 @@ export function PlotControls({ spec, plot, attrs, profiles, loading, busy, cohor {(plot?.shape === "box" || drawn.length > 0) && <div className="viewtb mt-0 mb-2.5"> {plot?.shape === "box" && <><div className="tbz"> <span className="tbl" title="colours each group's violin and points by a category">colour samples by</span> - <AxisPicker side="fill" axis={spec.colorBy} allowNone categorical attrs={attrs} profiles={profiles} loading={loading} busy={busy} + <AxisPicker side="fill" axis={spec.colorBy} allowNone categorical attrs={attrs} catalog={catalog} loading={loading} busy={busy} onChange={(a) => update((vv) => { const n = { ...vv }; if (a) n.colorBy = a; else delete n.colorBy; return n; })} /> </div> <div className="tbz"> - <AxisPicker side="ring" axis={spec.ringBy} allowNone categorical attrs={attrs} profiles={profiles} loading={loading} busy={busy} + <AxisPicker side="ring" axis={spec.ringBy} allowNone categorical attrs={attrs} catalog={catalog} loading={loading} busy={busy} onChange={(a) => update((vv) => { const n = { ...vv }; if (a) n.ringBy = a; else delete n.ringBy; return n; })} /> </div></>} {drawn.length > 0 && <div className="tbz plot-solo"> diff --git a/app/lab/portal-bodies.tsx b/app/lab/portal-bodies.tsx index 374d14f..7cfbbec 100644 --- a/app/lab/portal-bodies.tsx +++ b/app/lab/portal-bodies.tsx @@ -187,7 +187,7 @@ export function CorrelatesBody({ v, docJson }: { v: ViewResult; docJson?: () => const [sel, setSel] = useState<CoRow | null>(null); const picked = sel && rows.includes(sel) ? sel : [...rows].sort((a, b) => a.q - b.q || Math.abs(b.rho ?? 0) - Math.abs(a.rho ?? 0))[0] || null; const plot = usePairPlot(docJson, r.cohort, picked ? { kind: "plot", unit: "patient", - x: { source: "molecular", profileType: r.profileType, gene: r.gene }, y: { source: "molecular", profileType: r.otherProfile, gene: picked.gene } } : null); + x: { source: "profile", profile: r.profileType, gene: r.gene }, y: { source: "profile", profile: r.otherProfile, gene: picked.gene } } : null); const cols: Col<CoRow>[] = [ { key: "gene", label: "Correlated Gene", value: (g) => g.gene, render: (g) => <strong>{g.gene}</strong> }, { key: "cytoband", label: "Cytoband", value: (g) => g.cytoband || null }, diff --git a/app/lab/use-catalogs.ts b/app/lab/use-catalogs.ts index a14807e..3aeeb55 100644 --- a/app/lab/use-catalogs.ts +++ b/app/lab/use-catalogs.ts @@ -9,6 +9,7 @@ import type { StudyAccess } from "../../lib/view-coverage.ts"; import { api } from "./api.ts"; import { customDataKey } from "./custom-data-client.ts"; import { isCustomAttr } from "../../lib/custom-data-names.ts"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; export type StudyInfo = { id: string; name: string; cancerType?: string; patients: number; samples: number; timelinePatients: number; @@ -23,10 +24,12 @@ export type TrackAttrs = { profiles: string[]; /** generic-assay profiles (the assays view) */ gaProfiles: string[]; + /** every profile as the studies describe them, once per profile id (a plot axis's Data picker) */ + profileCatalog: ProfileInfo[]; }; const NO_STUDIES: StudyInfo[] = []; -const NO_TRACKS: TrackAttrs = { key: "", attrs: [], profiles: [], gaProfiles: [] }; +const NO_TRACKS: TrackAttrs = { key: "", attrs: [], profiles: [], gaProfiles: [], profileCatalog: [] }; async function answer(res: Response) { const j = await res.json(); @@ -75,6 +78,7 @@ export function useTrackAttrs(result: AnalysisResult | null): { trackAttrs: Trac const seen = new Map<string, { kind: string; patients: number }>(); // attribute -> kind + how many carry it const profs = new Set<string>(); const gas = new Set<string>(); + const described = new Map<string, ProfileInfo>(); for (const j of js) if (j) { if (j.clinical) for (const a of j.clinical) { const prev = seen.get(a.attribute); @@ -82,6 +86,7 @@ export function useTrackAttrs(result: AnalysisResult | null): { trackAttrs: Trac } for (const p of j.profiles || []) if (p !== "cna") profs.add(p); // discrete cna isn't a heatmap for (const p of j.gaProfiles || []) gas.add(p); + for (const p of (j.profileCatalog || []) as ProfileInfo[]) if (!described.has(p.profile)) described.set(p.profile, p); } // the reader's own uploads first, then most-covered first, the way the portal orders its picker return { @@ -89,7 +94,7 @@ export function useTrackAttrs(result: AnalysisResult | null): { trackAttrs: Trac attrs: [...seen.entries()].map(([attribute, v]) => ({ attribute, ...v })) .sort((x, y) => Number(isCustomAttr(y.attribute)) - Number(isCustomAttr(x.attribute)) || y.patients - x.patients || (x.attribute < y.attribute ? -1 : 1)), - profiles: [...profs].sort(), gaProfiles: [...gas].sort(), + profiles: [...profs].sort(), gaProfiles: [...gas].sort(), profileCatalog: [...described.values()], } as TrackAttrs; }, }); diff --git a/app/lab/use-view-options.ts b/app/lab/use-view-options.ts index 233902b..807b6f5 100644 --- a/app/lab/use-view-options.ts +++ b/app/lab/use-view-options.ts @@ -9,6 +9,7 @@ import type { AnalysisDoc, AnalysisResult } from "../../lib/types.ts"; import { foldEventNames, treatmentEventOf } from "../../lib/event-names.ts"; import { LINE_TYPE } from "../../lib/treatment-lines.ts"; import type { ExploreResult } from "../../lib/types.ts"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; import type { StudyInfo, TrackAttrs } from "./use-catalogs.ts"; import { cantDraw, cohortStudies } from "../../lib/view-coverage.ts"; @@ -94,6 +95,8 @@ export function useViewOptions(page: ViewOptionsPage) { const attrsLoading = (): boolean => !catalogAttrs().length && (trackAttrsLoading || exploreBusy); /** continuous molecular profiles for an axis picker: Explore's if loaded, else the shared catalog's */ const profileOptions = (): string[] => (explore && explore.profiles && explore.profiles.length ? explore.profiles : trackAttrs.profiles); + /** every profile as the studies describe them, for a plot axis's Data picker: Explore's if loaded, else the shared catalog's */ + const profileCatalog = (): ProfileInfo[] => (explore && explore.profileCatalog && explore.profileCatalog.length ? explore.profileCatalog : trackAttrs.profileCatalog); const assayOptions = (): string[] => (explore && explore.gaProfiles && explore.gaProfiles.length ? explore.gaProfiles : trackAttrs.gaProfiles); const contProfile = (): string | null => { @@ -176,5 +179,5 @@ export function useViewOptions(page: ViewOptionsPage) { return [...new Set([...fromExplore, ...legacy])].slice(0, 60); }; - return { viewGenes, viewAttrs, viewEvents, catalogAttrs, numericAttrs, attrsLoading, profileOptions, assayOptions, contProfile, noSurvivalWhy, VIEW_KINDS, drawsOwnHead, GENE_VIEWS, geneChoices, cantDrawCohort }; + return { viewGenes, viewAttrs, viewEvents, catalogAttrs, numericAttrs, attrsLoading, profileOptions, profileCatalog, assayOptions, contProfile, noSurvivalWhy, VIEW_KINDS, drawsOwnHead, GENE_VIEWS, geneChoices, cantDrawCohort }; } diff --git a/app/lab/view-cards.tsx b/app/lab/view-cards.tsx index b597025..1824684 100644 --- a/app/lab/view-cards.tsx +++ b/app/lab/view-cards.tsx @@ -14,6 +14,7 @@ import { MethodPanel } from "./method-panel.tsx"; import { viewRequest } from "./use-analysis.ts"; import { AxisPicker, ViewHelpButton, SplitByNote, PickMenu, CohortPicker, DrawnCohorts, ViewHead, ViewMenu, Latency, SeriesSwatch, Participation } from "./view-chrome.tsx"; import { ViewFigure } from "./figure.tsx"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; import { PlotControls } from "./plot-controls.tsx"; import { Plus, Settings2, X } from "lucide-react"; import { Button } from "../_ui/button"; @@ -56,6 +57,8 @@ export interface ViewCtx { catalogAttrs: () => { attribute: string; kind: string }[]; attrsLoading: () => boolean; profileOptions: () => string[]; + /** the studies' profiles as they describe them, for a plot axis's Data picker */ + profileCatalog: () => ProfileInfo[]; assayOptions: () => string[]; viewEvents: () => EventOption[]; drawsOwnHead: (v: any) => boolean; @@ -124,9 +127,9 @@ function placement({ result, moveView, removeView, noteEdit }: ViewCtx, vi: numb /** A regression view's covariates: each an axis, with the reference level its ratio is measured * against once the fit has seen more than one level. */ -function CovariateList({ covariates, ratio, observedLevels, busy, attrs, profiles, loading, onChange, onRemove, onAdd }: { +function CovariateList({ covariates, ratio, observedLevels, busy, attrs, catalog, loading, onChange, onRemove, onAdd }: { covariates: any[]; ratio: "hazard" | "odds"; observedLevels: (ci: number) => string[]; busy: boolean; - attrs: { attribute: string; kind: string }[]; profiles: string[]; loading: boolean; + attrs: { attribute: string; kind: string }[]; catalog: ProfileInfo[]; loading: boolean; onChange: (ci: number, next: any) => void; onRemove: (ci: number) => void; onAdd: () => void; }) { return ( @@ -136,7 +139,7 @@ function CovariateList({ covariates, ratio, observedLevels, busy, attrs, profile return ( <span key={ci} className="covrow"> <AxisPicker side={"covariate " + (ci + 1)} axis={c} attrs={attrs} - profiles={profiles} loading={loading} busy={busy} + catalog={catalog} loading={loading} busy={busy} onChange={(next) => onChange(ci, next)} /> {levels.length > 1 && <label className="tbfield" title={(ratio === "hazard" ? "the level a hazard ratio" : "the level an odds ratio") + " is measured against; unset uses this covariate's default (mutated/not always compares against 'Not mutated', other categoricals use whichever level is most frequent)"}> ref <Choice size="ctl" value={c.reference && levels.includes(c.reference) ? c.reference : ""} disabled={busy} @@ -489,7 +492,7 @@ function SurvivalCard({ ctx, v, vi }: CardProps) { } function CoxCard({ ctx, v, vi }: CardProps) { - const { attrsLoading, busy, cat, catLoading, catalogAttrs, cohortChoice, doc, docView, menuOpenRef, patchView, profileOptions, trackAttrs } = ctx; + const { attrsLoading, busy, cat, catLoading, catalogAttrs, cohortChoice, doc, docView, menuOpenRef, patchView, profileCatalog, trackAttrs } = ctx; return <> {v.kind === "cox" && <> <ViewHead menuStore={menuOpenRef.current} v={v} title="Cox regression" cohorts={cohortChoice(vi, docView(vi))} {...placement(ctx, vi)} @@ -541,7 +544,7 @@ function CoxCard({ ctx, v, vi }: CardProps) { stratify by cohort </label> <CovariateList covariates={covariates} ratio="hazard" observedLevels={observedLevels} busy={busy} - attrs={catalogAttrs()} profiles={profileOptions()} loading={attrsLoading()} + attrs={catalogAttrs()} catalog={profileCatalog()} loading={attrsLoading()} onChange={setCovariate} onRemove={removeCovariate} onAdd={addCovariate} /> </div> ); @@ -578,7 +581,7 @@ function KindCard({ ctx, v, vi }: CardProps) { } function LogisticCard({ ctx, v, vi }: CardProps) { - const { attrsLoading, busy, catalogAttrs, cohortChoice, doc, docView, menuOpenRef, patchView, profileOptions, trackAttrs } = ctx; + const { attrsLoading, busy, catalogAttrs, cohortChoice, doc, docView, menuOpenRef, patchView, profileCatalog, trackAttrs } = ctx; return <> {v.kind === "logistic" && <> <ViewHead menuStore={menuOpenRef.current} v={v} title="Logistic regression" cohorts={cohortChoice(vi, docView(vi))} {...placement(ctx, vi)} @@ -614,7 +617,7 @@ function LogisticCard({ ctx, v, vi }: CardProps) { <div className="basis wraps"> <span className="meta">outcome</span> <AxisPicker side="outcome" axis={sv.outcome || { source: "clinical", attribute: "" }} attrs={catalogAttrs()} - profiles={profileOptions()} loading={attrsLoading()} busy={busy} + catalog={profileCatalog()} loading={attrsLoading()} busy={busy} onChange={(next) => patchView(vi, { outcome: next })} /> {!outcomeIsAlteration && <label className="tbfield" title="which observed value of the outcome counts as the event (1); every other observed value is treated as 'not event' and disclosed in the caveat. Required for a categorical outcome; not used for a numeric one, which must already be coded 0/1"> event <Input type="text" list={"outcome-levels-" + vi} disabled={busy} className="h-ctl w-36 bg-card text-md md:text-md" @@ -625,7 +628,7 @@ function LogisticCard({ ctx, v, vi }: CardProps) { </datalist> </label>} <CovariateList covariates={covariates} ratio="odds" observedLevels={observedLevels} busy={busy} - attrs={catalogAttrs()} profiles={profileOptions()} loading={attrsLoading()} + attrs={catalogAttrs()} catalog={profileCatalog()} loading={attrsLoading()} onChange={setCovariate} onRemove={removeCovariate} onAdd={addCovariate} /> </div> ); @@ -650,7 +653,7 @@ function SequenceKeyHint({ ctx, v, vi }: CardProps) { /** a numeric plot axis on a log scale: log10(v + 1), so a zero stays on the axis (lib/plot.ts) */ function PlotCard({ ctx, v, vi }: CardProps) { - const { attrsLoading, busy, catalogAttrs, cohortChoice, docView, menuOpenRef, profileOptions, updateView } = ctx; + const { attrsLoading, busy, catalogAttrs, cohortChoice, docView, menuOpenRef, profileCatalog, updateView } = ctx; // every cohort the document evaluates, but the plotted one and a split (which stands for several groups, not one set of cases) const ch = cohortChoice(vi, docView(vi), 1); const cohortOptions = ch.nodes.filter((nd) => nd.kind !== "split" && nd.name !== ch.drawn[0]) @@ -659,7 +662,7 @@ function PlotCard({ ctx, v, vi }: CardProps) { {v.kind === "plot" && <> <ViewHead menuStore={menuOpenRef.current} v={v} title="Plot" cohorts={ch} {...placement(ctx, vi)} sub={v.plot ? <>{v.plot.shape === "scatter" ? "density" : v.plot.shape === "box" ? "box" : "contingency"} · {v.plot.n.toLocaleString()} {(docView(vi).unit || "sample")}s</> : "two axes over the same cases"} /> - <PlotControls spec={docView(vi)} plot={v.plot} attrs={catalogAttrs()} profiles={profileOptions()} loading={attrsLoading()} busy={busy} + <PlotControls spec={docView(vi)} plot={v.plot} attrs={catalogAttrs()} catalog={profileCatalog()} loading={attrsLoading()} busy={busy} cohortOptions={cohortOptions} update={(fn) => updateView(vi, fn)}> <ViewFigure v={v} empty={v.plot?.note || "nothing to draw"} /> </PlotControls> diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index d6a646e..f67cb6c 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -13,6 +13,8 @@ import { GeneQueryDialog } from "../_genes/GeneQueryDialog.tsx"; import type { GeneQueryMode, QueryCheck } from "../../lib/gene-query.ts"; import { oqlStatements } from "../../lib/oql-parse.ts"; import { ALTERATION_GENE_SET_CAP, OQL_STATEMENT_CAP } from "../../lib/plot-oql.ts"; +import { plottableProfiles, profileGroup, profileUse, type ProfileInfo } from "../../lib/profile-catalog.ts"; +import { Select, SelectContent, SelectGroup, SelectItem, SelectLabel, SelectTrigger, SelectValue } from "../_ui/select"; import { Button } from "../_ui/button"; import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; @@ -89,25 +91,27 @@ function QueryButton({ mode, value, max, title, description, busy, onApply }: { } /** - * One axis of a plot. The source decides what else is asked for: a clinical attribute, a gene - * in a molecular profile, or a gene's alterations. It is the same vocabulary the engine uses, - * so what the reader picks is what the document says. + * One axis of a plot. Its Data is one choice from a grouped list: a clinical attribute, each of the + * study's own profiles under its alteration type (named as the study names it), or a query across + * genes (OQL, a gene set). What the choice needs next follows from it: an attribute; a gene, and for + * a mutation profile how to group the mutations; or the query. */ -export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChange, allowNone, categorical, stacked }: { +export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange, allowNone, categorical, stacked }: { side: string; axis: any; busy?: boolean; - attrs: { attribute: string; kind: string }[]; profiles: string[]; + attrs: { attribute: string; kind: string }[]; + /** the studies' profiles as they describe them (lib/profile-catalog.ts) */ + catalog: ProfileInfo[]; /** the attribute/profile catalog is still being fetched: its select waits with a loader */ loading?: boolean; onChange: (a: any) => void; /** a colour is optional in a way an axis is not */ allowNone?: boolean; - /** colours come from categories, so a molecular value is not on offer */ + /** colours come from categories, so a profile of numbers is not on offer */ categorical?: boolean; /** one labelled field per row at the box's full width (a plot's axis sidebar), not a toolbar run */ stacked?: boolean; }) { const a = axis || {}; - const src = axis ? (a.source || "clinical") : "none"; const set = (next: any) => onChange(next); const w = stacked ? "w-full max-w-none" : "max-w-44"; /** a control with its name above it when stacked; inline, the toolbar's side label names the run */ @@ -115,103 +119,109 @@ export function AxisPicker({ side, axis, attrs, profiles, busy, loading, onChang stacked ? <div className="axfield flex flex-col gap-1"><span className="axfield-name text-sm font-semibold text-foreground">{name}</span> {extra ? <div className="flex min-w-0 gap-1">{control}{extra}</div> : control}</div> : <>{control}{extra}</>; - const srcName: Record<string, string> = { clinical: "Clinical attribute", molecular: "Molecular profile", alteration: "Genes" }; + const profiles = plottableProfiles(catalog); + const offered = (p: ProfileInfo) => { const u = profileUse(p); return !categorical || u === "mutation" || u === "sv" || u === "discrete" || u === "assayCategory"; }; + // what the axis holds, as one of the Data list's values; a form the document gives directly + // (a molecular profile, a gene's alterations) shows as the profile it reads + const byUse = (u: string) => profiles.find((p) => profileUse(p) === u)?.profile; + const data: string = !axis ? "none" + : a.source === "clinical" ? "clinical" + : a.source === "profile" ? "profile:" + a.profile + : a.source === "molecular" ? "profile:" + a.profileType + : a.source === "alteration" && a.oql != null ? "oql" + : a.source === "alteration" && Array.isArray(a.genes) && a.genes.length > 1 ? "genes" + : a.source === "alteration" ? "profile:" + (byUse(a.by === "cna" ? "discrete" : a.by === "sv" ? "sv" : "mutation") || "") + : "clinical"; + const current = data.startsWith("profile:") ? catalog.find((p) => "profile:" + p.profile === data) : undefined; + const use = current ? profileUse(current) : null; + const gene = String(a.gene || (Array.isArray(a.genes) ? a.genes[0] : "") || (a.oql ? (/^\s*([A-Za-z0-9.-]+)/.exec(a.oql) || [])[1] || "" : "") || "TP53").toUpperCase(); + const choose = (v: string) => { + if (v === "none") return set(undefined); + if (v === "clinical") return set({ source: "clinical", attribute: a.attribute || (attrs[0] || {}).attribute || "" }); + if (v === "oql") return set({ source: "alteration", oql: a.oql || gene + ": MUT" }); + if (v === "genes") return set({ source: "alteration", genes: Array.isArray(a.genes) && a.genes.length > 1 ? a.genes : [gene], by: "mutated" }); + const p = catalog.find((x) => "profile:" + x.profile === v); + if (p) set({ source: "profile", profile: p.profile, gene, ...(profileUse(p) === "mutation" ? { by: a.by === "mutated" ? "mutated" : "mutationType" } : {}) }); + }; + // the Data list: the profiles grouped under their type, in the study's own order + const groups: { label: string; items: { value: string; label: string; disabled?: string }[] }[] = []; + groups.push({ label: "Clinical", items: [{ value: "clinical", label: "Clinical attribute" }] }); + for (const p of profiles) { + if (!offered(p)) continue; + const g = profileGroup(p); + const at = groups.find((x) => x.label === g) || (groups.push({ label: g, items: [] }), groups[groups.length - 1]); + const assay = p.type === "GENERIC_ASSAY"; + at.items.push({ value: "profile:" + p.profile, label: p.name, ...(assay ? { disabled: "generic assays (" + p.name + ") come to plot axes next" } : {}) }); + } + groups.push({ label: "Across genes", items: [{ value: "oql", label: "OQL query" }, { value: "genes", label: "Gene set (mutated in any)" }] }); + const dataPicker = loading && !catalog.length && !attrs.length ? <Waiting what="profiles" /> : ( + <Select value={data} onValueChange={choose} disabled={busy}> + <SelectTrigger size="sm" aria-label={side + " data"} title={current ? current.name : "what this axis measures"} + className={cn("h-ctl gap-1 bg-card px-2 text-md data-[size=sm]:h-ctl [&_svg:not([class*='size-'])]:size-3.5 [&>span]:truncate", w)}> + <SelectValue /> + </SelectTrigger> + <SelectContent className="max-h-96"> + {allowNone && <SelectItem value="none" className="text-md">none</SelectItem>} + {!groups.some((g) => g.items.some((i) => i.value === data)) && data !== "none" && + <SelectItem value={data} className="text-md">{data.replace(/^profile:/, "")} (not in this study)</SelectItem>} + {groups.map((g) => ( + <SelectGroup key={g.label}> + <SelectLabel className="text-xs font-semibold text-muted-foreground">{g.label}</SelectLabel> + {g.items.map((i) => <SelectItem key={i.value} value={i.value} disabled={!!i.disabled} title={i.disabled || i.label} className="text-md">{i.label}</SelectItem>)} + </SelectGroup> + ))} + </SelectContent> + </Select> + ); + const isOql = data === "oql", isSet = data === "genes"; + const genes: string[] = Array.isArray(a.genes) && a.genes.length ? a.genes : a.gene ? [a.gene] : []; + const commitGenes = (next: string[]) => set(next.length > 1 ? { source: "alteration", genes: next, by: "mutated" } : { source: "alteration", genes: next.slice(0, 1), by: "mutated" }); return ( <span className={stacked ? "axispick-stack flex flex-col gap-2.5" : "axispick"}> {!stacked && <span className="tbl">{side}</span>} - {field("Data type", <OptionSelect className={w} value={src} disabled={busy} title="what this axis measures" ariaLabel={side + " data type"} - onValueChange={(v) => { - const s = v; - // Keep whatever this axis last held for each source (spreading `a` rather than - // starting fresh), so switching away and back round-trips instead of always - // reverting to the catalog's first entry — a molecular axis already did this for - // its gene; clinical's attribute and molecular's profile now do too. - set(s === "none" ? undefined - : s === "clinical" ? { ...a, source: "clinical", attribute: a.attribute || (attrs[0] || {}).attribute || "" } - : s === "molecular" ? { ...a, source: "molecular", profileType: a.profileType || profiles[0] || "", gene: a.gene || "TP53" } - : { ...a, source: "alteration", gene: a.gene || "TP53", by: a.by || "mutationType" }); - }}> - {allowNone && <option value="none">none</option>} - <option value="clinical">clinical</option> - {!categorical && <option value="molecular">molecular</option>} - <option value="alteration">alteration</option> - </OptionSelect>)} - {src === "clinical" && loading && !attrs.length && field(srcName.clinical, <Waiting what="attributes" />)} - {src === "clinical" && !(loading && !attrs.length) && field(srcName.clinical, + {field("Data", dataPicker)} + {data === "clinical" && loading && !attrs.length && field("Clinical attribute", <Waiting what="attributes" />)} + {data === "clinical" && !(loading && !attrs.length) && field("Clinical attribute", <OptionSelect className={w} value={a.attribute || ""} disabled={busy} title="which clinical attribute" ariaLabel={side + " clinical attribute"} onValueChange={(v) => set({ ...a, attribute: v })}> {!attrs.some((x) => x.attribute === a.attribute) && <option value={a.attribute || ""}>{a.attribute || "(none)"}</option>} {attrs.map((x) => <option key={x.attribute} value={x.attribute}>{attributeLabel(x.attribute)}{x.kind === "num" ? "" : " (categorical)"}</option>)} </OptionSelect> )} - {src === "molecular" && <> - {field(srcName.molecular, loading && !profiles.length ? <Waiting what="profiles" /> : - <OptionSelect className={w} value={a.profileType || ""} disabled={busy} title="which molecular profile" ariaLabel={side + " molecular profile"} - onValueChange={(v) => set({ ...a, profileType: v })}> - {!profiles.includes(a.profileType || "") && <option value={a.profileType || ""}>{a.profileType || "(none)"}</option>} - {profiles.map((pt) => <option key={pt} value={pt}>{pt}</option>)} - </OptionSelect>)} - {field("Gene", <GeneInput gene={a.gene || ""} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(gene) => set({ ...a, gene })} />, - <QueryButton mode="genes" value={a.gene || ""} max={1} busy={busy} title="Gene for this axis" - description="The gene whose value in the profile this axis reads." onApply={(_t, c) => set({ ...a, gene: c.genes[0] || "" })} />)} - </>} - {src === "alteration" && (() => { - const isOql = a.oql != null; - const genes: string[] = a.genes && a.genes.length ? a.genes : a.gene ? [a.gene] : []; - const isSet = !isOql && genes.length > 1; - const groupBy = <OptionSelect className={w} value={isOql ? (a.by === "statement" ? "oqlEach" : "oql") : a.by || (isSet ? "mutated" : "mutationType")} disabled={busy} - ariaLabel={side + " alteration grouping"} - title={isOql ? (a.by === "statement" ? "a group for each line of the query; a case matching two is their pair, none Unaltered (or Not profiled)" - : "Altered when a case matches any line of the query; Unaltered when it matches none and is profiled for every gene, else Not profiled") - : isSet ? "a gene set is \"mutated vs not\", or write it as an OQL query" : "how the gene's alterations are grouped"} - onValueChange={(v) => { - const by = v; - // to OQL: the genes become a query (a bare gene is "any alteration"); from OQL: the - // query's first gene, so a round trip keeps the gene - if (by === "oql" || by === "oqlEach") { - const oql = isOql ? a.oql : genes.length ? genes.join("; ") : "TP53: MUT"; - set(by === "oqlEach" ? { source: "alteration", oql, by: "statement" } : { source: "alteration", oql }); - } - else if (isOql) set({ source: "alteration", gene: (/^\s*([A-Za-z0-9.-]+)/.exec(a.oql) || [])[1]?.toUpperCase() || "TP53", by }); - else set({ ...a, by }); - }}> - <option value="mutationType" disabled={isSet}>mutation type</option> + {current && use !== "assayNumber" && use !== "assayCategory" && <> + {field("Gene", <GeneInput gene={gene} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} + onCommit={(g) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: g })} />, + <QueryButton mode="genes" value={gene} max={1} busy={busy} title="Gene for this axis" + description={"The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query or Gene set under Data."} + onApply={(_t, c) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)} + {use === "mutation" && field("Plot mutations by", + <OptionSelect className={w} value={a.by === "mutated" ? "mutated" : "mutationType"} disabled={busy} ariaLabel={side + " mutations by"} + title="group the cases by their mutation's type, or by whether they have one" + onValueChange={(v) => set({ source: "profile", profile: current.profile, gene, by: v })}> + <option value="mutationType">mutation type</option> <option value="mutated">mutated vs not</option> - <option value="cna" disabled={isSet}>copy number</option> - <option value="oql">matches OQL (any line)</option> - <option value="oqlEach">OQL, one group per line</option> - </OptionSelect>; - // stacked, the grouping comes first: it decides whether the box below takes genes or a query - return <> - {stacked && field("Group cases by", groupBy)} - {field(isOql ? "OQL query" : "Gene(s)", isOql - ? <OqlInput query={a.oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} /> - : <GeneSetInput genes={genes} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} - onCommit={(next) => { - // A gene SET (2+) only supports "mutated vs not" — combining several genes' - // mutation types or copy-number states into one category has no honest reading, - // so switching to a set falls back to that rather than leaving an axis the - // engine would refuse. - const by = next.length > 1 && a.by && a.by !== "mutated" ? "mutated" : a.by; - set(next.length > 1 ? { ...a, gene: undefined, genes: next, by } - : { ...a, genes: undefined, gene: next[0] || "", by }); - }} />, - isOql - ? <QueryButton mode="oql" value={oqlStatements(a.oql).join("\n")} busy={busy} title="OQL for this axis" - max={a.by === "statement" ? OQL_STATEMENT_CAP : 30} - description={a.by === "statement" ? "Each line is its own group; a case matching two is their pair. A bare gene means any alteration in it." - : "A case is Altered if any line matches. A bare gene means any alteration in it; add OQL to be specific."} - onApply={(text) => set({ ...a, source: "alteration", oql: oqlStatements(text).join(", ") })} /> - : <QueryButton mode="genes" value={genes.join(" ")} max={ALTERATION_GENE_SET_CAP} busy={busy} title="Genes for this axis" - description="One gene, or several: a case counts as mutated when any of them is. For anything finer, group cases by OQL." - onApply={(_t, c) => { - const next = c.genes; - const by = next.length > 1 && a.by && a.by !== "mutated" ? "mutated" : a.by; - set(next.length > 1 ? { ...a, gene: undefined, genes: next, by } : { ...a, genes: undefined, gene: next[0] || "", by }); - }} />)} - {!stacked && groupBy} - </>; - })()} + </OptionSelect>)} + </>} + {isOql && <> + {field("Group cases by", + <OptionSelect className={w} value={a.by === "statement" ? "statement" : "any"} disabled={busy} ariaLabel={side + " OQL grouping"} + title={a.by === "statement" ? "a group for each line of the query; a case matching two is their pair, none Unaltered (or Not profiled)" + : "Altered when a case matches any line of the query; Unaltered when it matches none and is profiled for every gene, else Not profiled"} + onValueChange={(v) => set(v === "statement" ? { source: "alteration", oql: a.oql, by: "statement" } : { source: "alteration", oql: a.oql })}> + <option value="any">any line (Altered or not)</option> + <option value="statement">one group per line</option> + </OptionSelect>)} + {field("OQL query", <OqlInput query={a.oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} />, + <QueryButton mode="oql" value={oqlStatements(a.oql || "").join("\n")} busy={busy} title="OQL for this axis" + max={a.by === "statement" ? OQL_STATEMENT_CAP : 30} + description={a.by === "statement" ? "Each line is its own group; a case matching two is their pair. A bare gene means any alteration in it." + : "A case is Altered if any line matches. A bare gene means any alteration in it; add OQL to be specific."} + onApply={(text) => set({ ...a, source: "alteration", oql: oqlStatements(text).join(", ") })} />)} + </>} + {isSet && field("Genes", <GeneSetInput genes={genes} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={commitGenes} />, + <QueryButton mode="genes" value={genes.join(" ")} max={ALTERATION_GENE_SET_CAP} busy={busy} title="Genes for this axis" + description="A case counts as mutated when any of these genes is. For anything finer, choose OQL query under Data." + onApply={(_t, c) => commitGenes(c.genes)} />)} </span> ); } diff --git a/app/portal/PortalTabs.tsx b/app/portal/PortalTabs.tsx index 0dab91f..756cd8f 100644 --- a/app/portal/PortalTabs.tsx +++ b/app/portal/PortalTabs.tsx @@ -148,7 +148,7 @@ export function PortalTabs({ doc, tabs, titles, open, onOpen, onDoc, onCounts, c <ViewHead menuStore={menus.current} v={v && !v.error ? v : { kind: spec.kind }} title={titles[vi]} sub={drawn.length ? drawn.join(drawn.length === 2 ? " vs " : ", ") : undefined} /> {hasKindCard(spec.kind as string) && <KindCardControls c={mc} />} - {spec.kind === "plot" && <PlotControls spec={spec} plot={v && !v.error ? v.plot : undefined} attrs={catalog.attrs} profiles={catalog.profiles} + {spec.kind === "plot" && <PlotControls spec={spec} plot={v && !v.error ? v.plot : undefined} attrs={catalog.attrs} catalog={catalog.profileCatalog || []} loading={catalog.loading} busy={busy} update={(fn) => edit(vi, fn(spec))} cohortOptions={Object.keys(doc.cohorts || {}).filter((n) => n !== drawn[0]).map((n) => ({ name: n, label: n, depth: 0 }))}> {figureArea} diff --git a/app/portal/comparison/page.tsx b/app/portal/comparison/page.tsx index d3dee03..6ff2da6 100644 --- a/app/portal/comparison/page.tsx +++ b/app/portal/comparison/page.tsx @@ -16,7 +16,7 @@ const CLINICAL_SHOWN = 8; function comparisonLayout(linked: AnalysisDoc, study: string, known: StudyKnown): PageLayout { const names = Object.keys(linked.cohorts || {}); - const expr = expressionProfile(known.profiles); + const expr = expressionProfile(known.profileCatalog); const ga = known.gaProfiles.find((p) => /signature/i.test(p) && /contribution/i.test(p)) || known.gaProfiles.find((p) => !/methylation/i.test(p)) || null; const v = (title: string, view: Record<string, unknown>) => ({ title, view: view as unknown as View }); const left = [!known.clinical.length && "Clinical (no attributes)", !(expr && names.length === 2) && "mRNA (" + (expr ? "compares exactly two cohorts" : "no mRNA profile") + ")", @@ -55,7 +55,7 @@ export default function ComparisonPage() { {!ready ? <><div className="skeleton mb-4 h-10" /><div className="card viewcard viewloading" aria-busy="true"><div className="skeleton skeleton-view" /></div></> : <PortalTabs doc={page.doc!} tabs={fresh!.tabs} titles={fresh!.titles} open={page.open} onOpen={page.openTab} onDoc={page.update} - catalog={{ attrs: known!.attrs, profiles: known!.profiles, loading: false }} + catalog={{ attrs: known!.attrs, profiles: known!.profiles, profileCatalog: known!.profileCatalog, loading: false }} onCounts={(c) => setCounts((o) => o || c)} />} </>); } diff --git a/app/portal/results/page.tsx b/app/portal/results/page.tsx index 0a23214..124288b 100644 --- a/app/portal/results/page.tsx +++ b/app/portal/results/page.tsx @@ -6,7 +6,7 @@ import { useMemo, useState } from "react"; import { useSearchParams } from "next/navigation"; import type { CohortDef, View } from "../../../lib/types"; import { PortalTabs } from "../PortalTabs"; -import { useStudyKnown, expressionProfile, type StudyKnown } from "../study-known"; +import { useStudyKnown, expressionProfile, discreteCnaProfile, type StudyKnown } from "../study-known"; import { arrange, decodeDoc, usePageDoc, type PageLayout } from "../page-doc"; import { ALTERATION_CELLS, cellLabel } from "../../../lib/lollipop"; import { Button } from "../../_ui/button"; @@ -48,7 +48,7 @@ function resultsLayout(study: string, entries: string[], genes: string[], known: Altered: { all: [{ any: entries.map((q) => ({ kind: "oql", q })) }] } as CohortDef, Unaltered: { all: entries.map((q) => ({ not: { kind: "oql", q } })) } as CohortDef, }; - const expr = expressionProfile(known.profiles); + const expr = expressionProfile(known.profileCatalog), cna = discreteCnaProfile(known.profileCatalog); // the portal plots and correlates over every queried sample, not one group if (expr) cohorts["All patients"] = { all: [] } as CohortDef; const both = ["Altered", "Unaltered"]; @@ -58,7 +58,8 @@ function resultsLayout(study: string, entries: string[], genes: string[], known: { title: "Oncoprint", views: [v("Oncoprint", { kind: "oncoprint", genes, cohorts: both })] }, { title: "Cancer Types Summary", views: [v("Cancer types summary", { kind: "cancertypes", genes, cohorts: ["Altered"] })] }, ...(genes.length >= 2 ? [{ title: "Mutual Exclusivity", views: [v("Mutual exclusivity", { kind: "cooccurrence", genes, cohorts: both })] }] : []), - ...(expr ? [{ title: "Plots", views: [v("Plot", { kind: "plot", x: { source: "alteration", gene: genes[0], by: "cna" }, y: { source: "molecular", profileType: expr, gene: genes[0] }, cohorts: ["All patients"], points: [{ cohort: "Altered" }] })] }] : []), + ...(expr ? [{ title: "Plots", views: [v("Plot", { kind: "plot", x: cna ? { source: "profile", profile: cna, gene: genes[0] } : { source: "alteration", gene: genes[0], by: "cna" }, + y: { source: "profile", profile: expr, gene: genes[0] }, cohorts: ["All patients"], points: [{ cohort: "Altered" }] })] }] : []), // the portal's Mutations tab: one gene at a time, its lollipop over its mutation table { title: "Mutations", views: [v("Mutation diagram", { kind: "hotspots", genes: [genes[0]], cohorts: ["Altered"] }), v("Mutation table", { kind: "mutations", gene: genes[0], cohorts: ["Altered"] })], @@ -119,7 +120,7 @@ export default function ResultsPage() { {!ready ? <><div className="skeleton mb-4 h-10" /><div className="card viewcard viewloading" aria-busy="true"><div className="skeleton skeleton-view" /></div></> : <PortalTabs doc={page.doc!} tabs={fresh!.tabs} titles={fresh!.titles} open={page.open} onOpen={page.openTab} onDoc={page.update} - catalog={{ attrs: known!.attrs, profiles: known!.profiles, loading: false }} + catalog={{ attrs: known!.attrs, profiles: known!.profiles, profileCatalog: known!.profileCatalog, loading: false }} onCounts={(c) => setCounts((o) => o || c)} />} </>); } diff --git a/app/portal/study-known.ts b/app/portal/study-known.ts index 43ddae1..24bbe72 100644 --- a/app/portal/study-known.ts +++ b/app/portal/study-known.ts @@ -3,9 +3,12 @@ // molecular profiles, generic-assay profiles and clinical attributes (most covered first). import { useEffect, useState } from "react"; import { api } from "../lab/api.ts"; +import type { ProfileInfo } from "../../lib/profile-catalog.ts"; export interface StudyKnown { profiles: string[]; gaProfiles: string[]; clinical: string[]; + /** every profile as the study describes it, for a plot axis's Data picker */ + profileCatalog: ProfileInfo[]; /** the clinical attributes with their kind and coverage, for the oncoprint's Tracks menu */ attrs: { attribute: string; kind: string; patients: number }[]; /** the study's display name and size, for the page header */ @@ -30,6 +33,7 @@ export function useStudyKnown(study: string): { known: StudyKnown | null; error: const attrs = sorted.map((c) => ({ attribute: c.attribute, kind: c.kind || "cat", patients: c.patients || 0 })); const meta = ((st.studies || []) as { id: string; name: string }[]).find((x) => x.id === study); setKnown({ profiles: (j.profiles || []).filter((p: string) => p !== "cna"), gaProfiles: j.gaProfiles || [], clinical, attrs, + profileCatalog: j.profileCatalog || [], name: meta?.name || study, patients: j.totals?.patients || 0, samples: j.totals?.samples || 0 }); }) .catch((e) => setError(String(e))); @@ -37,8 +41,12 @@ export function useStudyKnown(study: string): { known: StudyKnown | null; error: return { known, error }; } -/** an mRNA profile for expression, preferring values over z-scores */ -export function expressionProfile(profiles: string[]): string | null { - const m = profiles.filter((p) => /mrna|rna_seq/i.test(p)); - return m.find((p) => !/zscore/i.test(p)) || m[0] || null; +/** an mRNA profile for expression, as the study describes its profiles: values before z-scores */ +export function expressionProfile(catalog: ProfileInfo[]): string | null { + const m = catalog.filter((p) => p.type === "MRNA_EXPRESSION").sort((a, b) => a.order - b.order); + return (m.find((p) => p.datatype === "CONTINUOUS") || m.find((p) => p.datatype === "Z-SCORE") || m[0])?.profile || null; +} +/** the study's discrete copy-number profile (GISTIC, or a panel study's calls) */ +export function discreteCnaProfile(catalog: ProfileInfo[]): string | null { + return catalog.find((p) => p.type === "COPY_NUMBER_ALTERATION" && p.datatype === "DISCRETE")?.profile || null; } diff --git a/docs/dsl.md b/docs/dsl.md index c396343..ca90469 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -334,13 +334,28 @@ column id or a filter that does not suit its column is refused with the ids acce ### Plot axes -An axis is one of four things, and the pair decides the shape — the portal's own rule: +An axis is a clinical attribute, one of the study's own molecular profiles, or a query across genes, +and the pair decides the shape — the portal's own rule: | `source` | shape of the axis | fields | | --- | --- | --- | | `clinical` | number or category, **as the study declared it** | `attribute` | -| `molecular` | number | `profileType`, `gene` | -| `alteration` | category | `gene` **or** `genes` (2-20), `by`: `"mutationType"` (default) \| `"mutated"` \| `"cna"` | +| `profile` | **as the study describes the profile** (below) | `profile` (its id without the study prefix), `gene`, `by` for a mutation profile, `log` for numbers | +| `alteration` | category | `oql` (below), or `genes` (2-20, any mutated) | + +**A profile axis** reads one gene in one of the study's profiles, and what it is follows from the +study's own description of that profile (`genetic_profile`: its alteration type and datatype), never +from its name: a **mutation** profile (MAF) groups cases by mutation type, or with `by: "mutated"` +mutated or not; a **structural-variant** profile by whether the case has one; a **discrete +copy-number** profile by its own calls (Deep deletion, Shallow deletion, Diploid, Gain, +Amplification, drawn in that order); and a **continuous** one (mRNA, z-scores, log2 copy number, +protein) is a number. A case with no mutation, SV or high-level call in the gene is the wild-type +answer only when it is profiled for that gene; otherwise it is **Not profiled**. A profile a study +does not have is refused with the list it does have, and one that two pooled studies describe +differently is refused, never read as one thing. Generic-assay profiles (methylation, arm-level copy +number, …) are not plot axes yet. The engine runs a profile axis as the lower-level forms it +resolves to, which a document may also give directly: `{"source":"molecular","profileType","gene", +"discrete"?}` and `{"source":"alteration","gene","by":"mutationType"|"mutated"|"cna"|"sv"}`. number × number draws a **density scatter**, number × category a **box**, category × category a **contingency table**. Every statistic is computed where the rows are and is therefore exact over @@ -354,7 +369,7 @@ inventing a second mechanism. Only `by:"mutated"` (the default when `genes` is u supported for a set — combining several genes' mutation TYPES or copy-number states into one category has no single honest reading, so that combination is refused rather than guessed at. -Two things are refused rather than approximated. A molecular axis **must name one gene** — a +Two things are refused rather than approximated. A profile axis **must name one gene** — a whole profile is 630 million values in this warehouse, and reading them to draw one picture is never what was meant. And a categorical axis with more than 150 distinct values is refused, because a truncated contingency table is a wrong one. @@ -420,7 +435,7 @@ profiled by the same rule (at most 6 statements). It is the same OQL a cohort's EXP/PROT thresholds), and it matches per patient or per sample following the plot's `unit`. Because it is a categorical axis, it also works as `colorBy`/`ringBy` and as a Cox or logistic covariate, where "Unaltered" is the reference level. -OQL is for alteration axes only: a molecular axis stays one gene's value. +OQL is for alteration axes only: a profile axis stays one gene in one profile. **Points.** Every case is a point when there are at most 10,000 of them and every study may show cases (`allPoints: false` draws the density alone at any size). Over that the plotted cohort is a @@ -2271,7 +2286,57 @@ Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups } ``` -### Example 21: Merged gene rows: a pathway as one track +### Example 21: Plot axes from the study's own profiles: ERBB2 copy number against expression + +TCGA breast cancer (PanCancer Atlas), every sample. The first view groups ERBB2 mRNA z-scores (relative to all samples) by ERBB2's GISTIC copy-number call, from Shallow deletion to Amplification: the 122 amplified samples sit far above the rest. The second groups fraction of genome altered by TP53's mutation type from the study's mutation profile; the 18 samples not in that profile are Not profiled, not counted as unmutated. + +```json +{ + "study": "brca_tcga_pan_can_atlas_2018", + "cohorts": { + "all samples": { + "all": [] + } + }, + "views": [ + { + "kind": "plot", + "cohorts": [ + "all samples" + ], + "unit": "sample", + "x": { + "source": "profile", + "profile": "gistic", + "gene": "ERBB2" + }, + "y": { + "source": "profile", + "profile": "rna_seq_v2_mrna_median_all_sample_Zscores", + "gene": "ERBB2" + } + }, + { + "kind": "plot", + "cohorts": [ + "all samples" + ], + "unit": "sample", + "x": { + "source": "profile", + "profile": "mutations", + "gene": "TP53" + }, + "y": { + "source": "clinical", + "attribute": "FRACTION_GENOME_ALTERED" + } + } + ] +} +``` + +### Example 22: Merged gene rows: a pathway as one track Pancreatic adenocarcinoma. The first row pools TP53, MDM2 and MDM4: it is altered where any of them is, and its denominator is the union of the three genes' panels — not their intersection, and not the whole cohort. TP53 is drawn again on its own underneath, so the difference the other two make is readable rather than asserted. A row's members are listed in its tooltip and in the contract's `members` field. @@ -2327,7 +2392,7 @@ Pancreatic adenocarcinoma. The first row pools TP53, MDM2 and MDM4: it is altere } ``` -### Example 22: A federated z-score reference on an oncoprint +### Example 23: A federated z-score reference on an oncoprint A heatmap track with reference:"federation" is scored against a distribution pooled from EVERY participating study, including instances that disclose no patients and therefore contribute no columns — each returns three totals per gene (n, sum, sum of squares), which is enough to define the reference. Run with luad_tcga_pan_can_atlas_2018 aggregate-only, its patients are absent from the columns while its 510 patients still shape what the drawn columns mean. @@ -2369,7 +2434,7 @@ A heatmap track with reference:"federation" is scored against a distribution poo } ``` -### Example 23: The clinical course: who is still on treatment, and when +### Example 24: The clinical course: who is still on treatment, and when MSK-CHORD, aligned on each patient's FIRST treatment, split by TP53 status. Treatment carries real start/stop dates so it is drawn as a state patients are IN; surgery has no duration, so it is the share with a surgery in that bin — a distinction read from the data's shape, not from the event's name. Note surgery peaking BEFORE time zero: resection typically precedes first systemic therapy. The at-risk row under the axis is the denominator, and it falls steeply — a curve read without it would be badly misleading. @@ -2423,7 +2488,7 @@ MSK-CHORD, aligned on each patient's FIRST treatment, split by TP53 status. Trea } ``` -### Example 24: A reusable workup: one template, retargeted in a single edit +### Example 25: A reusable workup: one template, retargeted in a single edit The gene list appears once, in `params`, and every view refers to it as `$genes`; the study likewise. Change either and all three figures follow. The rule is only that a value which is exactly "$name" is replaced by that parameter — no interpolation, no expressions — so the resolved document is an ordinary one and compiles to exactly the SQL you would get by typing the values in. A `$name` with no declaration is a loud error naming the path, and a parameter declared but never referenced is reported back, which is what catches a rename that missed a reference. `notes` and each view’s `note` travel with the document. @@ -2484,7 +2549,7 @@ The gene list appears once, in `params`, and every view refers to it as `$genes` } ``` -### Example 25: What comes after what: lines of therapy, and what ending a path means +### Example 26: What comes after what: lines of therapy, and what ending a path means MSK-CHORD therapy lines by TP53 status, 90-day gap, two panels on one scale. Read the grey blocks first, and read them as TWO things: the darker terminal is patients the study watched for at least a line gap who began no further therapy; the lighter one is patients whose records simply stop, about whom nothing can be concluded. Over half of all paths end censored — collapsing them together (as a plain Sankey does) would turn record-keeping into an apparent clinical decision. The real signal is in first line: TP53 wild-type receives far more hormone therapy than TP53-mutant, which is what hormone-receptor-positive disease looks like. Lines are the one definition every line count shares: systemic-therapy records only, so 'Prior Medications to MSK' (a provenance marker rather than a therapy), radiation and bone treatment are not lines — they co-occur with almost everything, so leaving them in would explode the combination space. The caption says which records were used; a document that wants them anyway says so with lineClasses. @@ -2531,7 +2596,7 @@ MSK-CHORD therapy lines by TP53 status, 90-day gap, two panels on one scale. Rea } ``` -### Example 26: Lollipop plot: mutations by protein position, with a suppression floor +### Example 27: Lollipop plot: mutations by protein position, with a suppression floor Hotspots in colorectal cancer, drawn the way cBioPortal's mutation tab draws them: one stem per codon, its height the distinct patient + protein change pairs there, with the protein's Pfam domains on the backbone — so KRAS spikes at codons 12-13 and BRAF at V600. minCount is applied by each site before anything is returned — with it set to 3, a change carried by one or two patients never leaves the database holding it, which is where small-cell suppression belongs. @@ -2570,7 +2635,7 @@ Hotspots in colorectal cancer, drawn the way cBioPortal's mutation tab draws the } ``` -### Example 27: Co-expression from sums alone +### Example 28: Co-expression from sums alone Pearson correlation between gene pairs in TCGA breast cancer. A correlation depends on the data only through six running totals, so each site can contribute without transmitting a single expression value — and because the totals add, the federated answer is exact rather than approximate. ERBB2/GRB7 (adjacent on the 17q12 amplicon) comes out near 0.87. @@ -2606,7 +2671,7 @@ Pearson correlation between gene pairs in TCGA breast cancer. A correlation depe } ``` -### Example 28: Clinical attributes compared between cohorts +### Example 29: Clinical attributes compared between cohorts How TP53-mutant and wild-type tumors differ clinically. Categorical attributes are compared with a chi-squared over the full value vocabulary; numeric ones with Welch's t computed from three additive totals per site (n, sum, sum of squares) rather than from any transmitted value. An attribute the study does not declare is reported as skipped, not silently dropped. @@ -2656,7 +2721,7 @@ How TP53-mutant and wild-type tumors differ clinically. Categorical attributes a } ``` -### Example 29: Co-occurrence and mutual exclusivity in NSCLC +### Example 30: Co-occurrence and mutual exclusivity in NSCLC Pairwise associations among lung cancer driver genes. Each site returns four counts per pair — both altered, one, the other, neither — among the patients its own panels profiled for BOTH genes, and nothing else; the coordinator does the inference. Reproduces the textbook pattern: STK11/KEAP1 co-occur, EGFR/KRAS and EGFR/ALK are mutually exclusive. @@ -2696,7 +2761,7 @@ Pairwise associations among lung cancer driver genes. Each site returns four cou } ``` -### Example 30: Survival from first chemotherapy, left-truncated +### Example 31: Survival from first chemotherapy, left-truncated TP53-mutant vs wild-type, overall survival measured from each patient's first chemotherapy rather than from sequencing. Patients whose first chemotherapy falls after day 0, or who never had chemotherapy, are excluded rather than given negative follow-up. The result reports how long after the anchor patients became observable, so an anchor that is matching earlier events than intended shows up as a long tail. @@ -2745,7 +2810,7 @@ TP53-mutant vs wild-type, overall survival measured from each patient's first ch } ``` -### Example 31: Study-stratified survival across two studies +### Example 32: Study-stratified survival across two studies TP53-mutant vs wild-type, each cohort spanning two studies. Every site answers with (months, event, n) counts only — no survival time for any individual crosses an instance boundary — and because the log-rank statistic is a sum over strata, those counts are enough to compute the test STRATIFIED by study: each study is compared only against itself. The unstratified pooled p is reported alongside, since the two can disagree when case mix differs between sites. @@ -2796,7 +2861,7 @@ TP53-mutant vs wild-type, each cohort spanning two studies. Every site answers w } ``` -### Example 32: extends: a nested cohort without repeating its parent's filter +### Example 33: extends: a nested cohort without repeating its parent's filter APC-mutant colorectal cancer, then the same population restricted to liver metastasis — without re-stating the APC clause. `compare` shows both share the same APC-mutant base, so a difference between them is attributable to the liver-mets split, not to two independently-typed cohort definitions quietly drifting apart. @@ -2842,7 +2907,7 @@ APC-mutant colorectal cancer, then the same population restricted to liver metas } ``` -### Example 33: Splits: a cohort divided into arms that add up to it +### Example 34: Splits: a cohort divided into arms that add up to it Non-small cell lung cancer split by KRAS allele: G12C first, then any other KRAS mutation, then wild-type. Because G12C is tested first, "other KRAS" needs no hand-written "and not G12C". 916 + 1,278 + 5,615 = 7,809, the whole cohort, with nobody unprofiled for KRAS in this study. STK11 and KEAP1 are enriched in both KRAS-mutant arms and TP53 in wild-type — the known co-mutation pattern — while overall survival does not differ significantly across the three arms (k-group log-rank p≈0.1). @@ -2906,7 +2971,7 @@ Non-small cell lung cancer split by KRAS allele: G12C first, then any other KRAS } ``` -### Example 34: Median/IQR and a rank-sum test, single-study only +### Example 35: Median/IQR and a rank-sum test, single-study only APC-mutant colorectal cancer, split by liver metastasis. Alongside mean/SD, numeric attributes report an exact median/IQR and a Mann-Whitney U — available here only because both cohorts stay within one study (crc_apc_impact_2020): median and rank are not additive across sites the way (n, Σx, Σx²) is, so this is refused rather than approximated the moment a cohort spans more than one study. @@ -2961,7 +3026,7 @@ APC-mutant colorectal cancer, split by liver metastasis. Alongside mean/SD, nume } ``` -### Example 35: Cox with a gene-SET covariate: any of three genes, one term +### Example 36: Cox with a gene-SET covariate: any of three genes, one term Every patient in the study, one term: mutated in any of TP53/KRAS/PIK3CA versus none of them. Compiles to a single hugo_gene_symbol IN (...) — the same shape as a single-gene covariate, just with a broader gene list, so it costs nothing extra downstream (dummy-coding, the forest plot, the reference-level override all just work). @@ -2995,7 +3060,7 @@ Every patient in the study, one term: mutated in any of TP53/KRAS/PIK3CA versus } ``` -### Example 36: Cox stratified by cohort: does a gene's hazard ratio hold up in both arms? +### Example 37: Cox stratified by cohort: does a gene's hazard ratio hold up in both arms? APC mutation status regressed on overall survival, separately in the MSI-high and MSS arms of the same study. Pooled, APC reads HR=0.61 (95% CI 0.48-0.78) — a real effect, but stratifying shows it is almost entirely a microsatellite-stable-population effect (MSS: HR=0.54, p<0.0001) that does not reach significance in the small MSI-high arm (HR=0.87, 95% CI 0.31-2.44, n=27) — a pooled fit alone would not have shown that the effect is concentrated in one subgroup. @@ -3046,7 +3111,7 @@ APC mutation status regressed on overall survival, separately in the MSI-high an } ``` -### Example 37: Logistic regression: does APC status predict MSI-high status? +### Example 38: Logistic regression: does APC status predict MSI-high status? MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do not report) regressed on APC mutation status. APC-mutant patients have substantially lower odds of MSI-high status (OR=0.28, 95% CI 0.13-0.63, p=0.0018) — consistent with APC-driven CRC typically arising through the chromosomal-instability pathway rather than the mismatch-repair-deficient pathway MSI-high status marks. @@ -3081,7 +3146,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no } ``` -### Example 38: splitBy: one cohort, auto-split into an arm per receptor subtype +### Example 39: splitBy: one cohort, auto-split into an arm per receptor subtype "all patients" split by RECEPTOR_STATUS_PRIMARY gives the same 4 clinically standard subtypes a hand-written 4-cohort document would (HR+/HER2-, Triple Negative, HR+/HER2+, HR-/HER2+), plus a few small ones this study also records (HR+/HER2_Unknown, Unk/ND, HR+/HER2_Equivocal) — all within the default 8-arm cap, so nothing here needs pooling into "Other". TP53 reads 26.8%/90.4%/49.2%/77% across the four main subtypes, matching the published literature on triple-negative breast cancer's markedly higher TP53 mutation rate. @@ -3109,7 +3174,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no } ``` -### Example 39: splitBy: gene mutation status, without hand-writing the mutant/wildtype pair +### Example 40: splitBy: gene mutation status, without hand-writing the mutant/wildtype pair "all patients" split by APC mutation status gives the same two arms a hand-written {"kind":"oql","q":"APC: MUT"} cohort pair would (325 mutant, 146 wildtype) — APC-mutant patients have better overall survival (median 25.6mo vs 17.9mo, log-rank p=9.1e-5), matching this study's own headline finding (see the crc_apc_impact_2020 Cox example above) from a single view, no separate cohorts to define. @@ -3133,7 +3198,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no } ``` -### Example 40: splitBy: a list of OQL queries, arms that overlap and a disclosed "Other" +### Example 41: splitBy: a list of OQL queries, arms that overlap and a disclosed "Other" "all patients" split by ["TP53: MUT", "KRAS: MUT"] gives 13,105 TP53-mutant, 7,115 KRAS-mutant, and 9,019 in neither ("Other") — summing to more than the 24,950-patient base cohort because 4,289 patients carry both mutations, which the result discloses as `splitBy.overlapCount` rather than leaving the arm counts to look inconsistent on their own (13,105 + 7,115 − 4,289 + 9,019 = 24,950, exactly the base count). @@ -3164,7 +3229,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no } ``` -### Example 41: Survival split three ways: the k-group log-rank +### Example 42: Survival split three ways: the k-group log-rank MSI status in three groups (stable / instable / indeterminate) instead of the usual two-arm split. The test asks whether the three curves differ AT ALL, not which pair does — a different question from running the 2-arm test three times pairwise, which would need its own multiple-testing correction. @@ -3223,7 +3288,7 @@ MSI status in three groups (stable / instable / indeterminate) instead of the us } ``` -### Example 42: A missing alteration type is disclosed, never a false negative +### Example 43: A missing alteration type is disclosed, never a false negative TP53 (a real mutation hotspot here) and MDM2 (never assayed for copy-number in this study) side by side. Both show 0%-or-real altered figures, but only the view's note — and each cell's per-type breakdown (mutation/CNA/structural-variant counts, visible on hover in the app) — reveals that MDM2's 0% means "never checked" while TP53's is a real, profiled result. @@ -3259,7 +3324,7 @@ TP53 (a real mutation hotspot here) and MDM2 (never assayed for copy-number in t } ``` -### Example 43: Four receptor subtypes, one real test: Welch's ANOVA +### Example 44: Four receptor subtypes, one real test: Welch's ANOVA Breast cancer's four receptor subtypes, compared on genomic instability (FRACTION_GENOME_ALTERED) and mutation burden (TMB_NONSYNONYMOUS). Triple-negative's markedly higher genomic instability is exactly the known biology, and both the parametric (ANOVA) and rank-based (Kruskal-Wallis) tests agree emphatically; TMB does not differ significantly by subtype here, and again both tests agree. @@ -3334,7 +3399,7 @@ Breast cancer's four receptor subtypes, compared on genomic instability (FRACTIO } ``` -### Example 44: A non-standard survival endpoint, named directly +### Example 45: A non-standard survival endpoint, named directly MSI-H vs MSS endometrial cancer, progression-free survival from first-line treatment — the study's actual recorded outcome, reachable only because the exact column names are given rather than assumed from an OS/PFS/DFS-style prefix. @@ -3382,7 +3447,7 @@ MSI-H vs MSS endometrial cancer, progression-free survival from first-line treat } ``` -### Example 45: Top 20% TMB, within one cancer type: a percentile cutoff +### Example 46: Top 20% TMB, within one cancer type: a percentile cutoff tmb_mskcc_2018 is pan-cancer, so a plain percentile would mix histologies with very different baseline TMB. "NSCLC high-TMB"/"NSCLC low-TMB" both restrict to NSCLC first, then split on whether TMB is at or above the 80th percentile COMPUTED WITHIN THAT NSCLC POPULATION (percentileOf:"NSCLC") — the readback shows the resolved number and what it was a percentile of, not just a bare threshold. @@ -3455,7 +3520,7 @@ tmb_mskcc_2018 is pan-cancer, so a plain percentile would mix histologies with v } ``` -### Example 46: Cohort attrition: which criterion removed whom +### Example 47: Cohort attrition: which criterion removed whom KRAS G12C lung adenocarcinoma treated with immunotherapy, with at least six months of follow-up, beside KRAS wild-type lung adenocarcinoma. The G12C arm loses most of its patients to the genotype, and a further 78 to the follow-up requirement — the kind of step a reader needs to see before trusting a survival curve drawn from what is left. @@ -3532,7 +3597,7 @@ KRAS G12C lung adenocarcinoma treated with immunotherapy, with at least six mont } ``` -### Example 47: Genome-wide copy number: colorectal against breast +### Example 48: Genome-wide copy number: colorectal against breast Colorectal and breast cancers in MSK-IMPACT, 2 Mb bins. The textbook colorectal pattern is all there — 20q gained in 60%, 13q in 42%, 8q in 32%, 18q lost in 54% — beside breast cancer's 1q and 8q gains and 16q loss. The strip underneath marks every bin where the two differ at q < 0.05, coloured by which cohort is higher. @@ -3578,7 +3643,7 @@ Colorectal and breast cancers in MSK-IMPACT, 2 Mb bins. The textbook colorectal } ``` -### Example 48: Oncogenic signalling pathways: colorectal against pancreatic +### Example 49: Oncogenic signalling pathways: colorectal against pancreatic Colorectal and pancreatic cancer in MSK-CHORD. WNT separates them almost completely (82% against 8%, nearly all APC); RTK-RAS is higher in pancreatic cancer (84%, KRAS 81%), and so is the cell cycle (39%, CDKN2A 33%); PI3K and NOTCH (FBXW7) are colorectal. The canonical pathways also show how much a targeted panel misses: 56 of WNT's 68 genes are on no panel here. @@ -3623,7 +3688,7 @@ Colorectal and pancreatic cancer in MSK-CHORD. WNT separates them almost complet } ``` -### Example 49: Differential expression and enrichment: basal-like against luminal A breast cancer +### Example 50: Differential expression and enrichment: basal-like against luminal A breast cancer TCGA PanCancer Atlas breast cancer, PAM50 basal-like against luminal A, on mRNA z-scores. The luminal genes (ESR1, PGR, FOXA1, GATA3) and the basal ones (FOXC1, and proliferation — MKI67, CDC20, AURKB) come out as expected, and so does the enrichment: E2F targets, the G2M checkpoint and MYC targets in basal-like tumours, early estrogen response in luminal A. @@ -3667,7 +3732,7 @@ TCGA PanCancer Atlas breast cancer, PAM50 basal-like against luminal A, on mRNA } ``` -### Example 50: Paired samples: what differs between a patient's primary and metastasis +### Example 51: Paired samples: what differs between a patient's primary and metastasis MSK-IMPACT, the 2,278 patients sequenced at both a primary and a metastatic site. Truncal drivers agree (TP53 78% concordant); what the metastasis alone carries is the resistance biology — ESR1 mutations (49 metastasis-only against 7 primary-only) and AR alterations (71 against 1), both selected under hormone therapy — and MYC amplification and CDKN2A loss. @@ -3700,7 +3765,7 @@ MSK-IMPACT, the 2,278 patients sequenced at both a primary and a metastatic site } ``` -### Example 51: Paired samples, every gene: what the metastasis carries that the primary does not +### Example 52: Paired samples, every gene: what the metastasis carries that the primary does not MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list. Of 347 genes with a discordant patient, the scan puts AR (71 metastasis-only against 1 primary-only) and ESR1 (49 against 7) first — the hormone-therapy resistance genes the hand-picked version above had to be told to look at — followed by MYC, NKX3-1 and PTEN. @@ -3724,7 +3789,7 @@ MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list } ``` -### Example 52: Before and after treatment, pooled across studies that label it differently +### Example 53: Before and after treatment, pooled across studies that label it differently 111 patients with a sample on each side across gastroesophageal (two studies), breast (alpelisib, and a second cohort) and bladder cancer. At this size nothing survives correction across the 278 genes tested — the honest answer — and the most one-sided genes run the other way: CCND1 and MYC amplifications seen before treatment and not after (10 and 10 patients against 1 and 2), as a lower-purity post-treatment sample would also show. Read it as a lead, not a finding; the same view over a large serially sampled cohort is where acquired resistance would show. @@ -3788,7 +3853,7 @@ MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list } ``` -### Example 53: Clonality, LOH and genome doubling from FACETS +### Example 54: Clonality, LOH and genome doubling from FACETS Colorectal against breast cancer. Tumour suppressors show the classic second hit — TP53 is ~88% clonal with its other allele lost 77–89% of the time, CDH1 in breast 95% — while the oncogene KRAS rarely loses its partner. PIK3CA is markedly more often subclonal in colorectal cancer (68% clonal) than in breast (84%), in keeping with a later event there. Around a third of both cohorts' tumours have doubled their genome. @@ -3842,7 +3907,7 @@ Colorectal against breast cancer. Tumour suppressors show the classic second hit } ``` -### Example 54: Brain metastasis, with death as a competing risk +### Example 55: Brain metastasis, with death as a competing risk MSK-CHORD, from each patient's day 0, among those without a brain metastasis by then. At five years 24.7% of NSCLC patients have one — where 1 − Kaplan–Meier would have said 30.4% — against 20.6% in breast and 10.8% in colorectal cancer (1 − KM: 14.4%, a third too high, because so many colorectal patients die first). @@ -3910,7 +3975,7 @@ MSK-CHORD, from each patient's day 0, among those without a brain metastasis by } ``` -### Example 55: PSA response to hormone therapy, by TP53 status +### Example 56: PSA response to hormone therapy, by TP53 status Prostate cancer in MSK-CHORD, from the first hormone therapy. PSA falls by a median of more than 95% within three months in both groups, but 93.0% of TP53 wild-type patients reach PSA50 against 85.8% of TP53-mutant ones (Fisher p = 1.4e-6), and the mutant group's upper quartile turns back upward within a year. @@ -3971,7 +4036,7 @@ Prostate cancer in MSK-CHORD, from the first hormone therapy. PSA falls by a med } ``` -### Example 56: A survival risk model from mutated genes in lung cancer +### Example 57: A survival risk model from mutated genes in lung cancer Non-small cell lung cancer in MSK-CHORD. The model keeps KEAP1, STK11, SMARCA4 and TP53 as adverse and EGFR as favourable, as the literature does, and its risk thirds separate clearly; a held-out C-index near 0.61 says mutations alone predict survival only modestly. Five runs keep the example quick; OncoCast recommends fifty. Its Table 1 (`describe`) shows who is in each third: the high-risk third is more often male, a former or current smoker and at stage 4 than the low-risk third. KEAP1 and TP53 are marked “in the model” — the score is built from them, so their enrichment is by construction and gets no q-value — and ZFHX3, which not every panel covers, is counted over each group's patients whose panel does. @@ -4017,7 +4082,7 @@ Non-small cell lung cancer in MSK-CHORD. The model keeps KEAP1, STK11, SMARCA4 a } ``` -### Example 57: Custom data: a curated biopsy site on every sample +### Example 58: Custom data: a curated biopsy site on every sample luad_mskimpact_2021 is 93 EGFR-mutant lung adenocarcinomas with 186 primary and metastatic samples. The demo upload gives every sample a BIOPSY_SITE, grouped from the study's own SAMPLE_TYPE and free-text METASTATIC_SITE the way a reviewer would (scripts/gen-custom-demo.ts), standing in for a clinician-curated annotation. "CNS met" filters on it: 8 patients. Ordered by site, the oncoprint puts each patient's primary and metastases in their own groups. Fraction of the genome altered has a median of 0.47 in the 11 CNS samples and 0.36 in liver, against 0.21 in the 76 primaries. Split by site, TP53 is altered in 7 of 8 CNS-met patients (87.5%) against 65.6% of the patients with a primary sample. The split arms overlap, because a patient whose primary and metastasis were both sequenced belongs to both arms. With 8 to 61 patients per arm, these are illustrations, not findings. @@ -4100,7 +4165,7 @@ Runs on custom data: `luad-biopsy-site` (LUAD biopsy site (demo), 186 samples, c } ``` -### Example 58: Custom data: single-cell features of ovarian tumours (MSK SPECTRUM) +### Example 59: Custom data: single-cell features of ovarian tumours (MSK SPECTRUM) msk_spectrum_tme_2022 is 42 untreated high-grade serous ovarian cancers; 41 had scRNA-seq at several anatomical sites. scripts/gen-spectrum-sc-demo.py reads the public CELLxGENE cell annotations (cell states, per-cell pathway scores, per-cell 6p allele fraction) and summarises them per patient (solid sites pooled; ADX_ adnexal, NADX_ non-adnexal) and per cBioPortal sample (the scRNA-seq site of the same patient, 61 of 82 samples). Dysfunctional T/NK cells (TNK_DYSFUNC) have a median of 10.8% of T/NK cells in the 16 HRD-Dup tumours against 6.0% in the 14 FBI tumours, and naive/central-memory cells the reverse (7.5% against 11.1%), as in the paper's Fig. 4b. In adnexal cancer cells, JAK-STAT pathway activity has a median of 0.03 in HRD-Dup against -0.57 in FBI (Fig. 3e), and cancer-cell and T/NK-cell JAK-STAT correlate across patients (Spearman 0.74, Fig. 4e). Clonal 6p loss of heterozygosity (over 80% of cancer cells) is called in 4 patients, 3 of them HRD-Dup — the paper reports 4 of 41. By site, T cells are a median 43% of immune cells in the 22 adnexal samples against 54% in the 18 omental ones. These are summaries of public cell metadata, not the paper's statistical models. @@ -4217,7 +4282,7 @@ Runs on custom data: `spectrum-sc-composition` (SPECTRUM scRNA: cell-type compos } ``` -### Example 59: Composition: T and NK cell states by mutational signature (MSK SPECTRUM) +### Example 60: Composition: T and NK cell states by mutational signature (MSK SPECTRUM) The 36 SPECTRUM patients with a consensus signature other than Undetermined, split as the paper splits them. Dysfunctional CXCL13+ CD8 T cells are a mean 7.0% of T/NK cells in the 16 HRD-Dup patients against 4.9% in HRD-Del and 4.4% in FBI, and naive CD4 cells the reverse (15.2% against 19.6% and 20.1%) — the direction of the paper's Fig. 4b. The clearest single difference is cycling T/NK cells, higher in HRD-Dup (8.6% against 5.7%; Welch p = 0.003, q = 0.08 across the 30 comparisons); none reaches q < 0.05 at this size, which the matrix shows as no outlined dot. Per sample, from the study's cell-type fractions, T cells are 24% of HRD-Dup samples' cells against 32% of FBI's. The paper fits a binomial model on cell counts with site and sort as covariates; this view compares shares between groups and says so. @@ -4297,7 +4362,7 @@ Runs on custom data: `spectrum-sc-composition` (SPECTRUM scRNA: cell-type compos } ``` -### Example 60: Composition: adenomas and serrated polyps, from a generic single-cell summary (HTAN Vanderbilt) +### Example 61: Composition: adenomas and serrated polyps, from a generic single-cell summary (HTAN Vanderbilt) crc_hta11_htan_2021 is 52 patients with colorectal polyps (Chen et al., Cell 2021). The summary covers 55 of its 61 samples; its counts equal the study's own absolute_counts profile for all 670 sample and cell-type pairs it reports. Per sample, adenoma-specific cells (ASC) are 32.1% of the epithelium in the 30 conventional adenomas against 4.1% in the 20 serrated polyps (q = 2e-5), and serrated-specific cells (SSC) 1.5% against 19.7% (q = 0.003): the two paths to malignancy the paper describes. Stem cells (STM) differ little here (3.1% against 2.5%). The study's own fractions, over every cell type, give the same picture (ASC 29.9% against 3.7%). Cohorts are patients, so a patient with polyps of both types counts in both; 5 of the 52 have both. @@ -4366,7 +4431,7 @@ Runs on custom data: `hta11-sc-p` (HTAN Vanderbilt colorectal polyp scRNA-seq (C } ``` -### Example 61: Cancer types summary: RAS genes across 50,000 tumours +### Example 62: Cancer types summary: RAS genes across 50,000 tumours KRAS, NRAS and HRAS across MSK-IMPACT 50K. Pancreatic cancer leads at 82%, appendiceal and colorectal follow, and the bars show where RAS is amplified rather than mutated — germ cell tumours and esophagogastric cancer. @@ -4395,7 +4460,7 @@ KRAS, NRAS and HRAS across MSK-IMPACT 50K. Pancreatic cancer leads at 82%, appen } ``` -### Example 62: Cohort overlap: how four groups share their patients +### Example 63: Cohort overlap: how four groups share their patients TP53-mutant, KRAS-mutant, immunotherapy-treated and NSCLC patients in MSK-CHORD. 5,314 TP53-mutant patients belong to none of the other three groups; 435 are in all four. @@ -4465,7 +4530,7 @@ TP53-mutant, KRAS-mutant, immunotherapy-treated and NSCLC patients in MSK-CHORD. } ``` -### Example 63: Alteration enrichment: what separates colorectal from pancreatic tumours +### Example 64: Alteration enrichment: what separates colorectal from pancreatic tumours Microsatellite-stable colorectal against pancreatic cancer in MSK-CHORD. APC, TCF7L2, PIK3CA, FBXW7 and SOX9 mark colorectal cancer; KRAS and CDKN2A mark pancreatic cancer. @@ -4530,7 +4595,7 @@ Microsatellite-stable colorectal against pancreatic cancer in MSK-CHORD. APC, TC } ``` -### Example 64: Mutational signatures: what UV light and tobacco leave behind +### Example 65: Mutational signatures: what UV light and tobacco leave behind Melanoma against non-small cell lung cancer in MSK-IMPACT 50K. The UV signature (7) makes up about half of a melanoma's mutations and almost none of a lung cancer's; the smoking signature (4) is the reverse. @@ -4576,7 +4641,7 @@ Melanoma against non-small cell lung cancer in MSK-IMPACT 50K. The UV signature } ``` -### Example 65: Co-expression: the genes that rise and fall with ESR1 +### Example 66: Co-expression: the genes that rise and fall with ESR1 TCGA PanCancer Atlas breast cancer. The oestrogen-receptor programme lines up behind ESR1 — CCDC170 (its neighbour on 6q25, often co-amplified), THSD4, FOXA1, TBC1D9 — and the genes of basal-like tumours fall away from it. @@ -4601,7 +4666,7 @@ TCGA PanCancer Atlas breast cancer. The oestrogen-receptor programme lines up be } ``` -### Example 66: Clinical data: every pancreatic cancer sample, oldest patients first +### Example 67: Clinical data: every pancreatic cancer sample, oldest patients first Pancreatic cancer in MSK-CHORD, sorted by age, highest first. @@ -4638,7 +4703,7 @@ Pancreatic cancer in MSK-CHORD, sorted by age, highest first. } ``` -### Example 67: A patient's timeline: treatments, diagnoses, samples and lab values from diagnosis +### Example 68: A patient's timeline: treatments, diagnoses, samples and lab values from diagnosis MSK-CHORD breast cancer patients with the most timeline events first; P-0000012 drawn, with chemotherapy folded into one row, and P-0000012's alterations beside it. @@ -4691,7 +4756,7 @@ MSK-CHORD breast cancer patients with the most timeline events first; P-0000012 } ``` -### Example 68: A patient's mutations, copy-number alterations and structural variants +### Example 69: A patient's mutations, copy-number alterations and structural variants MSK-CHORD prostate cancer patients, those with the most structural variants first; P-0024249's tables, mutations kept to OncoKB-oncogenic calls. @@ -4739,7 +4804,7 @@ MSK-CHORD prostate cancer patients, those with the most structural variants firs } ``` -### Example 69: KRAS codon 12/13 mutations in colorectal cancer, clonal-looking calls first +### Example 70: KRAS codon 12/13 mutations in colorectal cancer, clonal-looking calls first MSK-CHORD patients with a colorectal cancer sample: their KRAS G12/G13 calls at an allele frequency of 0.2 or more, highest first. diff --git a/e2e.cjs b/e2e.cjs index dbe8551..38d2f01 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -568,7 +568,7 @@ async function main() { try { const v = JSON.parse(document.querySelector("#docjson").value).views[0]; return v.x.attribute === "FRACTION_GENOME_ALTERED" && v.y.attribute === "MUTATION_COUNT"; } catch { return false; } }, null, { timeout: 30000 }); - sideOk = blocks.join(",") === "Horizontal axis,Vertical axis" && names.join(",") === "Data type,Clinical attribute" && !!beside && heights.length === 1; + sideOk = blocks.join(",") === "Horizontal axis,Vertical axis" && names.join(",") === "Data,Clinical attribute" && !!beside && heights.length === 1; sideWhat = "blocks " + blocks.join(",") + "; fields " + names.join(",") + "; figure beside: " + beside + "; select heights " + heights.join(",") + "; swap rewrote x/y"; } catch (e) { sideWhat = String(e.message).split("\n")[0]; } ok("a plot's axes are defined in a sidebar beside the figure, and Swap axes swaps them", sideOk, sideWhat); diff --git a/lib/analysis.ts b/lib/analysis.ts index 2f70140..98f28a6 100644 --- a/lib/analysis.ts +++ b/lib/analysis.ts @@ -9,6 +9,7 @@ import { applyLocks, lockDate, lockSamples } from "./cohort-lock.ts"; import { chQuery, STUDY, PRIMARY, type ChConn } from "./ch.ts"; import { sq } from "./compiler.ts"; +import type { ProfileInfo } from "./profile-catalog.ts"; import { describeCohortDef } from "./cohort2.ts"; import type { AnalysisDoc, AnalysisResult, CohortDef, PartitionDef, ViewResult, ViewParticipation, TemplateDoc, SplitBy, LockDrift } from "./types.ts"; import { resolveSplitBy, type AttributeDistribution } from "./cohort-splitby.ts"; @@ -34,11 +35,16 @@ async function oncokbPin(): Promise<string | null> { } catch { return null; } } -const _profileCache = new Map<string, { profiles: string[]; gaProfiles: string[] }>(); // per-study -export async function studyProfiles(study: string): Promise<{ profiles: string[]; gaProfiles: string[] }> { +const _profileCache = new Map<string, StudyProfiles>(); // per-study +export interface StudyProfiles { + profiles: string[]; gaProfiles: string[]; + /** every profile as the study describes it (genetic_profile), for what a plot axis may read */ + catalog: ProfileInfo[]; +} +export async function studyProfiles(study: string): Promise<StudyProfiles> { if (_profileCache.has(study)) return _profileCache.get(study)!; const { peerForStudy } = await import("./federation-remote.ts"); - if (await peerForStudy(study)) return { profiles: [], gaProfiles: [] }; // the peer owns its own profile inventory + if (await peerForStudy(study)) return { profiles: [], gaProfiles: [], catalog: [] }; // the peer owns its own profile inventory const conn = (await studyRouting()).get(study) || PRIMARY; const profiles = (await chQuery( "SELECT DISTINCT profile_type FROM " + conn.db + ".genetic_alteration_derived WHERE cancer_study_identifier = " + sq(study), conn)).data.map((r) => r.profile_type); @@ -47,7 +53,15 @@ export async function studyProfiles(study: string): Promise<{ profiles: string[] "SELECT DISTINCT replaceOne(gp.stable_id, " + sq(study + "_") + ", '') AS pt FROM " + conn.db + ".genetic_profile gp\n" + "JOIN " + conn.db + ".cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id\n" + "WHERE cs.cancer_study_identifier = " + sq(study) + " AND gp.genetic_alteration_type = 'GENERIC_ASSAY'", conn)).data.map((r) => r.pt); - const out = { profiles, gaProfiles }; + const catalog = (await chQuery( + "SELECT replaceOne(gp.stable_id, " + sq(study + "_") + ", '') AS profile, gp.genetic_alteration_type AS type, gp.datatype AS datatype,\n" + + " gp.name AS name, ifNull(toString(gp.generic_assay_type), '') AS assay, toInt32OrZero(ifNull(toString(gp.sort_order), '')) AS ord\n" + + "FROM " + conn.db + ".genetic_profile gp\n" + + "JOIN " + conn.db + ".cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id\n" + + "WHERE cs.cancer_study_identifier = " + sq(study), conn)).data.map((r): ProfileInfo => ({ + profile: String(r.profile), type: String(r.type), datatype: String(r.datatype), name: String(r.name), + ...(r.assay ? { assayType: String(r.assay) } : {}), order: Number(r.ord) || 0 })); + const out = { profiles, gaProfiles, catalog }; _profileCache.set(study, out); return out; } diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index 0fdabab..0dae5cd 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -486,6 +486,24 @@ export const EXAMPLES: DslExample[] = [ ] } }, + { + title: "Plot axes from the study's own profiles: ERBB2 copy number against expression", + demonstrates: "A `profile` axis: one gene in one of the study's molecular profiles, read as the study describes the profile — discrete copy number as its levels, expression as a number, mutations by type.", + tags: ["profiles"], + note: "TCGA breast cancer (PanCancer Atlas), every sample. The first view groups ERBB2 mRNA z-scores (relative to all samples) by ERBB2's GISTIC copy-number call, from Shallow deletion to Amplification: the 122 amplified samples sit far above the rest. The second groups fraction of genome altered by TP53's mutation type from the study's mutation profile; the 18 samples not in that profile are Not profiled, not counted as unmutated.", + doc: { + study: "brca_tcga_pan_can_atlas_2018", + cohorts: { "all samples": { all: [] } }, + views: [ + { kind: "plot", cohorts: ["all samples"], unit: "sample", + x: { source: "profile", profile: "gistic", gene: "ERBB2" }, + y: { source: "profile", profile: "rna_seq_v2_mrna_median_all_sample_Zscores", gene: "ERBB2" } }, + { kind: "plot", cohorts: ["all samples"], unit: "sample", + x: { source: "profile", profile: "mutations", gene: "TP53" }, + y: { source: "clinical", attribute: "FRACTION_GENOME_ALTERED" } } + ] + } + }, { title: "Merged gene rows: a pathway as one track", demonstrates: "Several genes drawn as one row, the way cBioPortal's [TP53 MDM2 MDM4] does, with the members kept alongside so the pooling is visible.", @@ -1589,13 +1607,28 @@ column id or a filter that does not suit its column is refused with the ids acce ### Plot axes -An axis is one of four things, and the pair decides the shape — the portal's own rule: +An axis is a clinical attribute, one of the study's own molecular profiles, or a query across genes, +and the pair decides the shape — the portal's own rule: | \`source\` | shape of the axis | fields | | --- | --- | --- | | \`clinical\` | number or category, **as the study declared it** | \`attribute\` | -| \`molecular\` | number | \`profileType\`, \`gene\` | -| \`alteration\` | category | \`gene\` **or** \`genes\` (2-20), \`by\`: \`"mutationType"\` (default) \\| \`"mutated"\` \\| \`"cna"\` | +| \`profile\` | **as the study describes the profile** (below) | \`profile\` (its id without the study prefix), \`gene\`, \`by\` for a mutation profile, \`log\` for numbers | +| \`alteration\` | category | \`oql\` (below), or \`genes\` (2-20, any mutated) | + +**A profile axis** reads one gene in one of the study's profiles, and what it is follows from the +study's own description of that profile (\`genetic_profile\`: its alteration type and datatype), never +from its name: a **mutation** profile (MAF) groups cases by mutation type, or with \`by: "mutated"\` +mutated or not; a **structural-variant** profile by whether the case has one; a **discrete +copy-number** profile by its own calls (Deep deletion, Shallow deletion, Diploid, Gain, +Amplification, drawn in that order); and a **continuous** one (mRNA, z-scores, log2 copy number, +protein) is a number. A case with no mutation, SV or high-level call in the gene is the wild-type +answer only when it is profiled for that gene; otherwise it is **Not profiled**. A profile a study +does not have is refused with the list it does have, and one that two pooled studies describe +differently is refused, never read as one thing. Generic-assay profiles (methylation, arm-level copy +number, …) are not plot axes yet. The engine runs a profile axis as the lower-level forms it +resolves to, which a document may also give directly: \`{"source":"molecular","profileType","gene", +"discrete"?}\` and \`{"source":"alteration","gene","by":"mutationType"|"mutated"|"cna"|"sv"}\`. number × number draws a **density scatter**, number × category a **box**, category × category a **contingency table**. Every statistic is computed where the rows are and is therefore exact over @@ -1609,7 +1642,7 @@ inventing a second mechanism. Only \`by:"mutated"\` (the default when \`genes\` supported for a set — combining several genes' mutation TYPES or copy-number states into one category has no single honest reading, so that combination is refused rather than guessed at. -Two things are refused rather than approximated. A molecular axis **must name one gene** — a +Two things are refused rather than approximated. A profile axis **must name one gene** — a whole profile is 630 million values in this warehouse, and reading them to draw one picture is never what was meant. And a categorical axis with more than 150 distinct values is refused, because a truncated contingency table is a wrong one. @@ -1675,7 +1708,7 @@ profiled by the same rule (at most ${OQL_STATEMENT_CAP} statements). It is the s EXP/PROT thresholds), and it matches per patient or per sample following the plot's \`unit\`. Because it is a categorical axis, it also works as \`colorBy\`/\`ringBy\` and as a Cox or logistic covariate, where "Unaltered" is the reference level. -OQL is for alteration axes only: a molecular axis stays one gene's value. +OQL is for alteration axes only: a profile axis stays one gene in one profile. **Points.** Every case is a point when there are at most ${POINTS_CAP_TEXT} of them and every study may show cases (\`allPoints: false\` draws the density alone at any size). Over that the plotted cohort is a @@ -1912,8 +1945,10 @@ export const AGENT_GRAMMAR = `Document shape: // <layer> = {"rule":"outliers"} | {"rule":"extremes","axis":"x|y","n":50?} | {"rule":"trend","n":50?} | {"cohort":C} // default [{"rule":"outliers"}]; [] draws none; the first layer draws a case two share // <axis> = {"source":"clinical","attribute":A,"log":?} - // | {"source":"molecular","profileType":P,"gene":G,"log":?} - // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|driver"?} + // | {"source":"profile","profile":P,"gene":G,"by":"mutationType|mutated"?,"log":?} + // // P is one of the study's profiles; what it is (mutations, SVs, discrete or + // // continuous copy number, mRNA, protein) comes from the study's own description + // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|sv"?} // what a profile axis resolves to // | {"source":"alteration","genes":[G,...(2-20)],"by":"mutated"?} // any-of-N as one category; only "mutated" // | {"source":"alteration","oql":"KRAS: G12C, STK11: MUT","by":"any|statement"?} // // any: Altered / Unaltered / Not profiled; statement: one group per statement (max 6) diff --git a/lib/plot-draw.ts b/lib/plot-draw.ts index eaf775c..ffdddd9 100644 --- a/lib/plot-draw.ts +++ b/lib/plot-draw.ts @@ -3,7 +3,7 @@ /** categories that mean "this case has none of that", which draw as neutral and never as a ring */ export function isAbsent(label: string): boolean { - return /^(not mutated|unaltered|neutral|no data|wild type|none|)$/i.test(String(label).trim()); + return /^(not mutated|unaltered|neutral|diploid|no structural variant|not profiled|no data|wild type|none|)$/i.test(String(label).trim()); } /** diff --git a/lib/plot-profile.ts b/lib/plot-profile.ts new file mode 100644 index 0000000..cba8d27 --- /dev/null +++ b/lib/plot-profile.ts @@ -0,0 +1,42 @@ +// A profile axis resolved against the studies it is drawn over: the study's own description of the +// profile (lib/profile-catalog.ts) decides what the axis is, and the engine then runs the form that +// reading calls for. A profile a study lacks, or one two pooled studies describe differently, is +// refused by name rather than read as something it is not. +import type { PlotAxis } from "./plot.ts"; +import { profileUse, plottableProfiles, type ProfileInfo } from "./profile-catalog.ts"; + +/** each study the axis is drawn over, with its profiles as it describes them */ +export type StudyCatalog = { study: string; catalog: ProfileInfo[] }; + +export function resolveProfileAxis(a: PlotAxis, which: string, studies: StudyCatalog[]): PlotAxis { + if (!a || a.source !== "profile") return a; + const found = studies.map((s) => ({ study: s.study, catalog: s.catalog, info: s.catalog.find((p) => p.profile === a.profile) })); + const missing = found.filter((f) => !f.info); + if (missing.length) { + const offer = plottableProfiles(missing[0].catalog).map((p) => p.profile); + throw new Error(which + " axis: " + missing.map((m) => m.study).join(", ") + " has no profile '" + a.profile + "'" + + (offer.length ? "; it has " + offer.slice(0, 12).join(", ") + (offer.length > 12 ? " and " + (offer.length - 12) + " more" : "") : "; it has none a plot axis can read")); + } + const uses = [...new Set(found.map((f) => profileUse(f.info!)))]; + if (uses.length > 1) + throw new Error(which + " axis: the profile '" + a.profile + "' is described differently by " + found.map((f) => f.study + " (" + f.info!.type + ", " + f.info!.datatype + ")").join(" and ") + + ", so one axis cannot read it across them"); + const info = found[0].info!, use = uses[0]; + if (!use) throw new Error(which + " axis: '" + a.profile + "' (" + info.type + ", " + info.datatype + ") is not a profile a plot axis reads"); + if (a.by != null && use !== "mutation") throw new Error(which + " axis: only a mutation profile groups by mutation type or mutated; '" + a.profile + "' is " + info.name); + if (a.log && use !== "number") throw new Error(which + " axis: a log scale applies to numbers; '" + a.profile + "' is " + info.name); + const gene = String(a.gene).toUpperCase(); + const label = gene + ": " + info.name; + if (use === "mutation") return { source: "alteration", gene, by: a.by || "mutationType" }; + if (use === "sv") return { source: "alteration", gene, by: "sv" }; + if (use === "discrete") return { source: "molecular", profileType: a.profile, gene, discrete: true, label }; + if (use === "number") return { source: "molecular", profileType: a.profile, gene, ...(a.log ? { log: true } : {}), label }; + throw new Error(which + " axis: '" + a.profile + "' is a generic assay (" + info.name + "), which a plot axis does not read yet"); +} + +/** every axis of a view resolved; `null` axes (an unset colour) pass through */ +export function resolveAxes<T extends Record<string, PlotAxis | undefined>>(axes: T, studies: StudyCatalog[]): T { + const out: Record<string, PlotAxis | undefined> = {}; + for (const [k, a] of Object.entries(axes)) out[k] = a ? resolveProfileAxis(a, k, studies) : a; + return out as T; +} diff --git a/lib/plot.ts b/lib/plot.ts index 6ab959e..89aae63 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -29,7 +29,22 @@ const OQL_PROFILED_TYPES: Partial<Record<string, ("MUTATION" | "CNA" | "SV")[]>> /** what goes on one axis */ export type PlotAxis = | { source: "clinical"; attribute: string; log?: boolean } - | { source: "molecular"; profileType: string; gene: string; log?: boolean } + /** + * One of the study's own molecular profiles, by its id without the study prefix, and a gene. What + * the axis is follows from the profile's own type and datatype (lib/profile-catalog.ts): a + * mutation profile groups cases by mutation type (or `by: "mutated"`), a structural-variant + * profile by whether the case has one, a discrete copy-number profile by its levels, and a + * continuous one (expression, z-scores, log2 copy number, protein) is a number. Resolved against + * the study before it runs (lib/plot-profile.ts) into the forms below; a profile the study does + * not have is refused by name. + */ + | { source: "profile"; profile: string; gene: string; by?: "mutationType" | "mutated"; log?: boolean } + /** + * The forms a profile axis resolves to, which the engine runs, and which a document may also give + * directly: one gene's value in a profile (`discrete` reads it as copy-number levels), and below, + * a gene's alterations from the event table. `label` is what the figure calls the axis. + */ + | { source: "molecular"; profileType: string; gene: string; log?: boolean; discrete?: boolean; label?: string } /** * `genes` (2+) asks "is ANY of these altered" as one categorical — a pathway rollup (e.g. a * DDR gene set) instead of one gene at a time — and only for `by:"mutated"`: combining several @@ -48,8 +63,8 @@ export type PlotAxis = */ | { source: "alteration"; oql: string; by?: "any" | "statement" } | { source: "none" }; -export type AlterationBy = "mutated" | "mutationType" | "cna"; -const AXIS_SOURCES: readonly string[] = ["clinical", "molecular", "alteration", "none"] satisfies PlotAxis["source"][]; +export type AlterationBy = "mutated" | "mutationType" | "cna" | "sv"; +const AXIS_SOURCES: readonly string[] = ["clinical", "profile", "molecular", "alteration", "none"] satisfies PlotAxis["source"][]; /** an alteration axis given as an OQL query, rather than as genes */ function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string; by?: "any" | "statement" } { return a.source === "alteration" && "oql" in a && a.oql != null; @@ -125,7 +140,11 @@ export function plotShape(x: AxisKind, y: AxisKind): PlotShape { /** what an axis is called on the figure */ export function axisLabel(a: PlotAxis): string { if (a.source === "clinical") return a.log ? "log10(" + a.attribute + " + 1)" : a.attribute; - if (a.source === "molecular") return a.log ? "log10(" + a.gene + " " + a.profileType + " + 1)" : a.gene + " " + a.profileType; + if (a.source === "molecular") { + const name = a.label || a.gene + " " + a.profileType; + return a.log ? "log10(" + name + " + 1)" : name; + } + if (a.source === "profile") return a.gene + " " + a.profile; if (isOqlAxis(a)) { const q = oqlStatements(a.oql).join("; "); return (q.length > 48 ? q.slice(0, 47) + "…" : q) + (a.by === "statement" ? " (OQL, by line)" : " (OQL)"); @@ -133,7 +152,7 @@ export function axisLabel(a: PlotAxis): string { if (a.source === "alteration") { const genes = alterationGenes(a); const name = genes.length > 1 ? (genes.length <= 4 ? genes.join("/") + " (any)" : genes.length + " genes (any)") : genes[0]; - return name + (a.by === "cna" ? " copy number" : a.by === "mutationType" ? " mutation type" : " mutation status"); + return name + (a.by === "cna" ? " copy number" : a.by === "sv" ? " structural variant" : a.by === "mutationType" ? " mutation type" : " mutation status"); } return "ordered cases"; } @@ -145,7 +164,8 @@ export function axisLabel(a: PlotAxis): string { */ export function axisKind(a: PlotAxis, clinicalKind?: (attr: string) => "num" | "cat" | undefined): AxisKind { if (a.source === "none") return "none"; - if (a.source === "molecular") return "num"; + if (a.source === "profile") throw new Error("a profile axis is read only once resolved against the study's profiles (lib/plot-profile.ts)"); + if (a.source === "molecular") return a.discrete ? "cat" : "num"; if (a.source === "alteration") return "cat"; const k = clinicalKind ? clinicalKind(a.attribute) : undefined; if (!k) throw new Error("clinical attribute '" + a.attribute + "' is not declared by this study, so its type is unknown"); @@ -158,6 +178,13 @@ export function validateAxis(a: PlotAxis, which: string): void { throw new Error(which + " axis: unknown source '" + String(a.source).slice(0, 40) + "' — use one of " + AXIS_SOURCES.join(", ")); if (a.source === "clinical" && !ATTR_RE.test(String(a.attribute || ""))) throw new Error(which + " axis: '" + a.attribute + "' is not a clinical attribute name"); + if (a.source === "profile") { + if (!PROFILE_RE.test(String(a.profile || ""))) throw new Error(which + " axis: '" + a.profile + "' is not a profile"); + if (!GENE_RE.test(String(a.gene || "").toUpperCase())) + throw new Error(which + " axis: a profile axis reads one gene; name it"); + if (a.by != null && a.by !== "mutationType" && a.by !== "mutated") + throw new Error(which + " axis: a mutation profile groups by \"mutationType\" or \"mutated\"; not " + JSON.stringify(a.by)); + } if (a.source === "molecular") { if (!PROFILE_RE.test(String(a.profileType || ""))) throw new Error(which + " axis: '" + a.profileType + "' is not a profile type"); // A molecular axis without a gene would ask the database for every gene of every case — @@ -242,6 +269,21 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s return "SELECT id, if(countIf(pref = 1) > 0, anyIf(v, pref = 1), " + combined + ") AS v FROM (\n" + own + "\n UNION ALL\n" + other + "\n) GROUP BY id"; } + if (a.source === "molecular" && a.discrete) { + // A discrete copy-number profile's own calls, one level per case, labelled as the portal labels + // them; a value outside the five levels is its own category, as written. In patient mode a + // patient whose samples disagree is "Multiple", the rule every categorical axis follows. + const gene = String(a.gene).toUpperCase(); + const level = "multiIf(x = -2, 'Deep deletion', x = -1, 'Shallow deletion', x = 0, 'Diploid', x = 1, 'Gain', x = 2, 'Amplification', toString(x))"; + return "SELECT id, if(uniqExact(lv) > 1, " + sq(MULTIPLE) + ", any(lv)) AS v FROM (\n" + + "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, "toFloat64OrNull(ga.alteration_value)") + " AS lv\n" + + "FROM " + db + ".genetic_alteration_derived ga\n" + + "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" + + "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) + + " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" + + " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" + + "GROUP BY id"; + } if (a.source === "molecular") { const gene = String(a.gene).toUpperCase(); // patient mode averages a patient's samples, the same rule the oncoprint's heatmap uses; a @@ -290,19 +332,29 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s const events = "SELECT sample_unique_id, mutation_type, variant_type, cna_alteration\n" + " FROM " + db + ".genomic_event_derived\n" + " WHERE cancer_study_identifier = " + sq(study) + " AND hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")"; + const hit = by === "cna" ? "abs(g.cna_alteration) >= 2" : by === "sv" ? "g.variant_type = 'structural_variant'" : "g.variant_type = 'mutation'"; const value = by === "cna" ? "if(countIf(g.cna_alteration >= 2) > 0, 'Amplification', if(countIf(g.cna_alteration <= -2) > 0, 'Deep deletion', 'Neutral'))" + : by === "sv" + ? "if(countIf(" + hit + ") > 0, 'Structural variant', 'No structural variant')" : by === "mutationType" // one class per case; a case carrying two of them is its own honest category ? "multiIf(countIf(g.variant_type = 'mutation') = 0, 'Not mutated'," + " length(groupUniqArrayIf(" + MUT_CLASS + ", g.variant_type = 'mutation')) > 1, 'Multiple'," + " arrayElement(groupUniqArrayIf(" + MUT_CLASS + ", g.variant_type = 'mutation'), 1))" : "if(countIf(g.variant_type = 'mutation') > 0, 'Mutated', 'Not mutated')"; - return "SELECT " + idCol + " AS id, " + value + " AS v\n" + + // A case with no such alteration is "not mutated" (neutral, no SV) only when it is profiled for + // every gene of the axis in that alteration type; otherwise it is "Not profiled", never counted + // as wild type (the panel-aware rule). The case column is alt_case, not id, for the reason the + // OQL axis's is oql_case. + const type = by === "cna" ? "CNA" as const : by === "sv" ? "SV" as const : "MUTATION" as const; + const profiled = genes.map((g) => "alt_case IN (SELECT profiled_id FROM (" + profiledCasesSql(g, type, { study, db, unit: ctx.unit }) + "))").join(" AND "); + return "SELECT alt_case AS id, if(hits = 0 AND NOT (" + profiled + "), " + sq(OQL_NOT_PROFILED) + ", v0) AS v FROM (\n" + + "SELECT " + idCol + " AS alt_case, " + value + " AS v0, countIf(" + hit + ") AS hits\n" + "FROM " + db + ".sample_derived sd\n" + "LEFT JOIN (\n " + events + "\n) g ON g.sample_unique_id = sd.sample_unique_id\n" + "WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "\n" + - "GROUP BY id"; + "GROUP BY alt_case)"; } throw new Error("axis source '" + (a as { source: string }).source + "' cannot be selected"); } diff --git a/lib/profile-catalog.ts b/lib/profile-catalog.ts new file mode 100644 index 0000000..c7a10ad --- /dev/null +++ b/lib/profile-catalog.ts @@ -0,0 +1,62 @@ +// A study's molecular profiles as the study itself describes them (cBioPortal's genetic_profile): +// each profile's alteration type, datatype and name. What a plot axis can do with a profile follows +// from these, never from its name. Browser-safe, so the page's Data picker and the engine read the +// same description. + +/** one profile of one study, its id without the study prefix (the derived tables' profile_type) */ +export interface ProfileInfo { + profile: string; + /** cBioPortal's genetic_alteration_type: MRNA_EXPRESSION, COPY_NUMBER_ALTERATION, GENERIC_ASSAY, … */ + type: string; + /** cBioPortal's datatype: CONTINUOUS, Z-SCORE, LOG2-VALUE, DISCRETE, MAF, SV, CATEGORICAL, LIMIT-VALUE, … */ + datatype: string; + name: string; + /** a generic assay's own kind: METHYLATION, PHOSPHOSITE_QUANTIFICATION, ARMLEVEL_CNA, … */ + assayType?: string; + /** the study's own order among its profiles */ + order: number; +} + +/** + * What an axis does with a profile, from its type and datatype: a mutation (grouped by mutation + * type, or mutated or not), a structural variant (present or not), a discrete copy-number call + * (one category per level), a number per case (continuous, z-scores, log2), or, for a generic assay, + * a number or a category per case for one entity. null: not something a plot axis reads. + */ +export type ProfileUse = "mutation" | "sv" | "discrete" | "number" | "assayNumber" | "assayCategory"; +export function profileUse(p: Pick<ProfileInfo, "type" | "datatype">): ProfileUse | null { + const t = p.type.toUpperCase(), d = p.datatype.toUpperCase(); + if (t === "MUTATION_EXTENDED") return d === "MAF" ? "mutation" : null; + if (t === "STRUCTURAL_VARIANT") return "sv"; + if (t === "GENERIC_ASSAY") return d === "CATEGORICAL" || d === "BINARY" ? "assayCategory" : "assayNumber"; + if (t === "COPY_NUMBER_ALTERATION" && d === "DISCRETE") return "discrete"; + if (t === "COPY_NUMBER_ALTERATION" || t === "MRNA_EXPRESSION" || t === "PROTEIN_LEVEL" || t === "METHYLATION") + return d === "CONTINUOUS" || d === "Z-SCORE" || d === "LOG2-VALUE" || d === "LOG-VALUE" ? "number" : null; + return null; +} + +/** the portal's order of data types in its Plots tab */ +const TYPE_ORDER = ["MUTATION_EXTENDED", "STRUCTURAL_VARIANT", "COPY_NUMBER_ALTERATION", "MRNA_EXPRESSION", "PROTEIN_LEVEL", "METHYLATION", "GENERIC_ASSAY"]; +/** cBioPortal's display names for its alteration types, as its Plots tab shows them */ +const TYPE_LABEL: Record<string, string> = { + MUTATION_EXTENDED: "Mutation", STRUCTURAL_VARIANT: "Structural variant", COPY_NUMBER_ALTERATION: "Copy number", + MRNA_EXPRESSION: "mRNA", PROTEIN_LEVEL: "Protein level", METHYLATION: "DNA methylation", +}; +/** cBioPortal's display names for the generic-assay kinds it knows; any other is shown as written */ +const ASSAY_LABEL: Record<string, string> = { + METHYLATION: "DNA methylation", PHOSPHOSITE_QUANTIFICATION: "Phosphosite quantification", ARMLEVEL_CNA: "Arm-level copy number", + TREATMENT_RESPONSE: "Treatment response", MUTATIONAL_SIGNATURE: "Mutational signature", GENETIC_ANCESTRY: "Genetic ancestry", +}; + +/** the Data picker's group a profile sits under */ +export function profileGroup(p: Pick<ProfileInfo, "type" | "assayType">): string { + if (p.type === "GENERIC_ASSAY") return ASSAY_LABEL[p.assayType || ""] || (p.assayType || "Generic assay").replace(/_/g, " ").toLowerCase().replace(/^./, (c) => c.toUpperCase()); + return TYPE_LABEL[p.type] || p.type; +} + +/** the profiles a plot axis can read, in the portal's order: by type, then the study's own order */ +export function plottableProfiles(ps: ProfileInfo[]): ProfileInfo[] { + const rank = (p: ProfileInfo) => { const i = TYPE_ORDER.indexOf(p.type); return i < 0 ? TYPE_ORDER.length : i; }; + return ps.filter((p) => profileUse(p)).sort((a, b) => rank(a) - rank(b) || + profileGroup(a).localeCompare(profileGroup(b)) || a.order - b.order || a.name.localeCompare(b.name)); +} diff --git a/lib/types.ts b/lib/types.ts index acf1495..c2cc6c2 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -1523,6 +1523,7 @@ export interface AlterationsResult extends TableRows<string | number | null> { } import type { PlotAxis, PlotData } from "./plot.ts"; import type { PointLayerSpec } from "./plot-layers.ts"; +import type { ProfileInfo } from "./profile-catalog.ts"; export type { PlotAxis, PlotData }; export type View = CompareView | SurvivalView | OncoprintView | CooccurrenceView | ClinicalView | CoexpressionView | HotspotsView | CourseView | SequenceView | PlotView | CoxView | LogisticView | AttritionView | GenomeView | PathwaysView | DiffexprView | PairedView | ClonalityView | IncidenceView | LabsView @@ -2177,5 +2178,7 @@ export interface ExploreResult { totals?: { patients: number; samples: number }; caseLists: { id: string; name: string; n: number }[]; profiles: string[]; gaProfiles: string[]; + /** every profile as the studies describe them (genetic_profile), for a plot axis's Data picker */ + profileCatalog?: ProfileInfo[]; stats: { ms: number; perStudy?: Record<string, number | null> }; } diff --git a/lib/views/cox.ts b/lib/views/cox.ts index 6dbb83a..c555c1f 100644 --- a/lib/views/cox.ts +++ b/lib/views/cox.ts @@ -6,7 +6,8 @@ import { isPatientLevel, type EndpointColumns } from "../federation.ts"; import { axisLabel, validateAxis, OQL_UNALTERED, type AxisKind } from "../plot.ts"; import { coxMethod, type CoxTrace } from "../method.ts"; import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts"; -import { overLimit } from "./shared.ts"; +import { overLimit, studyCatalogs } from "./shared.ts"; +import { resolveProfileAxis } from "../plot-profile.ts"; export interface CoxColumn { axisIndex: number; label: string; reference?: string; level?: string } /** @@ -69,10 +70,15 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un endpointColumns = { months: monthsAttr, status: statusAttr }; } const endpointLabel = endpointColumns ? endpointColumns.months + "/" + endpointColumns.status : endpoint; - const covariates = Array.isArray(cx.covariates) ? cx.covariates : []; + let covariates = Array.isArray(cx.covariates) ? cx.covariates : []; if (!covariates.length) return untimed({ kind: "cox", endpoint, error: "a cox view needs at least one covariate" }); { const e = overLimit("a cox view", 8, covariates.length); if (e) return untimed({ kind: "cox", endpoint, error: e.replace("at most 8", "at most 8 covariates") }); } - try { covariates.forEach((c, i) => validateAxis(c, "covariate " + (i + 1))); } + try { + // a profile covariate is read as the studies describe the profile; its reference level stays + const catalogs = await studyCatalogs(c, vStudies); + covariates = covariates.map((cv, i) => ({ ...resolveProfileAxis(cv, "covariate " + (i + 1), catalogs), ...(cv.reference != null ? { reference: cv.reference } : {}) })); + covariates.forEach((cv, i) => validateAxis(cv, "covariate " + (i + 1))); + } catch (e) { return untimed({ kind: "cox", endpoint, error: (e as Error).message }); } const rawCutoff = cx.cutoffMonths; let cutoffMonths: number | undefined; diff --git a/lib/views/logistic.ts b/lib/views/logistic.ts index 4b95919..5180fdb 100644 --- a/lib/views/logistic.ts +++ b/lib/views/logistic.ts @@ -7,7 +7,8 @@ import { axisLabel, validateAxis, type AxisKind } from "../plot.ts"; import { logisticMethod, type CoxTrace } from "../method.ts"; import { buildCoxColumns } from "./cox.ts"; import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts"; -import { overLimit } from "./shared.ts"; +import { overLimit, studyCatalogs } from "./shared.ts"; +import { resolveProfileAxis } from "../plot-profile.ts"; export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult | Untimed> { const { doc, cohorts, resolvers, vNames, vStudies, participationFor } = c; @@ -15,8 +16,15 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult // (buildCoxColumns, the same reference-level rules), same patient-level-only boundary — // but a binary outcome rather than a time-to-event one, so there is no endpoint/months // convention here: the outcome is just another axis, exactly like a covariate. - const lv = view as LogisticView; - try { validateAxis(lv.outcome, "outcome"); } + let lv = view as LogisticView; + try { + // profile axes are read as the studies describe the profile + const catalogs = await studyCatalogs(c, vStudies); + lv = { ...lv, outcome: resolveProfileAxis(lv.outcome, "outcome", catalogs), + ...(Array.isArray(lv.covariates) ? { covariates: lv.covariates.map((cv, i) => ({ ...resolveProfileAxis(cv, "covariate " + (i + 1), catalogs), + ...((cv as { reference?: string }).reference != null ? { reference: (cv as { reference?: string }).reference } : {}) })) } : {}) }; + validateAxis(lv.outcome, "outcome"); + } catch (e) { return untimed({ kind: "logistic", error: (e as Error).message }); } const outcomeLabel = axisLabel(lv.outcome); // alteration/molecular axes are deterministically categorical/numeric without a DB round diff --git a/lib/views/plot.ts b/lib/views/plot.ts index 98e6c96..85a1f0d 100644 --- a/lib/views/plot.ts +++ b/lib/views/plot.ts @@ -7,6 +7,7 @@ import { DEFAULT_BINS, CATEGORY_CAP, POINTS_CAP, OUTLIER_CELL_MAX, tukey, covera type AxisKind, type HighlightPoint, type PlotData, type PlotGroup, type PlotLayer, type PlotPoint, type PlotRequest, type PointRank } from "../plot.ts"; import { DEFAULT_POINT_LAYERS, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../plot-layers.ts"; import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts"; +import { resolveAxes } from "../plot-profile.ts"; type BoxRow = { g: string; n: number; lo: number; hi: number; q1: number; median: number; q3: number }; @@ -31,6 +32,16 @@ function poolBoxRows(parts: BoxRow[][]): PlotGroup[] { } type Def = ViewRun["doc"]["cohorts"][string]; + +/** copy-number levels in their own order, as the portal draws them; anything else keeps its place after them */ +const CNA_LEVELS = ["Deep deletion", "Shallow deletion", "Diploid", "Gain", "Amplification"]; +/** a box grouped by a discrete copy-number profile runs from deep deletion to amplification, not by size */ +function orderLevels(data: PlotData, spec: PlotRequest): void { + const cat = data.x.kind === "cat" ? spec.x : spec.y; + if (data.shape !== "box" || !data.groups || cat.source !== "molecular" || !cat.discrete) return; + const rank = (g: string) => { const i = CNA_LEVELS.indexOf(g); return i < 0 ? CNA_LEVELS.length : i; }; + data.groups = [...data.groups].sort((a, b) => rank(a.g) - rank(b.g)); +} const TABLE_WHY = "a contingency table counts cases and has no points to mark"; const fmt = (n: number) => n.toLocaleString("en-US"); @@ -298,8 +309,14 @@ export async function runPlot(view: DocView, c: ViewRun): Promise<ViewResult | U const sites = cohorts[cn].studies; // a point is a sample unless the view asks for patients, as on cBioPortal's Plots tab const unit = pv.unit === "patient" ? "patient" as const : "sample" as const; - const spec = { x: pv.x, y: pv.y, unit, - ...(pv.colorBy ? { colorBy: pv.colorBy } : {}), ...(pv.ringBy ? { ringBy: pv.ringBy } : {}) }; + // a profile axis is read as the studies describe the profile + let axes: { x: PlotView["x"]; y: PlotView["y"]; colorBy?: PlotView["x"]; ringBy?: PlotView["x"] }; + try { + const catalogs = await Promise.all(sites.map(async (st) => ({ study: st, catalog: (await c.studyProfiles(st)).catalog || [] }))); + axes = resolveAxes({ x: pv.x, y: pv.y, colorBy: pv.colorBy, ringBy: pv.ringBy }, catalogs); + } catch (e) { return untimed({ kind: "plot", error: String((e as Error).message || e) }); } + const spec = { x: axes.x, y: axes.y, unit, + ...(axes.colorBy ? { colorBy: axes.colorBy } : {}), ...(axes.ringBy ? { ringBy: axes.ringBy } : {}) }; const bins: [number, number] = Array.isArray(pv.bins) && pv.bins.length === 2 ? [Number(pv.bins[0]), Number(pv.bins[1])] : DEFAULT_BINS; const problem = layersProblem(pv, c, cn); @@ -308,6 +325,7 @@ export async function runPlot(view: DocView, c: ViewRun): Promise<ViewResult | U if (sites.length === 1) { const r0 = resolvers[sites[0]]; const data = await r0.plot(doc.cohorts[cn], { ...spec, bins: pv.bins ? bins : undefined }); + orderLevels(data, spec); await basePoints(pv, r0, doc.cohorts[cn], spec, data, sites[0]); const ls = await pointLayers(pv, data, c, cn, sites, spec); if (ls.length) data.layers = ls; @@ -418,6 +436,7 @@ export async function runPlot(view: DocView, c: ViewRun): Promise<ViewResult | U } } + orderLevels(data, spec); // each site counts its own cases and their axes; the sums are exact. A failed count fails the // view: dropping the coverage note would hide how many cases the figure leaves out const covs = await Promise.all(sites.map((s) => fr[s].plotCoverage(doc.cohorts[cn], spec, xk, yk))); diff --git a/lib/views/run.ts b/lib/views/run.ts index c7ed482..ef3e92b 100644 --- a/lib/views/run.ts +++ b/lib/views/run.ts @@ -1,5 +1,6 @@ // The contract between runAnalysis and the view runners: the context each view gets, and the refusal // that skips timing. +import type { ProfileInfo } from "../profile-catalog.ts"; import type { AnalysisDoc, ViewResult, ViewParticipation } from "../types.ts"; import type { Resolver } from "../federation.ts"; import type { RiskFit, RiskFitArgs } from "./riskmodel.ts"; @@ -27,7 +28,7 @@ export interface ViewRun { participationFor: (studies: string[], requiresPatientLevel: boolean) => ViewParticipation; pin: string | null; /** the molecular and generic-assay profiles a study holds (runAnalysis's cached inventory) */ - studyProfiles: (study: string) => Promise<{ profiles: string[]; gaProfiles: string[] }>; + studyProfiles: (study: string) => Promise<{ profiles: string[]; gaProfiles: string[]; catalog?: ProfileInfo[] }>; /** a paged view's page (the mutation and clinical tables): rows offset to offset + limit of its one global order */ page?: { offset: number; limit: number }; /** a table view's column whose values to return instead of rows (what its filter offers) */ diff --git a/lib/views/shared.ts b/lib/views/shared.ts index f81e7c3..e140ec3 100644 --- a/lib/views/shared.ts +++ b/lib/views/shared.ts @@ -4,6 +4,7 @@ import type { ViewResult } from "../types.ts"; import { hasViewAggregates, type Resolver, type ViewAggregateCapable } from "../federation.ts"; import { geneCytobandsSql } from "../federation-views.ts"; import type { ViewRun } from "./run.ts"; +import type { StudyCatalog } from "../plot-profile.ts"; /** 12 genes = 66 pairs; beyond this the multiplicity correction eats the signal */ export const COOCCUR_GENE_LIMIT = 12; @@ -51,3 +52,8 @@ export async function cytobands(ok: Record<string, ViewResolver>, genes: string[ const rows = await first.viewMeta("cytobands", (s, d) => geneCytobandsSql(genes, s, d)).catch(() => []); return new Map(rows.map((r) => [String(r.gene), String(r.cytoband || "")])); } + +/** each study's profiles as it describes them, for resolving a profile axis (lib/plot-profile.ts) */ +export async function studyCatalogs(c: ViewRun, studies: string[]): Promise<StudyCatalog[]> { + return Promise.all(studies.map(async (st) => ({ study: st, catalog: (await c.studyProfiles(st)).catalog || [] }))); +} diff --git a/test/fixtures/example-counts.json b/test/fixtures/example-counts.json index 31d70c4..6de6cc0 100644 --- a/test/fixtures/example-counts.json +++ b/test/fixtures/example-counts.json @@ -1,7 +1,7 @@ { "note": "Expected cohort counts for every example in lib/dsl-reference.ts, asserted by npm run docs:check. Regenerate with `npm run docs:counts` ONLY after reviewing why a number moved.", "oncokbPin": "v7.5", - "generated": "2026-10-14", + "generated": "2026-10-20", "examples": { "Lung adenocarcinoma: a four-view workup": { "EGFR-mutant NSCLC": 1893, @@ -81,6 +81,9 @@ "OQL on a plot axis: STK11 or KEAP1 mutant lung cancer": { "NSCLC": 7809 }, + "Plot axes from the study's own profiles: ERBB2 copy number against expression": { + "all samples": 1084 + }, "Merged gene rows: a pathway as one track": { "Pancreatic": 3109 }, diff --git a/test/plot-profile.test.ts b/test/plot-profile.test.ts new file mode 100644 index 0000000..d37f410 --- /dev/null +++ b/test/plot-profile.test.ts @@ -0,0 +1,84 @@ +// A profile axis: what the study's own description of a profile makes of it, how it is refused when +// the study lacks it or two studies disagree about it, and the SQL each reading runs. +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { profileUse, profileGroup, plottableProfiles, type ProfileInfo } from "../lib/profile-catalog.ts"; +import { resolveProfileAxis } from "../lib/plot-profile.ts"; +import { axisSelect, axisKind, axisLabel, validateAxis } from "../lib/plot.ts"; + +const P = (profile: string, type: string, datatype: string, name: string, extra: Partial<ProfileInfo> = {}): ProfileInfo => + ({ profile, type, datatype, name, order: 0, ...extra }); +const BRCA: ProfileInfo[] = [ + P("mutations", "MUTATION_EXTENDED", "MAF", "Mutations"), + P("structural_variants", "STRUCTURAL_VARIANT", "SV", "Structural variants"), + P("gistic", "COPY_NUMBER_ALTERATION", "DISCRETE", "Putative copy-number alterations from GISTIC"), + P("log2CNA", "COPY_NUMBER_ALTERATION", "LOG2-VALUE", "Log2 copy-number values"), + P("rna_seq_v2_mrna", "MRNA_EXPRESSION", "CONTINUOUS", "mRNA Expression, RSEM"), + P("rna_seq_v2_mrna_median_Zscores", "MRNA_EXPRESSION", "Z-SCORE", "mRNA expression z-scores relative to diploid samples"), + P("methylation_hm450", "GENERIC_ASSAY", "LIMIT-VALUE", "Methylation (HM450)", { assayType: "METHYLATION" }), + P("armlevel_cna", "GENERIC_ASSAY", "CATEGORICAL", "Putative arm-level copy-number from GISTIC", { assayType: "ARMLEVEL_CNA" }), +]; +const one = [{ study: "brca", catalog: BRCA }]; +const ctx = { study: "x", db: "db", unit: "sample" as const, cohortSql: "SELECT pid FROM p" }; + +describe("what a profile is, from the study's own description", () => { + test("type and datatype decide; the name never does", () => { + assert.deepEqual(BRCA.map((p) => profileUse(p)), ["mutation", "sv", "discrete", "number", "number", "number", "assayNumber", "assayCategory"]); + assert.equal(profileUse(P("rna_seq_v2_mrna", "MRNA_EXPRESSION", "DISCRETE", "x")), null, "a discrete mRNA profile is nothing an axis reads"); + assert.equal(profileUse(P("cna", "COPY_NUMBER_ALTERATION", "DISCRETE", "named like anything")), "discrete"); + }); + test("grouped under cBioPortal's own type names, generic assays under their own kind, in the portal's order", () => { + assert.deepEqual(BRCA.map(profileGroup), ["Mutation", "Structural variant", "Copy number", "Copy number", "mRNA", "mRNA", "DNA methylation", "Arm-level copy number"]); + assert.equal(profileGroup({ type: "GENERIC_ASSAY", assayType: "SOMETHING_NEW" }), "Something new", "an unknown kind is shown as written"); + const shuffled = [...BRCA].reverse(); + assert.deepEqual(plottableProfiles(shuffled).map((p) => p.type).slice(0, 4), ["MUTATION_EXTENDED", "STRUCTURAL_VARIANT", "COPY_NUMBER_ALTERATION", "COPY_NUMBER_ALTERATION"]); + }); +}); + +describe("a profile axis resolved against the studies it is drawn over", () => { + test("each reading resolves to the form that runs it", () => { + assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "mutations", gene: "tp53" }, "x", one), { source: "alteration", gene: "TP53", by: "mutationType" }); + assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "mutations", gene: "TP53", by: "mutated" }, "x", one), { source: "alteration", gene: "TP53", by: "mutated" }); + assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "structural_variants", gene: "ERBB2" }, "x", one), { source: "alteration", gene: "ERBB2", by: "sv" }); + assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2" }, "x", one), + { source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true, label: "ERBB2: Putative copy-number alterations from GISTIC" }); + assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "rna_seq_v2_mrna", gene: "ESR1", log: true }, "y", one), + { source: "molecular", profileType: "rna_seq_v2_mrna", gene: "ESR1", log: true, label: "ESR1: mRNA Expression, RSEM" }); + const clin = { source: "clinical" as const, attribute: "AGE" }; + assert.equal(resolveProfileAxis(clin, "x", one), clin, "any other axis passes through untouched"); + }); + test("refused by name: a profile the study lacks (saying what it has), one two studies describe differently, and settings it cannot take", () => { + assert.throws(() => resolveProfileAxis({ source: "profile", profile: "rppa", gene: "ESR1" }, "x", one), /^Error: x axis: brca has no profile 'rppa'; it has mutations, structural_variants, log2CNA, gistic, rna_seq_v2_mrna_median_Zscores/); + const other = [{ study: "brca", catalog: BRCA }, { study: "panel", catalog: [P("gistic", "COPY_NUMBER_ALTERATION", "LOG2-VALUE", "other")] }]; + assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2" }, "x", other), /described differently by brca \(COPY_NUMBER_ALTERATION, DISCRETE\) and panel/); + assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2", by: "mutated" }, "x", one), /only a mutation profile groups by/); + assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2", log: true }, "x", one), /a log scale applies to numbers/); + assert.throws(() => resolveProfileAxis({ source: "profile", profile: "methylation_hm450", gene: "ERBB2" }, "x", one), /is a generic assay \(Methylation \(HM450\)\), which a plot axis does not read yet/); + }); + test("a profile axis is shaped before it is read, and only once resolved does it have a kind", () => { + assert.throws(() => validateAxis({ source: "profile", profile: "", gene: "TP53" }, "x"), /is not a profile/); + assert.throws(() => validateAxis({ source: "profile", profile: "mutations", gene: "" }, "x"), /reads one gene/); + assert.throws(() => axisKind({ source: "profile", profile: "mutations", gene: "TP53" }), /only once resolved/); + assert.equal(axisKind({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true }), "cat"); + assert.equal(axisLabel({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true, label: "ERBB2: GISTIC" }), "ERBB2: GISTIC"); + }); +}); + +describe("the SQL each reading runs", () => { + test("a discrete profile's own levels, the portal's names, and Multiple for a patient whose samples disagree", () => { + const sql = axisSelect({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true }, ctx, "cat"); + assert.match(sql, /= -2, 'Deep deletion',[\s\S]*?= -1, 'Shallow deletion',[\s\S]*?= 0, 'Diploid',[\s\S]*?= 1, 'Gain',[\s\S]*?= 2, 'Amplification', toString\(/); + assert.match(sql, /ga\.profile_type = 'gistic'/); + assert.match(sql, /if\(uniqExact\(lv\) > 1, 'Multiple', any\(lv\)\)/); + }); + test("a gene's mutations, copy number or SVs: no such event is the wild-type answer only when the case is profiled for it", () => { + for (const [by, type, label] of [["mutated", "MUTATION_EXTENDED", "Not mutated"], ["cna", "COPY_NUMBER_ALTERATION", "Neutral"], ["sv", "STRUCTURAL_VARIANT", "No structural variant"]] as const) { + const sql = axisSelect({ source: "alteration", gene: "ERBB2", by }, ctx, "cat"); + assert.match(sql, new RegExp("if\\(hits = 0 AND NOT \\(alt_case IN \\(SELECT profiled_id"), by); + assert.match(sql, new RegExp("alteration_type = '" + type + "'"), by); + assert.match(sql, new RegExp("'" + label + "'"), by); + assert.match(sql, /'Not profiled', v0\) AS v/, by); + } + assert.match(axisSelect({ source: "alteration", gene: "ERBB2", by: "sv" }, ctx, "cat"), /g\.variant_type = 'structural_variant'/); + }); +}); diff --git a/test/plot.test.ts b/test/plot.test.ts index c06dbef..427b824 100644 --- a/test/plot.test.ts +++ b/test/plot.test.ts @@ -37,7 +37,7 @@ test("an axis that would read a whole profile is refused by name", () => { /needs one gene.*name the gene you mean/s); assert.throws(() => validateAxis({ source: "alteration" } as never, "y"), /needs a gene, a gene set, or an OQL query/); assert.throws(() => validateAxis({ source: "alateration", gene: "APC" } as never, "covariate 1"), - /unknown source 'alateration' — use one of clinical, molecular, alteration, none/); + /unknown source 'alateration' — use one of clinical, profile, molecular, alteration, none/); assert.throws(() => validateAxis({ source: "clinical", attribute: "not a name!" }, "x"), /not a clinical attribute name/); assert.throws(() => validateAxis({ source: "alteration", gene: "TP53", by: "driver" } as never, "x"), /not supported yet/); validateAxis(num, "x"); // the ordinary cases do not throw From bbc654f4326da95b7e2c5e4e11503dff343f83d6 Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 22:01:54 +0000 Subject: [PATCH 06/10] Plot axis Data list: drop "Gene set (mutated in any)", which OQL already asks The gene-set entry gave two groups, mutated in any of the genes or not. "OQL query", grouped by any line, asks the same with `GENE: MUT` per gene, and can say more (any alteration, amplification, a protein change), so the two entries read as one choice under two names. "Across genes" now offers only the OQL query. The engine still accepts the `genes` form, so documents and the gene-set Cox example work unchanged. The picker shows such an axis as its OQL equivalent ("TP53: MUT, KRAS: MUT", any line), and the first edit makes it an OQL axis. GeneSetInput, used only there, is removed. Verified in the browser: an axis of genes [TP53, KRAS] shows Data "OQL query" with "TP53: MUT, KRAS: MUT"; "Across genes" lists only OQL query; switching to one group per line writes {oql: "TP53: MUT, KRAS: MUT", by: "statement"}. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/view-chrome.tsx | 57 ++++++++++------------------------------- 1 file changed, 14 insertions(+), 43 deletions(-) diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index f67cb6c..19344e7 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -12,7 +12,7 @@ import { ArrowDown, ArrowUp, Ellipsis, ExternalLink, ListChecks, Lock, LockOpen, import { GeneQueryDialog } from "../_genes/GeneQueryDialog.tsx"; import type { GeneQueryMode, QueryCheck } from "../../lib/gene-query.ts"; import { oqlStatements } from "../../lib/oql-parse.ts"; -import { ALTERATION_GENE_SET_CAP, OQL_STATEMENT_CAP } from "../../lib/plot-oql.ts"; +import { OQL_STATEMENT_CAP } from "../../lib/plot-oql.ts"; import { plottableProfiles, profileGroup, profileUse, type ProfileInfo } from "../../lib/profile-catalog.ts"; import { Select, SelectContent, SelectGroup, SelectItem, SelectLabel, SelectTrigger, SelectValue } from "../_ui/select"; import { Button } from "../_ui/button"; @@ -93,7 +93,7 @@ function QueryButton({ mode, value, max, title, description, busy, onApply }: { /** * One axis of a plot. Its Data is one choice from a grouped list: a clinical attribute, each of the * study's own profiles under its alteration type (named as the study names it), or a query across - * genes (OQL, a gene set). What the choice needs next follows from it: an attribute; a gene, and for + * genes (an OQL query). What the choice needs next follows from it: an attribute; a gene, and for * a mutation profile how to group the mutations; or the query. */ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange, allowNone, categorical, stacked }: { @@ -129,7 +129,8 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange : a.source === "profile" ? "profile:" + a.profile : a.source === "molecular" ? "profile:" + a.profileType : a.source === "alteration" && a.oql != null ? "oql" - : a.source === "alteration" && Array.isArray(a.genes) && a.genes.length > 1 ? "genes" + // a gene set (any mutated) shows as the OQL query that asks the same + : a.source === "alteration" && Array.isArray(a.genes) && a.genes.length > 1 ? "oql" : a.source === "alteration" ? "profile:" + (byUse(a.by === "cna" ? "discrete" : a.by === "sv" ? "sv" : "mutation") || "") : "clinical"; const current = data.startsWith("profile:") ? catalog.find((p) => "profile:" + p.profile === data) : undefined; @@ -139,7 +140,6 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange if (v === "none") return set(undefined); if (v === "clinical") return set({ source: "clinical", attribute: a.attribute || (attrs[0] || {}).attribute || "" }); if (v === "oql") return set({ source: "alteration", oql: a.oql || gene + ": MUT" }); - if (v === "genes") return set({ source: "alteration", genes: Array.isArray(a.genes) && a.genes.length > 1 ? a.genes : [gene], by: "mutated" }); const p = catalog.find((x) => "profile:" + x.profile === v); if (p) set({ source: "profile", profile: p.profile, gene, ...(profileUse(p) === "mutation" ? { by: a.by === "mutated" ? "mutated" : "mutationType" } : {}) }); }; @@ -153,7 +153,7 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange const assay = p.type === "GENERIC_ASSAY"; at.items.push({ value: "profile:" + p.profile, label: p.name, ...(assay ? { disabled: "generic assays (" + p.name + ") come to plot axes next" } : {}) }); } - groups.push({ label: "Across genes", items: [{ value: "oql", label: "OQL query" }, { value: "genes", label: "Gene set (mutated in any)" }] }); + groups.push({ label: "Across genes", items: [{ value: "oql", label: "OQL query" }] }); const dataPicker = loading && !catalog.length && !attrs.length ? <Waiting what="profiles" /> : ( <Select value={data} onValueChange={choose} disabled={busy}> <SelectTrigger size="sm" aria-label={side + " data"} title={current ? current.name : "what this axis measures"} @@ -173,9 +173,9 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange </SelectContent> </Select> ); - const isOql = data === "oql", isSet = data === "genes"; - const genes: string[] = Array.isArray(a.genes) && a.genes.length ? a.genes : a.gene ? [a.gene] : []; - const commitGenes = (next: string[]) => set(next.length > 1 ? { source: "alteration", genes: next, by: "mutated" } : { source: "alteration", genes: next.slice(0, 1), by: "mutated" }); + const isOql = data === "oql"; + // the query as the OQL field shows it: the axis's own, or a gene set's equivalent ("any mutated") + const oql: string = a.oql != null ? a.oql : Array.isArray(a.genes) ? a.genes.map((g: string) => String(g).toUpperCase() + ": MUT").join(", ") : ""; return ( <span className={stacked ? "axispick-stack flex flex-col gap-2.5" : "axispick"}> {!stacked && <span className="tbl">{side}</span>} @@ -192,7 +192,7 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange {field("Gene", <GeneInput gene={gene} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(g) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: g })} />, <QueryButton mode="genes" value={gene} max={1} busy={busy} title="Gene for this axis" - description={"The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query or Gene set under Data."} + description={"The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query under Data."} onApply={(_t, c) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)} {use === "mutation" && field("Plot mutations by", <OptionSelect className={w} value={a.by === "mutated" ? "mutated" : "mutationType"} disabled={busy} ariaLabel={side + " mutations by"} @@ -207,21 +207,18 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange <OptionSelect className={w} value={a.by === "statement" ? "statement" : "any"} disabled={busy} ariaLabel={side + " OQL grouping"} title={a.by === "statement" ? "a group for each line of the query; a case matching two is their pair, none Unaltered (or Not profiled)" : "Altered when a case matches any line of the query; Unaltered when it matches none and is profiled for every gene, else Not profiled"} - onValueChange={(v) => set(v === "statement" ? { source: "alteration", oql: a.oql, by: "statement" } : { source: "alteration", oql: a.oql })}> + onValueChange={(v) => set(v === "statement" ? { source: "alteration", oql, by: "statement" } : { source: "alteration", oql })}> <option value="any">any line (Altered or not)</option> <option value="statement">one group per line</option> </OptionSelect>)} - {field("OQL query", <OqlInput query={a.oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={(oql) => set({ ...a, source: "alteration", oql })} />, - <QueryButton mode="oql" value={oqlStatements(a.oql || "").join("\n")} busy={busy} title="OQL for this axis" + {field("OQL query", <OqlInput query={oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} + onCommit={(q) => set({ source: "alteration", oql: q, ...(a.by === "statement" ? { by: "statement" } : {}) })} />, + <QueryButton mode="oql" value={oqlStatements(oql).join("\n")} busy={busy} title="OQL for this axis" max={a.by === "statement" ? OQL_STATEMENT_CAP : 30} description={a.by === "statement" ? "Each line is its own group; a case matching two is their pair. A bare gene means any alteration in it." : "A case is Altered if any line matches. A bare gene means any alteration in it; add OQL to be specific."} - onApply={(text) => set({ ...a, source: "alteration", oql: oqlStatements(text).join(", ") })} />)} + onApply={(text) => set({ source: "alteration", oql: oqlStatements(text).join(", "), ...(a.by === "statement" ? { by: "statement" } : {}) })} />)} </>} - {isSet && field("Genes", <GeneSetInput genes={genes} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} onCommit={commitGenes} />, - <QueryButton mode="genes" value={genes.join(" ")} max={ALTERATION_GENE_SET_CAP} busy={busy} title="Genes for this axis" - description="A case counts as mutated when any of these genes is. For anything finer, choose OQL query under Data." - onApply={(_t, c) => commitGenes(c.genes)} />)} </span> ); } @@ -250,32 +247,6 @@ export function GeneInput({ gene, busy, onCommit, className }: { gene: string; b ); } -/** - * One gene, or a gene SET typed as a comma-separated list ("TP53,KRAS,PIK3CA") — an alteration - * axis's "is ANY of these altered" covariate. Same debounce-before-commit shape as GeneInput, for - * the same reason: committing every keystroke would briefly make an empty or half-typed gene - * list the shareable link, which the compiler then rejects. - */ -export function GeneSetInput({ genes, busy, onCommit, className }: { genes: string[]; busy?: boolean; onCommit: (genes: string[]) => void; className?: string }) { - const joined = genes.join(","); - const [draft, setDraft] = useState(joined); - const timer = useRef<ReturnType<typeof setTimeout> | null>(null); - useEffect(() => setDraft(joined), [joined]); - const parse = (v: string) => v.split(",").map((g) => g.trim().toUpperCase()).filter(Boolean); - const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); onCommit(parse(v)); }; - return ( - <Input className={cn(FIELD, "w-auto max-w-56 min-w-22 px-1.5 uppercase", className)} value={draft} disabled={busy} placeholder="gene, or gene,gene,... for a set" size={Math.max(8, Math.min(30, draft.length + 2))} - title={'one gene, or a comma-separated gene set ("is any of these altered")'} - onChange={(e) => { - const v = e.target.value.toUpperCase(); - setDraft(v); - if (timer.current) clearTimeout(timer.current); - timer.current = setTimeout(() => onCommit(parse(v)), 500); - }} - onBlur={() => commit(draft)} /> - ); -} - /** * An OQL query typed into an alteration axis ("KRAS: G12C; STK11: MUT"), committed after a pause, * on Enter or on leaving the box — like the gene inputs, so a half-typed query never becomes the From 598a26ac255a1622d16a5e60df88e18c36675181 Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 22:16:04 +0000 Subject: [PATCH 07/10] Plot axes: genes and OQL chosen only in the gene dialog; the Data list's groups read as groups The Gene and OQL query fields of an axis are no longer text boxes. Each shows its value and opens the app's gene query dialog when clicked or typed into, so a half-typed or misspelt symbol never reaches the document; every entry is checked in the dialog first, with alias fixes offered. The fields do not open on keyboard focus alone: the dialog hands focus back on closing, and tabbing past would open it. The separate checklist button goes, since the field is now the trigger. GeneInput and OqlInput, used only there, are removed. The Data list's headers are a tinted band in small bold text, with a rule between groups and the profiles indented under them, so a header no longer reads as one more choice. Verified in the browser: - the axis block holds no text input; clicking Gene opens the dialog with TP53, and applying KRAS sets the axis; - focusing the field opens nothing, and typing a letter opens the dialog; - the menu's headers render as bands over indented items; - no page errors. The e2e OQL check now also asserts the axis has no free-text field. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/view-chrome.tsx | 96 +++++++++++------------------------------ e2e.cjs | 6 ++- 2 files changed, 30 insertions(+), 72 deletions(-) diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index 19344e7..7c6996b 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -2,7 +2,7 @@ // The chrome every view card shares, and the small controls inside it: the header (ViewHead), its // menus, the cohort and value pickers, the axis picker, the gene inputs. Module-level components, // so React keeps their state across the analysis page's renders. -import { useEffect, useRef, useState } from "react"; +import { useState } from "react"; import { attributeLabel } from "../../lib/custom-data-names.ts"; import type { AnalysisSource, ViewParticipation } from "../../lib/types.ts"; import { VIEW_HELP } from "../../lib/view-help.ts"; @@ -16,7 +16,6 @@ import { OQL_STATEMENT_CAP } from "../../lib/plot-oql.ts"; import { plottableProfiles, profileGroup, profileUse, type ProfileInfo } from "../../lib/profile-catalog.ts"; import { Select, SelectContent, SelectGroup, SelectItem, SelectLabel, SelectTrigger, SelectValue } from "../_ui/select"; import { Button } from "../_ui/button"; -import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; import { OptionSelect } from "../_ui/choice.tsx"; import { FIELD } from "../_ui/toolbar.ts"; @@ -74,22 +73,31 @@ export function Participation({ p }: { p?: ViewParticipation }) { } /** - * Beside a field that takes genes or OQL, the app's gene query dialog: search, a check of every - * entry as it is typed, alias fixes and remembered lists. The field stays editable as text; the - * dialog hands back what it applied. + * A field that takes genes or OQL, entered only through the app's gene query dialog (search, a check + * of every entry, alias fixes, remembered lists): it shows the value and opens the dialog when + * clicked or typed into, so a half-typed or misspelt gene never reaches the document. It does not + * open on focus alone, because the dialog hands focus back to it on closing. */ -function QueryButton({ mode, value, max, title, description, busy, onApply }: { - mode: GeneQueryMode; value: string; max: number; title: string; description: string; busy?: boolean; - onApply: (text: string, check: QueryCheck) => void; +function QueryField({ mode, value, shown, placeholder, max, title, description, busy, className, onApply }: { + mode: GeneQueryMode; value: string; shown: string; placeholder: string; max: number; title: string; description: string; + busy?: boolean; className?: string; onApply: (text: string, check: QueryCheck) => void; }) { const [open, setOpen] = useState(false); return <> - <Button type="button" variant="outline" size="icon-sm" className="size-ctl flex-none" disabled={busy} - aria-label={title} title={title + ": search genes and gene sets, and check every entry"} onClick={() => setOpen(true)}><ListChecks /></Button> + <button type="button" disabled={busy} aria-haspopup="dialog" aria-label={title + (shown ? ": " + shown : "")} + title={(shown || placeholder) + " — click to choose"} + className={cn(FIELD, "flex min-w-0 items-center gap-1.5 rounded-md border border-input px-1.5 text-left shadow-xs outline-none", + "hover:border-primary focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:opacity-50", className)} + onClick={() => setOpen(true)} + onKeyDown={(e) => { if (e.key.length === 1 && !e.metaKey && !e.ctrlKey && !e.altKey) { e.preventDefault(); setOpen(true); } }}> + <span className={cn("min-w-0 flex-1 truncate", !shown && "text-muted-foreground")}>{shown || placeholder}</span> + <ListChecks className="size-3.5 flex-none text-muted-foreground" aria-hidden="true" /> + </button> <GeneQueryDialog open={open} onOpenChange={setOpen} title={title} description={description} mode={mode} value={value} max={max} onApply={onApply} /> </>; } + /** * One axis of a plot. Its Data is one choice from a grouped list: a clinical attribute, each of the * study's own profiles under its alteration type (named as the study names it), or a query across @@ -164,10 +172,10 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange {allowNone && <SelectItem value="none" className="text-md">none</SelectItem>} {!groups.some((g) => g.items.some((i) => i.value === data)) && data !== "none" && <SelectItem value={data} className="text-md">{data.replace(/^profile:/, "")} (not in this study)</SelectItem>} - {groups.map((g) => ( - <SelectGroup key={g.label}> - <SelectLabel className="text-xs font-semibold text-muted-foreground">{g.label}</SelectLabel> - {g.items.map((i) => <SelectItem key={i.value} value={i.value} disabled={!!i.disabled} title={i.disabled || i.label} className="text-md">{i.label}</SelectItem>)} + {groups.map((g, gi) => ( + <SelectGroup key={g.label} className={cn(gi > 0 && "mt-1 border-t pt-1")}> + <SelectLabel className="mx-1 rounded-xs bg-muted px-2 py-1 text-xs font-semibold text-foreground">{g.label}</SelectLabel> + {g.items.map((i) => <SelectItem key={i.value} value={i.value} disabled={!!i.disabled} title={i.disabled || i.label} className="pl-5 text-md">{i.label}</SelectItem>)} </SelectGroup> ))} </SelectContent> @@ -189,9 +197,8 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange </OptionSelect> )} {current && use !== "assayNumber" && use !== "assayCategory" && <> - {field("Gene", <GeneInput gene={gene} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} - onCommit={(g) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: g })} />, - <QueryButton mode="genes" value={gene} max={1} busy={busy} title="Gene for this axis" + {field("Gene", <QueryField mode="genes" value={gene} shown={gene} placeholder="choose a gene" max={1} busy={busy} + className={stacked ? "w-full" : "w-28"} title="Gene for this axis" description={"The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query under Data."} onApply={(_t, c) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)} {use === "mutation" && field("Plot mutations by", @@ -211,9 +218,8 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange <option value="any">any line (Altered or not)</option> <option value="statement">one group per line</option> </OptionSelect>)} - {field("OQL query", <OqlInput query={oql} busy={busy} className={stacked ? "min-w-0 flex-1" : undefined} - onCommit={(q) => set({ source: "alteration", oql: q, ...(a.by === "statement" ? { by: "statement" } : {}) })} />, - <QueryButton mode="oql" value={oqlStatements(oql).join("\n")} busy={busy} title="OQL for this axis" + {field("OQL query", <QueryField mode="oql" value={oqlStatements(oql).join("\n")} shown={oqlStatements(oql).join(", ")} placeholder="choose genes or OQL" + className={stacked ? "w-full" : "w-56"} busy={busy} title="OQL for this axis" max={a.by === "statement" ? OQL_STATEMENT_CAP : 30} description={a.by === "statement" ? "Each line is its own group; a case matching two is their pair. A bare gene means any alteration in it." : "A case is Altered if any line matches. A bare gene means any alteration in it; add OQL to be specific."} @@ -223,56 +229,6 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange ); } -/** - * A gene name typed into a plot axis, debounced before it reaches the document/URL. Committing - * on every keystroke (as the rest of this axis picker does for selects) meant an in-progress - * edit — including the empty string between deleting the old gene and typing the new one — - * briefly became the shareable link, which the compiler then rejects ("a molecular axis needs - * one gene"). A short pause before committing keeps that mid-edit state local to the input. - */ -export function GeneInput({ gene, busy, onCommit, className }: { gene: string; busy?: boolean; onCommit: (gene: string) => void; className?: string }) { - const [draft, setDraft] = useState(gene); - const timer = useRef<ReturnType<typeof setTimeout> | null>(null); - useEffect(() => setDraft(gene), [gene]); - const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); onCommit(v); }; - return ( - <Input className={cn(FIELD, "w-22 px-1.5 uppercase", className)} value={draft} disabled={busy} placeholder="gene" aria-label="gene" - onChange={(e) => { - const v = e.target.value.toUpperCase(); - setDraft(v); - if (timer.current) clearTimeout(timer.current); - timer.current = setTimeout(() => onCommit(v), 500); - }} - onBlur={() => commit(draft)} /> - ); -} - -/** - * An OQL query typed into an alteration axis ("KRAS: G12C; STK11: MUT"), committed after a pause, - * on Enter or on leaving the box — like the gene inputs, so a half-typed query never becomes the - * shareable link. The engine checks it and the view says what it refused. - */ -export function OqlInput({ query, busy, onCommit, className }: { query: string; busy?: boolean; onCommit: (oql: string) => void; className?: string }) { - const [draft, setDraft] = useState(query); - const timer = useRef<ReturnType<typeof setTimeout> | null>(null); - useEffect(() => setDraft(query), [query]); - const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); if (v.trim() && v !== query) onCommit(v.trim()); }; - return ( - <Input className={cn(FIELD, "w-auto max-w-88 px-1.5", className)} value={draft} disabled={busy} placeholder="KRAS: G12C, STK11: MUT" spellCheck={false} - size={Math.max(16, Math.min(44, draft.length + 2))} - title={"OQL, one statement per line, ';' or ',' — e.g. KRAS: G12C, STK11: MUT, EGFR: AMP, TP53: DRIVER, ERBB2: EXP>2. " + - "Grouped by any line (Altered or not) or one group per line, as chosen beside it."} - onChange={(e) => { - const v = e.target.value; - setDraft(v); - if (timer.current) clearTimeout(timer.current); - timer.current = setTimeout(() => commit(v), 900); - }} - onKeyDown={(e) => { if (e.key === "Enter") commit(draft); }} - onBlur={() => commit(draft)} /> - ); -} - /** * The "?" in every view's header: a plain-language explanation of what this KIND of view is — * for a reader who isn't a scientist, not the DSL's own precise documentation (lib/dsl-reference.ts diff --git a/e2e.cjs b/e2e.cjs index 38d2f01..93632b2 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -621,6 +621,8 @@ async function main() { y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" } }] })).toString("base64"))); await page.waitForSelector(".plot-side", { timeout: 300000 }); + // genes and OQL are chosen in the dialog only: the axis holds no free-text field + const freeText = await page.locator(".plot-side .plot-axis input").count(); await page.getByRole("button", { name: "OQL for this axis" }).click(); const dlg = page.locator("[role=dialog]"); await dlg.waitFor({ timeout: 30000 }); @@ -629,8 +631,8 @@ async function main() { await dlg.getByRole("button", { name: /^Apply/ }).click({ timeout: 30000 }); await page.waitForFunction(() => /KEAP1/.test(JSON.parse(document.querySelector("#docjson").value).views[0].x.oql), null, { timeout: 30000 }); const x = await page.evaluate(() => JSON.parse(document.querySelector("#docjson").value).views[0].x); - oqlOk = opened === "KRAS: G12C\nSTK11: MUT" && x.oql === "KRAS: G12C, STK11: MUT, KEAP1: MUT" && x.by === "statement"; - oqlWhat = "opened " + JSON.stringify(opened) + " -> axis " + JSON.stringify(x); + oqlOk = freeText === 0 && opened === "KRAS: G12C\nSTK11: MUT" && x.oql === "KRAS: G12C, STK11: MUT, KEAP1: MUT" && x.by === "statement"; + oqlWhat = "free-text fields " + freeText + "; opened " + JSON.stringify(opened) + " -> axis " + JSON.stringify(x); } catch (e) { oqlWhat = String(e.message).split("\n")[0]; } ok("a plot's OQL axis is edited in the gene query dialog, and its lines become the axis's statements", oqlOk, oqlWhat); From a38afc6fd35de487134065fbe116267a27aa964d Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 22:55:06 +0000 Subject: [PATCH 08/10] Plot axes: several genes applied to a mutation axis make an OQL query with a group per gene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation profile axis reads one gene, because its groups (mutation types, not mutated, not profiled) describe one gene's mutations. Applying more than one gene in its gene dialog no longer stops at the one-gene limit: the axis becomes the OQL query "GENE: MUT, …" grouped one group per line, which is what several genes' mutations can honestly be (each gene, each pair, three or more, unaltered, not profiled). The dialog's description says so, and the axis then shows as Data "OQL query". Other profile axes still take one gene. Verified in the browser on brca_tcga_pan_can_atlas_2018: applying TP53 and PIK3CA to a TP53 mutation axis wrote {oql: "TP53: MUT, PIK3CA: MUT", by: "statement"} and drew Unaltered 443, PIK3CA 263, TP53 263, both 81, Not profiled 18. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/view-chrome.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index 7c6996b..49c02bb 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -197,10 +197,16 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange </OptionSelect> )} {current && use !== "assayNumber" && use !== "assayCategory" && <> - {field("Gene", <QueryField mode="genes" value={gene} shown={gene} placeholder="choose a gene" max={1} busy={busy} + {field("Gene", <QueryField mode="genes" value={gene} shown={gene} placeholder="choose a gene" busy={busy} className={stacked ? "w-full" : "w-28"} title="Gene for this axis" - description={"The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query under Data."} - onApply={(_t, c) => set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)} + // a mutation axis reads one gene; several become an OQL query with a group per gene + max={use === "mutation" ? OQL_STATEMENT_CAP : 1} + description={use === "mutation" + ? "The gene whose mutations this axis groups. Apply several to give each its own group: the axis becomes an OQL query, one group per gene." + : "The gene whose " + current.name + " this axis reads. To compare several genes, choose OQL query under Data."} + onApply={(_t, c) => c.genes.length > 1 && use === "mutation" + ? set({ source: "alteration", oql: c.genes.map((g) => g + ": MUT").join(", "), by: "statement" }) + : set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)} {use === "mutation" && field("Plot mutations by", <OptionSelect className={w} value={a.by === "mutated" ? "mutated" : "mutationType"} disabled={busy} ariaLabel={side + " mutations by"} title="group the cases by their mutation's type, or by whether they have one" From 6532c0de814cbf3686e508c16c08078cb7f739a5 Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 23:08:53 +0000 Subject: [PATCH 09/10] Cox and logistic leave out patients not profiled for a covariate's gene, and say how many A gene's alteration axis (and a discrete copy-number profile) answers "Not profiled" for a case whose panel omits the gene. A plot shows that as its own group. A model has no use for it as a level: an unknown status is neither wild type nor a hazard ratio worth reporting. Before panel-aware axes, those patients were silently fitted as wild type, in the reference group. coxRows and logisticRows now drop "Not profiled" values of the outcome and of each undecidable covariate in the row query itself, the query the method record's script rebuilds and re-fits, so the script reproduces exactly the rows the view fitted. A second query (coxNotProfiled:<endpoint>, logisticNotProfiled, labels the method data does not read as rows) counts those patients. The caption says how many were left out, separately from those missing the endpoint or a value, and each study's fetch step in the method record says how many of its missing rows were not profiled. Verified on brca_tcga_pan_can_atlas_2018: - Cox OS ~ TP53 mutated + ESR1 z-score: 1,064 patients, TP53 HR 1.07, no "Not profiled" term; the caption names 18 not profiled and 2 without the endpoint. - Logistic PIK3CA mutated ~ TP53 mutated: 1,066 patients, OR 0.52, 18 named. check:methods: every gallery Cox and logistic script reproduces its view, unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- lib/federation.ts | 58 +++++++++++++++++++++++++++++++++++-------- lib/method.ts | 7 ++++-- lib/views/cox.ts | 20 ++++++++++----- lib/views/logistic.ts | 12 +++++---- 4 files changed, 73 insertions(+), 24 deletions(-) diff --git a/lib/federation.ts b/lib/federation.ts index 63c0c2e..2c33ae7 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -27,7 +27,7 @@ import { compileCohortDef, eventTypeMatchSql, memberSamplesSql, lockDriftSql } f import { axisKind, axisLabel, axisSelect, corrPValue, plotShape, tukey, validateAxis, categoryColor, CATEGORY_CAP, COLOR_CAP, DEFAULT_BINS, POINTS_CAP, VIOLIN_BINS, plotBinsSql, plotBoxSql, plotCountSql, plotCoverageSql, plotMomentsSql, plotPointsSql, plotStatsSql, plotTableSql, plotViolinSql, - highlightCohortSql, plotHighlightCountSql, plotScatterPointsSql, plotRankedPointsSql, + highlightCohortSql, plotHighlightCountSql, plotScatterPointsSql, plotRankedPointsSql, OQL_NOT_PROFILED, type AxisKind, type HighlightPoint, type PlotAxis, type PlotData, type PlotPoint, type PlotRequest, type PlotShape, type PlotXYPoint, type SparseCells, type Whiskers, type PointRank } from "./plot.ts"; // re-exported so test/boundary.test.ts sweeps them with the other aggregate SQL builders @@ -807,6 +807,24 @@ export function anchoredSurvivalSql( * patient-level only (see PatientLevelResolver.coxRows) rather than something every * participant can answer. */ +/** an axis that can answer "Not profiled": a gene's alterations, or a discrete copy-number profile */ +function undecidable(a: PlotAxis): boolean { + return a.source === "alteration" || (a.source === "molecular" && !!a.discrete); +} +/** an axis's rows without its "Not profiled" cases */ +function decidedOnly(sel: string): string { + return "SELECT id, v FROM (\n" + sel + "\n) WHERE v != " + sq(OQL_NOT_PROFILED); +} +/** + * The SQL counting the rows of a model's query (before "Not profiled" is left out) that are not + * profiled on an undecidable axis: the patients a fit leaves out for that reason. null when no axis + * of the model can be undecided. + */ +function notProfiledCountSql(unfiltered: string, sels: { label: string }[], axes: PlotAxis[]): string | null { + const cols = sels.filter((_, i) => undecidable(axes[i])).map((s) => s.label + " = " + sq(OQL_NOT_PROFILED)); + return cols.length ? "SELECT countIf(" + cols.join(" OR ") + ") AS n FROM (\n" + unfiltered + "\n)" : null; +} + export function coxRowsSql( cohortSql: string, study: string, db: string, endpoint: string, covariates: { label: string; sel: string }[], columns?: EndpointColumns @@ -1289,10 +1307,10 @@ export interface PatientLevelCapable extends AggregateCapable { members(def: CohortDef | null | undefined): Promise<string[]>; /** per-patient rows for a Cox fit — see PatientLevelResolver.coxRows for why this is here and not on AggregateCapable */ coxRows(def: CohortDef | null | undefined, endpoint: string, covariates: PlotAxis[], columns?: EndpointColumns): - Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[] }>; + Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[]; notProfiled: number }>; /** per-patient rows for a logistic fit — same reason as coxRows */ logisticRows(def: CohortDef | null | undefined, outcome: PlotAxis, covariates: PlotAxis[]): - Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[] }>; + Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[]; notProfiled: number }>; /** one row per patient (never an identifier) for a view in lib/views/ (a lab waterfall) — see PatientLevelResolver.viewRows */ viewRows(label: string, def: CohortDef | null | undefined, build: (cohortSql: string, study: string, db: string) => string): Promise<ChRow[]>; /** a table view's rows (lib/table-query.ts) in this instance, with their ids — see PatientLevelResolver.rowsPage */ @@ -2061,21 +2079,33 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa * single row carries an identifier. `kinds` says which covariates came back numeric so the * caller (which dummy-codes categoricals) doesn't have to re-derive it. */ + /** a count query's answer, or 0 when there is nothing to count */ + private async countIfAny(label: string, sql: string | null, conn: ChConn): Promise<number> { + if (!sql) return 0; + const r = await this.runLogged<ChRow>(label, sql, conn); + return Number(r.data[0]?.n || 0); + } + async coxRows(def: CohortDef | null | undefined, endpoint: string, covariates: PlotAxis[], columns?: EndpointColumns): - Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[] }> { + Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[]; notProfiled: number }> { covariates.forEach((c, i) => validateAxis(c, "covariate " + (i + 1))); const db = this.instance.conn.db, conn = this.instance.conn; const kindsMap = await this.clinicalKinds(covariates); const kinds = covariates.map((c) => axisKind(c, (a) => kindsMap.get(a))); const ctx = { study: this.study, db, unit: "patient" as const, cohortSql: this.cohortSql(def), ...this.oqlCtx() }; const sels = covariates.map((c, i) => ({ label: "c" + i, sel: axisSelect(c, ctx, kinds[i]) })); - const sql = coxRowsSql(ctx.cohortSql, this.study, db, endpoint, sels, columns); - const r = await this.runLogged<ChRow>("coxRows:" + endpoint, sql, conn); + // a patient not profiled for a covariate's gene has no value to fit, so the fit leaves them + // out (in the query the method record re-runs) and counts them, rather than coding "unknown" + const decided = sels.map((s, i) => undecidable(covariates[i]) ? { ...s, sel: decidedOnly(s.sel) } : s); + const sql = coxRowsSql(ctx.cohortSql, this.study, db, endpoint, decided, columns); + const [r, np] = await Promise.all([ + this.runLogged<ChRow>("coxRows:" + endpoint, sql, conn), + this.countIfAny("coxNotProfiled:" + endpoint, notProfiledCountSql(coxRowsSql(ctx.cohortSql, this.study, db, endpoint, sels, columns), sels, covariates), conn)]); const rows = r.data.map((row) => ({ t: Number(row.m), event: Number(row.ev), values: covariates.map((_, i) => kinds[i] === "num" ? Number(row["c" + i]) : String(row["c" + i])) })); - return { rows, kinds }; + return { rows, kinds, notProfiled: np }; } /** @@ -2145,7 +2175,7 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa * level, or which already-0/1 numeric value, counts as the event. */ async logisticRows(def: CohortDef | null | undefined, outcome: PlotAxis, covariates: PlotAxis[]): - Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[] }> { + Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[]; notProfiled: number }> { validateAxis(outcome, "outcome"); covariates.forEach((c, i) => validateAxis(c, "covariate " + (i + 1))); const db = this.instance.conn.db, conn = this.instance.conn; @@ -2155,13 +2185,19 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa const ctx = { study: this.study, db, unit: "patient" as const, cohortSql: this.cohortSql(def), ...this.oqlCtx() }; const outcomeSel = { label: "y0", sel: axisSelect(outcome, ctx, outcomeKind) }; const sels = covariates.map((c, i) => ({ label: "c" + i, sel: axisSelect(c, ctx, kinds[i]) })); - const sql = logisticRowsSql(ctx.cohortSql, outcomeSel, sels); - const r = await this.runLogged<ChRow>("logisticRows", sql, conn); + // as for Cox: not profiled for the outcome's or a covariate's gene is left out and counted + const all = [outcomeSel, ...sels], axes = [outcome, ...covariates]; + const [dOut, ...dSels] = all.map((s, i) => undecidable(axes[i]) ? { ...s, sel: decidedOnly(s.sel) } : s); + const sql = logisticRowsSql(ctx.cohortSql, dOut, dSels); + const counted = all.map((s, i) => ({ ...s, label: i === 0 ? "y" : s.label })); + const [r, np] = await Promise.all([ + this.runLogged<ChRow>("logisticRows", sql, conn), + this.countIfAny("logisticNotProfiled", notProfiledCountSql(logisticRowsSql(ctx.cohortSql, outcomeSel, sels), counted, axes), conn)]); const rows = r.data.map((row) => ({ y: outcomeKind === "num" ? Number(row.y) : String(row.y), values: covariates.map((_, i) => kinds[i] === "num" ? Number(row["c" + i]) : String(row["c" + i])) })); - return { rows, outcomeKind, kinds }; + return { rows, outcomeKind, kinds, notProfiled: np }; } } diff --git a/lib/method.ts b/lib/method.ts index 43b09d6..0d676f6 100644 --- a/lib/method.ts +++ b/lib/method.ts @@ -115,7 +115,8 @@ export interface FitTrace { /** the cohort this fit belongs to, when each cohort is fitted separately */ scope?: string; /** per study: patients in the cohort there, and rows its query returned */ - perStudy: { study: string; cohortPatients: number; rows: number }[]; + /** each study's cohort patients, rows fetched, and patients left out as not profiled for an axis's gene */ + perStudy: { study: string; cohortPatients: number; rows: number; notProfiled?: number }[]; covariates: CoxCovariate[]; kinds: AxisKind[]; columns: MethodColumn[]; @@ -160,7 +161,9 @@ function fetchSteps(t: FitTrace, query: string, keep: string): MethodStep[] { kind: "fetch" as const, title: "Fetch one row per patient", detail: "The " + query + " query above. " + keep + " " + - (s.cohortPatients > s.rows ? plural(s.cohortPatients - s.rows, "cohort patient") + " had no row." : "Every cohort patient had a row."), + (s.cohortPatients > s.rows ? plural(s.cohortPatients - s.rows, "cohort patient") + " had no row" + + (s.notProfiled ? ", " + s.notProfiled.toLocaleString("en-US") + " of them because " + (s.notProfiled === 1 ? "it was" : "they were") + + " not profiled for the gene of an axis (a value of “Not profiled” is left out, never fitted as a level)" : "") + "." : "Every cohort patient had a row."), scope: s.study, rowsIn: s.cohortPatients, rowsOut: s.rows, diff --git a/lib/views/cox.ts b/lib/views/cox.ts index c555c1f..4d4c16d 100644 --- a/lib/views/cox.ts +++ b/lib/views/cox.ts @@ -47,6 +47,12 @@ export function buildCoxColumns(covariates: CoxCovariate[], kindsFinal: AxisKind return columns; } +/** what a model's caption says about the patients it left out as not profiled for a covariate's gene */ +export function notProfiledNote(n: number): string { + return n > 0 ? " " + n.toLocaleString("en-US") + " patient" + (n === 1 ? " was" : "s were") + " not profiled for the gene of a covariate and " + + (n === 1 ? "was" : "were") + " left out: an unknown alteration status is not wild type, and not a level to fit." : ""; +} + export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Untimed> { const { doc, cohorts, resolvers, vNames, vStudies, participationFor } = c; // Multivariate Cox: unlike survival/compare, this is fit from real per-patient rows @@ -94,7 +100,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un type RawCoxRow = { t: number; event: number; values: (number | string)[]; study: string }; type CoxFit = { error: string } | { coefficients: CoxCoefficient[]; n: number; events: number; termCount: number; - excludedRemote: { study: string; patients: number }[]; patientLevelTotal: number; trace: CoxTrace; + excludedRemote: { study: string; patients: number }[]; patientLevelTotal: number; notProfiled: number; trace: CoxTrace; }; const fitCoxFor = async (names: string[]): Promise<CoxFit> => { const cohortTotal = names.reduce((n, cn) => n + cohorts[cn].count, 0); @@ -104,7 +110,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un const excludedRemote: { study: string; patients: number }[] = []; const rawRows: RawCoxRow[] = []; let kinds: AxisKind[] | null = null; - let patientLevelTotal = 0; + let patientLevelTotal = 0, notProfiled = 0; const perStudy: CoxTrace["perStudy"] = []; for (const n of names) { for (const st of cohorts[n].studies) { @@ -114,10 +120,11 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un continue; } patientLevelTotal += cohorts[n].perStudy[st]?.count || 0; - const { rows, kinds: k } = await res.coxRows(doc.cohorts[n], endpoint, covariates, endpointColumns); + const { rows, kinds: k, notProfiled: np } = await res.coxRows(doc.cohorts[n], endpoint, covariates, endpointColumns); if (!kinds) kinds = k; + notProfiled += np; for (const r of rows) rawRows.push({ ...r, study: st }); - perStudy.push({ study: st, cohortPatients: cohorts[n].perStudy[st]?.count || 0, rows: rows.length }); + perStudy.push({ study: st, cohortPatients: cohorts[n].perStudy[st]?.count || 0, rows: rows.length, ...(np ? { notProfiled: np } : {}) }); } } if (!rawRows.length) { @@ -163,7 +170,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un censoredAtCutoff, droppedBadTime: rowsCut.length - coxInput.length, coefficients: columns.map((col, i) => ({ label: col.label, beta: fit.beta[i], se: fit.se[i], p: fit.p[i] })), }; - return { coefficients, n: fit.n, events: fit.events, termCount: columns.length, excludedRemote, patientLevelTotal, trace }; + return { coefficients, n: fit.n, events: fit.events, termCount: columns.length, excludedRemote, patientLevelTotal, notProfiled, trace }; }; const coxMethodOptions = { @@ -205,7 +212,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un } else { const fit0 = await fitCoxFor(vNames); if ("error" in fit0) return untimed({ kind: "cox", endpoint, cohorts: vNames, error: fit0.error }); - const missing = fit0.patientLevelTotal - fit0.n; + const missing = fit0.patientLevelTotal - fit0.n - fit0.notProfiled; const coxResult: ViewResult = { kind: "cox", cohorts: vNames, endpoint, ...(cutoffMonths != null ? { cutoffMonths } : {}), cox: { coefficients: fit0.coefficients, n: fit0.n, events: fit0.events, ...(fit0.excludedRemote.length ? { excludedRemote: fit0.excludedRemote } : {}) }, @@ -213,6 +220,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise<ViewResult | Un caveat: "Cox proportional-hazards regression (Efron ties) on " + fit0.n + " patients, " + fit0.events + " events, " + fit0.termCount + " term" + (fit0.termCount === 1 ? "" : "s") + "." + (missing > 0 ? " " + missing + " patient" + (missing === 1 ? "" : "s") + " from patient-level participants lacked the endpoint or a covariate and were excluded." : "") + + notProfiledNote(fit0.notProfiled) + (cutoffMonths != null ? " Administratively censored at " + cutoffMonths + " months." : "") + (cx.stratifyByStudy ? " Stratified by study: each study fits its own baseline hazard, so a case-mix difference between studies cannot masquerade as a covariate effect." diff --git a/lib/views/logistic.ts b/lib/views/logistic.ts index 5180fdb..4d0ff87 100644 --- a/lib/views/logistic.ts +++ b/lib/views/logistic.ts @@ -5,7 +5,7 @@ import type { ViewResult, LogisticView, LogisticCoefficient } from "../types.ts" import { isPatientLevel } from "../federation.ts"; import { axisLabel, validateAxis, type AxisKind } from "../plot.ts"; import { logisticMethod, type CoxTrace } from "../method.ts"; -import { buildCoxColumns } from "./cox.ts"; +import { buildCoxColumns, notProfiledNote } from "./cox.ts"; import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts"; import { overLimit, studyCatalogs } from "./shared.ts"; import { resolveProfileAxis } from "../plot-profile.ts"; @@ -53,7 +53,7 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult const lRawRows: RawLogisticRow[] = []; let outcomeKind: AxisKind | null = null; let lKinds: AxisKind[] | null = null; - let lPatientLevelTotal = 0; + let lPatientLevelTotal = 0, lNotProfiled = 0; const lPerStudy: CoxTrace["perStudy"] = []; for (const n of vNames) { for (const st of cohorts[n].studies) { @@ -63,11 +63,12 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult continue; } lPatientLevelTotal += cohorts[n].perStudy[st]?.count || 0; - const { rows, outcomeKind: ok, kinds: k } = await res.logisticRows(doc.cohorts[n], lv.outcome, lCovariates); + const { rows, outcomeKind: ok, kinds: k, notProfiled: np } = await res.logisticRows(doc.cohorts[n], lv.outcome, lCovariates); if (!outcomeKind) outcomeKind = ok; if (!lKinds) lKinds = k; + lNotProfiled += np; for (const r of rows) lRawRows.push({ ...r, study: st }); - lPerStudy.push({ study: st, cohortPatients: cohorts[n].perStudy[st]?.count || 0, rows: rows.length }); + lPerStudy.push({ study: st, cohortPatients: cohorts[n].perStudy[st]?.count || 0, rows: rows.length, ...(np ? { notProfiled: np } : {}) }); } } if (!lRawRows.length) { @@ -124,7 +125,7 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult try { lFit = logisticRegression(logisticInput); } catch (e) { return untimed({ kind: "logistic", cohorts: vNames, error: (e as Error).message }); } - const lMissing = lPatientLevelTotal - lFit.n; + const lMissing = lPatientLevelTotal - lFit.n - lNotProfiled; const lCoefficients: LogisticCoefficient[] = lColumns.map((col, i) => ({ label: col.label, ...(col.reference ? { reference: col.reference } : {}), ...(col.level ? { level: col.level } : {}), axisIndex: col.axisIndex, @@ -138,6 +139,7 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise<ViewResult (outcomeKindFinal === "cat" ? " = " + lv.outcomeEvent : "") + " (" + lColumns.length + " term" + (lColumns.length === 1 ? "" : "s") + ")." + (outcomeOther.length ? " Every other observed value of " + outcomeLabel + " (" + outcomeOther.join(", ") + ") was treated as “not event.”" : "") + (lMissing > 0 ? " " + lMissing + " patient" + (lMissing === 1 ? "" : "s") + " from patient-level participants lacked the outcome or a covariate and were excluded." : "") + + notProfiledNote(lNotProfiled).replace("the gene of a covariate", "the gene of the outcome or a covariate") + (vStudies.length > 1 ? " Clinical-attribute vocabularies and declared types are NOT harmonized across studies." : "") + (lExcludedRemote.length ? " " + lExcludedRemote.map((e) => e.study + " (" + e.patients + " patients)").join(", ") + From e47d59965c4145e321dc727a607659909302dacb Mon Sep 17 00:00:00 2001 From: Aaron Lisman <lisman.aaron@gmail.com> Date: Tue, 20 Oct 2026 23:14:46 +0000 Subject: [PATCH 10/10] Plot: hide the plotted cases or any point layer from its chip; the Data list's groups as bold text The row above the figure was "show one layer alone", which with a single layer could only fade the plotted cohort, never the layer's own points. It is now one toggle chip per drawn thing: the plotted cases ("all samples (13,087)") and each layer. Any of them can be hidden, alone or together, and Esc shows everything. It stays a reading gesture kept in browser state (invariant 9), re-running nothing. A case two layers share is drawn by the first. Its copy in a later layer is now tagged with the layer that drew it: the runner records sharedOwners beside `shared`, and the renderer writes one hidden group per owner, <g data-layer="k" data-shared data-owner="j">. So hiding layer j brings its shared cases back in layer k's colour, rather than hiding them from both. One CSS rule per key fades a hidden group (0.08) and shows the copies a hidden owner leaves. The Data list's group headers are plain bold text, a little space between groups, the profiles indented beneath, with no band or rule. Verified in the browser on msk_chord TP53-mutant samples with [outliers, trend]: - hiding outliers fades its group and shows trend's copies of the cases outliers drew; - hiding "all samples" fades the density; - Esc restores every group; the document is unchanged; no page errors. The figure test checks the owner-tagged shared groups; the e2e check is rewritten for the toggles. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/globals.css | 23 ++++++++++++----------- app/lab/plot-controls.tsx | 37 ++++++++++++++++++++----------------- app/lab/view-chrome.tsx | 6 +++--- e2e.cjs | 29 ++++++++++++++++++----------- lib/figures/plot.ts | 22 ++++++++++++++-------- lib/plot.ts | 4 +++- lib/views/plot.ts | 13 ++++++++----- test/view-svg.test.ts | 8 ++++---- 8 files changed, 82 insertions(+), 60 deletions(-) diff --git a/app/globals.css b/app/globals.css index 493a37b..06a3613 100644 --- a/app/globals.css +++ b/app/globals.css @@ -1235,17 +1235,18 @@ details.cohortpick .gearmenu-items { min-width: 22rem; max-width: 30rem; gap: 0; } .menu-aside > .meta { display: block; } -/* a plot's point layers: one shown alone, the rest stepped back, and the cases it shares with an - earlier layer revealed (app/lab/plot-controls.tsx; the groups are lib/figures/plot.ts's) */ -.plot-figure[data-solo] g[data-layer] { opacity: 0.08; } -.plot-figure[data-solo="0"] g[data-layer="0"], -.plot-figure[data-solo="1"] g[data-layer="1"], -.plot-figure[data-solo="2"] g[data-layer="2"], -.plot-figure[data-solo="3"] g[data-layer="3"] { opacity: 1; } -.plot-figure[data-solo="0"] g[data-layer="0"][data-shared], -.plot-figure[data-solo="1"] g[data-layer="1"][data-shared], -.plot-figure[data-solo="2"] g[data-layer="2"][data-shared], -.plot-figure[data-solo="3"] g[data-layer="3"][data-shared] { display: inline; } +/* a plot's point layers: the plotted cases and each layer hidden or shown, and a case a hidden + layer drew shown in the next layer that holds it (app/lab/plot-controls.tsx; the groups are + lib/figures/plot.ts's) */ +.plot-figure[data-hide~="base"] g[data-layer="base"], +.plot-figure[data-hide~="0"] g[data-layer="0"], +.plot-figure[data-hide~="1"] g[data-layer="1"], +.plot-figure[data-hide~="2"] g[data-layer="2"], +.plot-figure[data-hide~="3"] g[data-layer="3"] { opacity: 0.08; } +.plot-figure[data-hide~="0"] g[data-shared][data-owner="0"], +.plot-figure[data-hide~="1"] g[data-shared][data-owner="1"], +.plot-figure[data-hide~="2"] g[data-shared][data-owner="2"], +.plot-figure[data-hide~="3"] g[data-shared][data-owner="3"] { display: inline; } /* a plot's axis controls: source, then whatever that source needs */ .axispick { display: inline-flex; align-items: center; gap: var(--s1); } diff --git a/app/lab/plot-controls.tsx b/app/lab/plot-controls.tsx index 4b9ef44..759aaa2 100644 --- a/app/lab/plot-controls.tsx +++ b/app/lab/plot-controls.tsx @@ -14,7 +14,7 @@ import { Input } from "../_ui/input"; import { cn } from "../../lib/cn"; import { CHIP, FIELD } from "../_ui/toolbar.ts"; import type { ProfileInfo } from "../../lib/profile-catalog.ts"; -import { DEFAULT_POINT_LAYERS, POINTS_CAP, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../../lib/plot-layers.ts"; +import { DEFAULT_POINT_LAYERS, POINTS_CAP, LAYER_BASE, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../../lib/plot-layers.ts"; type Update = (fn: (view: any) => any) => void; /** the points cap as the controls print it */ @@ -129,18 +129,26 @@ export function PlotControls({ spec, plot, attrs, catalog, loading, busy, cohort children: React.ReactNode; }) { const layers: PointLayerSpec[] = Array.isArray(spec.points) ? spec.points : DEFAULT_POINT_LAYERS; - // One layer shown alone, the rest stepped back: a reading gesture, kept in the browser like a - // hover (invariant 9), and dropped when the layers drawn change. + // Which of the plotted cohort and its layers are hidden: a reading gesture, kept in the browser + // like a hover (invariant 9), and dropped when the layers drawn change. const drawn = (plot?.layers || []).map((l, i) => ({ ...l, i })).filter((l) => l.shown !== "none"); const drawnKey = drawn.map((l) => l.i + ":" + l.label).join("|"); - const [solo, setSolo] = useState<number | null>(null); - useEffect(() => setSolo(null), [drawnKey]); + const [hidden, setHidden] = useState<string[]>([]); + useEffect(() => setHidden([]), [drawnKey]); useEffect(() => { - if (solo == null) return; - const off = (e: KeyboardEvent) => { if (e.key === "Escape") setSolo(null); }; + if (!hidden.length) return; + const off = (e: KeyboardEvent) => { if (e.key === "Escape") setHidden([]); }; window.addEventListener("keydown", off); return () => window.removeEventListener("keydown", off); - }, [solo]); + }, [hidden]); + const toggle = (k: string) => setHidden((h) => h.includes(k) ? h.filter((x) => x !== k) : [...h, k]); + const chip = (k: string, color: string, text: string) => { + const shown = !hidden.includes(k); + return <Button key={k} type="button" variant="outline" size="xs" className={cn(CHIP, "h-ctl gap-1.5", !shown && "text-muted-foreground line-through")} aria-pressed={shown} + title={shown ? "hide " + text + " (Esc shows everything)" : "show " + text} onClick={() => toggle(k)}> + <span className="size-2.5 flex-none rounded-full" style={{ background: color, opacity: shown ? 1 : 0.35 }} />{text} + </Button>; + }; const axisName = (side: "x" | "y") => plot?.[side].label || side; const unit = spec.unit || "sample"; // A box is drawn with its categories across, whichever axis the document holds them on, so a @@ -213,17 +221,12 @@ export function PlotControls({ spec, plot, attrs, catalog, loading, busy, cohort onChange={(a) => update((vv) => { const n = { ...vv }; if (a) n.ringBy = a; else delete n.ringBy; return n; })} /> </div></>} {drawn.length > 0 && <div className="tbz plot-solo"> - <span className="tbl" title="this browser only: show one layer alone, the rest stepped back; Esc shows them all">show</span> - {drawn.map((l) => ( - <Button key={l.i} type="button" variant="outline" size="xs" className={cn(CHIP, "h-ctl gap-1.5")} aria-pressed={solo === l.i} - title={solo === l.i ? "show every layer again (Esc)" : "show “" + l.label + "” alone"} - onClick={() => setSolo(solo === l.i ? null : l.i)}> - <span className="size-2.5 flex-none rounded-full" style={{ background: l.color }} />{l.label} ({l.n.toLocaleString()}) - </Button> - ))} + <span className="tbl" title="this browser only: hide or show the plotted cases and each layer; Esc shows everything">show</span> + {chip("base", LAYER_BASE, "all " + unit + "s (" + plot!.n.toLocaleString() + ")")} + {drawn.map((l) => chip(String(l.i), l.color, l.label + " (" + l.n.toLocaleString() + ")"))} </div>} </div>} - <div className="plot-figure" data-solo={solo ?? undefined}>{children}</div> + <div className="plot-figure" data-hide={hidden.length ? hidden.join(" ") : undefined}>{children}</div> </div> </div> </div> diff --git a/app/lab/view-chrome.tsx b/app/lab/view-chrome.tsx index 49c02bb..9d1f115 100644 --- a/app/lab/view-chrome.tsx +++ b/app/lab/view-chrome.tsx @@ -173,9 +173,9 @@ export function AxisPicker({ side, axis, attrs, catalog, busy, loading, onChange {!groups.some((g) => g.items.some((i) => i.value === data)) && data !== "none" && <SelectItem value={data} className="text-md">{data.replace(/^profile:/, "")} (not in this study)</SelectItem>} {groups.map((g, gi) => ( - <SelectGroup key={g.label} className={cn(gi > 0 && "mt-1 border-t pt-1")}> - <SelectLabel className="mx-1 rounded-xs bg-muted px-2 py-1 text-xs font-semibold text-foreground">{g.label}</SelectLabel> - {g.items.map((i) => <SelectItem key={i.value} value={i.value} disabled={!!i.disabled} title={i.disabled || i.label} className="pl-5 text-md">{i.label}</SelectItem>)} + <SelectGroup key={g.label} className={cn(gi > 0 && "mt-1.5")}> + <SelectLabel className="px-2 pt-1 pb-0.5 text-md font-semibold text-foreground">{g.label}</SelectLabel> + {g.items.map((i) => <SelectItem key={i.value} value={i.value} disabled={!!i.disabled} title={i.disabled || i.label} className="pl-6 text-md">{i.label}</SelectItem>)} </SelectGroup> ))} </SelectContent> diff --git a/e2e.cjs b/e2e.cjs index 93632b2..e7a2231 100644 --- a/e2e.cjs +++ b/e2e.cjs @@ -592,24 +592,31 @@ async function main() { } catch (e) { layerWhat = String(e.message).split("\n")[0]; } ok("a plot's point layers are listed in the sidebar, and adding one draws it with a key entry", layerOk, layerWhat); - // Showing one layer alone is instant and the browser's own: the others step back, Esc restores. + // Hiding the plotted cases or a layer is instant and the browser's own: its group steps back, + // a case it drew reappears in the next layer holding it, Esc shows everything. let soloWhat = "no layer buttons", soloOk = false; try { - // every group, a layer's shared cases included: [layer, opacity, display] const groups = () => page.evaluate(() => [...document.querySelectorAll(".plot-figure g[data-layer]")] - .map((g) => [g.getAttribute("data-layer"), getComputedStyle(g).opacity, getComputedStyle(g).display])); + .map((g) => [g.getAttribute("data-layer") + (g.hasAttribute("data-shared") ? "<" + g.getAttribute("data-owner") : ""), getComputedStyle(g).opacity, getComputedStyle(g).display])); const docBefore = await page.locator("#docjson").inputValue(); - await page.locator(".plot-solo button").last().click(); - const solo = await page.locator(".plot-figure").getAttribute("data-solo"); - const alone = await groups(); + const chips = page.locator(".plot-solo button"); + const names = await chips.allInnerTexts(); + await chips.nth(1).click(); // the first layer + const hid = await groups(); + await chips.first().click(); // the plotted cases too + const both = await groups(); await page.keyboard.press("Escape"); const back = await groups(); - soloOk = solo != null && alone.some(([k]) => k === solo) && - alone.every(([k, o, d]) => k === solo ? o === "1" && d !== "none" : o !== "1") && - back.every(([, o]) => o === "1") && (await page.locator("#docjson").inputValue()) === docBefore; - soloWhat = "layer " + solo + " alone " + JSON.stringify(alone) + "; after Esc " + JSON.stringify(back) + "; document unchanged"; + const of = (gs, k) => gs.filter(([key]) => key === k); + soloOk = /^all samples/.test(names[0]) && names.length >= 2 && + of(hid, "0").every(([, o]) => o !== "1") && of(hid, "base").every(([, o]) => o === "1") && + hid.filter(([k]) => /<0$/.test(k)).every(([, o, d]) => o === "1" && d !== "none") && + of(both, "base").every(([, o]) => o !== "1") && + back.filter(([k]) => !k.includes("<")).every(([, o]) => o === "1") && + (await page.locator("#docjson").inputValue()) === docBefore; + soloWhat = "chips " + JSON.stringify(names) + "; first layer hidden " + JSON.stringify(hid) + "; after Esc " + JSON.stringify(back) + "; document unchanged"; } catch (e) { soloWhat = String(e.message).split("\n")[0]; } - ok("clicking a layer's button shows it alone, Esc shows every layer, and the document is untouched", soloOk, soloWhat); + ok("the plotted cases and each layer can be hidden from their chips, shared cases follow, Esc shows everything, and the document is untouched", soloOk, soloWhat); // An OQL axis is edited through the app's gene query dialog: it opens with one line per // statement, and what it applies is the axis's query. diff --git a/lib/figures/plot.ts b/lib/figures/plot.ts index 63e114e..4d8f899 100644 --- a/lib/figures/plot.ts +++ b/lib/figures/plot.ts @@ -5,7 +5,7 @@ // picture draws for a hundred cases or four hundred thousand. The only cases drawn one by one are // the points the engine chose (every case under the cap, the outliers over it), at most POINTS_CAP. import type { ViewResult } from "../types.ts"; -import type { HighlightPoint, PlotData } from "../plot.ts"; +import type { HighlightPoint, PlotData, PlotLayer } from "../plot.ts"; import { isAbsent, jitterFor } from "../plot-draw.ts"; import { LAYER_BASE } from "../plot-layers.ts"; import { @@ -67,12 +67,18 @@ function layerItems(d: PlotData): LegendItem[] { /** the layers drawn at all: with any, the plotted cohort steps back to a grey so their colours read */ const drawnLayers = (d: PlotData) => (d.layers || []).filter((l) => l.shown !== "none"); /** - * One layer's marks as a group tagged with its place in the view's list, so a page can show that - * layer alone; `base` is the plotted cohort. A layer's cases an earlier layer drew are a second, - * hidden group the page reveals only when that layer stands alone. + * One layer's marks as a group tagged with its place in the view's list, so a page can hide it; + * `base` is the plotted cohort. A layer's cases an earlier layer drew are hidden groups, one per + * layer that drew them (`owner`), which the page shows when that owner is hidden. */ -const layerGroup = (key: number | "base", marks: string[], shared = false) => - marks.length ? `<g data-layer="${key}"${shared ? ` data-shared="" display="none"` : ""}>${marks.join("")}</g>` : ""; +const layerGroup = (key: number | "base", marks: string[], owner?: number) => + marks.length ? `<g data-layer="${key}"${owner != null ? ` data-shared="" data-owner="${owner}" display="none"` : ""}>${marks.join("")}</g>` : ""; +/** a layer's shared cases, grouped by the layer that drew each */ +function sharedGroups(h: PlotLayer, key: number, mark: (pt: HighlightPoint) => string[]): string { + const by = new Map<number, string[]>(); + (h.shared || []).forEach((pt, i) => { const o = h.sharedOwners?.[i] ?? 0; (by.get(o) ?? by.set(o, []).get(o)!).push(...mark(pt)); }); + return [...by.entries()].map(([o, marks]) => layerGroup(key, marks, o)).join(""); +} function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { const { x0, x1, y0, y1 } = d.extent!, [gx, gy] = d.grid!; @@ -122,7 +128,7 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { } if (h.shown === "points") { out.push(layerGroup(key, h.points!.flatMap((pt) => "x" in pt ? [dot(pt, h.color)] : []))); - out.push(layerGroup(key, (h.shared || []).flatMap((pt) => "x" in pt ? [dot(pt, h.color)] : []), true)); + out.push(sharedGroups(h, key, (pt) => "x" in pt ? [dot(pt, h.color)] : [])); } } // the fitted line, over every case rather than over what is visible, clipped to the frame @@ -304,7 +310,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { }); const key = d.layers!.indexOf(h); out.push(layerGroup(key, marks(h.points!, true))); - out.push(layerGroup(key, marks(h.shared || [], false), true)); + out.push(sharedGroups(h, key, (pt) => marks([pt], false))); } }); out.push(`<line x1="${L}" y1="${T + PLOT}" x2="${W - R}" y2="${T + PLOT}" stroke="${p.muted}" stroke-width="${SW.line}"/>`); diff --git a/lib/plot.ts b/lib/plot.ts index 89aae63..e4f6a04 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -760,8 +760,10 @@ export interface PlotLayer { n: number; /** points drawn, once cases an earlier layer already drew are left to it */ drawn?: number; - /** the layer's cases an earlier layer drew: hidden on the figure, shown when the reader picks this layer alone */ + /** the layer's cases an earlier layer drew: hidden on the figure, shown when the layer that drew them is hidden */ shared?: HighlightPoint[]; + /** for each shared case, the place in the list of the layer that drew it */ + sharedOwners?: number[]; shown: "points" | "density" | "boxes" | "none"; /** why it is not drawn as points, when it is not */ why?: string; diff --git a/lib/views/plot.ts b/lib/views/plot.ts index 85a1f0d..7e8bc91 100644 --- a/lib/views/plot.ts +++ b/lib/views/plot.ts @@ -132,7 +132,8 @@ async function pointLayers(pv: PlotView, data: PlotData, c: ViewRun, cn: string, const def = c.doc.cohorts[cn]; const xk = data.x.kind, yk = data.y.kind; const closed = sites.filter((s) => !isPatientLevel(c.resolvers[s])); - const seen = new Set<string>(); + // each drawn case, and the layer (its place in the list) that drew it + const seen = new Map<string, number>(); let budget = POINTS_CAP; const out: PlotLayer[] = []; for (const [i, l] of specs.entries()) { @@ -147,10 +148,12 @@ async function pointLayers(pv: PlotView, data: PlotData, c: ViewRun, cn: string, const { layer, keys } = picked; if (layer.shown === "points" && layer.points && keys) { const kept = layer.points.filter((_, j) => !seen.has(keys[j])); - const shared = layer.points.filter((_, j) => seen.has(keys[j])); - for (const k of keys) seen.add(k); - layer.drawn = kept.length; layer.points = kept; - if (shared.length) layer.shared = shared; + const sharedAt = keys.map((k, j) => seen.has(k) ? j : -1).filter((j) => j >= 0); + const owners = sharedAt.map((j) => seen.get(keys[j])!); + for (const k of keys) if (!seen.has(k)) seen.set(k, i); + layer.drawn = kept.length; + if (sharedAt.length) { layer.shared = sharedAt.map((j) => layer.points![j]); layer.sharedOwners = owners; } + layer.points = kept; budget -= kept.length; } out.push(layer); diff --git a/test/view-svg.test.ts b/test/view-svg.test.ts index 72dae3b..d9248b7 100644 --- a/test/view-svg.test.ts +++ b/test/view-svg.test.ts @@ -404,16 +404,16 @@ describe("plot point layers: subsets drawn over the plotted cohort", () => { assert.match(svg, /1 value of a point layer lies more than a whisker span past the/); }); - test("each layer is a group tagged with its place in the list, its shared cases a hidden group of their own", () => { + test("each layer is a group tagged with its place in the list, its shared cases hidden groups by the layer that drew them", () => { const svg = viewToSvg(scatter([ hl({ points: [{ id: "a", x: 10, y: 5 }] }), - hl({ name: "S", color: LAYER_COLORS[1], points: [{ id: "b", x: 20, y: 8 }], shared: [{ id: "a", x: 10, y: 5 }] }), + hl({ name: "S", color: LAYER_COLORS[1], points: [{ id: "b", x: 20, y: 8 }], shared: [{ id: "a", x: 10, y: 5 }], sharedOwners: [0] }), ]))!; assert.match(svg, /<g data-layer="base">/); assert.match(svg, /<g data-layer="0"><circle/); assert.match(svg, /<g data-layer="1"><circle[^>]*fill="#0072B2"/); - assert.match(svg, /<g data-layer="1" data-shared="" display="none"><circle cx="[^"]*" cy="[^"]*" r="2.5" fill="#0072B2"/, - "the case layer 0 drew is drawn for layer 1 too, hidden until that layer is shown alone"); + assert.match(svg, /<g data-layer="1" data-shared="" data-owner="0" display="none"><circle cx="[^"]*" cy="[^"]*" r="2.5" fill="#0072B2"/, + "the case layer 0 drew is drawn for layer 1 too, hidden until layer 0 is hidden"); }); test("with the cases coloured by a category, a layer is a ring round the case, not a second fill", () => {