+ : <>{control}{extra}>;
+ const profiles = plottableProfiles(catalog);
+ const offered = (p: ProfileInfo) => { const u = profileUse(p); return !categorical || u === "mutation" || u === "sv" || u === "discrete" || u === "assayCategory"; };
+ // what the axis holds, as one of the Data list's values; a form the document gives directly
+ // (a molecular profile, a gene's alterations) shows as the profile it reads
+ const byUse = (u: string) => profiles.find((p) => profileUse(p) === u)?.profile;
+ const data: string = !axis ? "none"
+ : a.source === "clinical" ? "clinical"
+ : a.source === "profile" ? "profile:" + a.profile
+ : a.source === "molecular" ? "profile:" + a.profileType
+ : a.source === "alteration" && a.oql != null ? "oql"
+ // a gene set (any mutated) shows as the OQL query that asks the same
+ : a.source === "alteration" && Array.isArray(a.genes) && a.genes.length > 1 ? "oql"
+ : a.source === "alteration" ? "profile:" + (byUse(a.by === "cna" ? "discrete" : a.by === "sv" ? "sv" : "mutation") || "")
+ : "clinical";
+ const current = data.startsWith("profile:") ? catalog.find((p) => "profile:" + p.profile === data) : undefined;
+ const use = current ? profileUse(current) : null;
+ const gene = String(a.gene || (Array.isArray(a.genes) ? a.genes[0] : "") || (a.oql ? (/^\s*([A-Za-z0-9.-]+)/.exec(a.oql) || [])[1] || "" : "") || "TP53").toUpperCase();
+ const choose = (v: string) => {
+ if (v === "none") return set(undefined);
+ if (v === "clinical") return set({ source: "clinical", attribute: a.attribute || (attrs[0] || {}).attribute || "" });
+ if (v === "oql") return set({ source: "alteration", oql: a.oql || gene + ": MUT" });
+ const p = catalog.find((x) => "profile:" + x.profile === v);
+ if (p) set({ source: "profile", profile: p.profile, gene, ...(profileUse(p) === "mutation" ? { by: a.by === "mutated" ? "mutated" : "mutationType" } : {}) });
+ };
+ // the Data list: the profiles grouped under their type, in the study's own order
+ const groups: { label: string; items: { value: string; label: string; disabled?: string }[] }[] = [];
+ groups.push({ label: "Clinical", items: [{ value: "clinical", label: "Clinical attribute" }] });
+ for (const p of profiles) {
+ if (!offered(p)) continue;
+ const g = profileGroup(p);
+ const at = groups.find((x) => x.label === g) || (groups.push({ label: g, items: [] }), groups[groups.length - 1]);
+ const assay = p.type === "GENERIC_ASSAY";
+ at.items.push({ value: "profile:" + p.profile, label: p.name, ...(assay ? { disabled: "generic assays (" + p.name + ") come to plot axes next" } : {}) });
+ }
+ groups.push({ label: "Across genes", items: [{ value: "oql", label: "OQL query" }] });
+ const dataPicker = loading && !catalog.length && !attrs.length ? : (
+
+ );
+ const isOql = data === "oql";
+ // the query as the OQL field shows it: the axis's own, or a gene set's equivalent ("any mutated")
+ const oql: string = a.oql != null ? a.oql : Array.isArray(a.genes) ? a.genes.map((g: string) => String(g).toUpperCase() + ": MUT").join(", ") : "";
return (
{!stacked && {side}}
- {field("Data type", {
- const s = v;
- // Keep whatever this axis last held for each source (spreading `a` rather than
- // starting fresh), so switching away and back round-trips instead of always
- // reverting to the catalog's first entry — a molecular axis already did this for
- // its gene; clinical's attribute and molecular's profile now do too.
- set(s === "none" ? undefined
- : s === "clinical" ? { ...a, source: "clinical", attribute: a.attribute || (attrs[0] || {}).attribute || "" }
- : s === "molecular" ? { ...a, source: "molecular", profileType: a.profileType || profiles[0] || "", gene: a.gene || "TP53" }
- : { ...a, source: "alteration", gene: a.gene || "TP53", by: a.by || "mutationType" });
- }}>
- {allowNone && }
-
- {!categorical && }
-
- )}
- {src === "clinical" && loading && !attrs.length && field(srcName.clinical, )}
- {src === "clinical" && !(loading && !attrs.length) && field(srcName.clinical,
+ {field("Data", dataPicker)}
+ {data === "clinical" && loading && !attrs.length && field("Clinical attribute", )}
+ {data === "clinical" && !(loading && !attrs.length) && field("Clinical attribute",
set({ ...a, attribute: v })}>
{!attrs.some((x) => x.attribute === a.attribute) && }
{attrs.map((x) => )}
)}
- {src === "molecular" && <>
- {field(srcName.molecular, loading && !profiles.length ? :
- set({ ...a, profileType: v })}>
- {!profiles.includes(a.profileType || "") && }
- {profiles.map((pt) => )}
- )}
- {field("Gene", set({ ...a, gene })} />)}
- >}
- {src === "alteration" && (() => {
- const isOql = a.oql != null;
- const genes: string[] = a.genes && a.genes.length ? a.genes : a.gene ? [a.gene] : [];
- const isSet = !isOql && genes.length > 1;
- const groupBy = {
- const by = v;
- // to OQL: the genes become a query (a bare gene is "any alteration"); from OQL: the
- // query's first gene, so a round trip keeps the gene
- if (by === "oql") set({ source: "alteration", oql: genes.length ? genes.join("; ") : "TP53: MUT" });
- else if (isOql) set({ source: "alteration", gene: (/^\s*([A-Za-z0-9.-]+)/.exec(a.oql) || [])[1]?.toUpperCase() || "TP53", by });
- else set({ ...a, by });
- }}>
-
+ {current && use !== "assayNumber" && use !== "assayCategory" && <>
+ {field("Gene", c.genes.length > 1 && use === "mutation"
+ ? set({ source: "alteration", oql: c.genes.map((g) => g + ": MUT").join(", "), by: "statement" })
+ : set({ ...(a.source === "profile" ? a : { source: "profile", profile: current.profile }), gene: c.genes[0] || "" })} />)}
+ {use === "mutation" && field("Plot mutations by",
+ set({ source: "profile", profile: current.profile, gene, by: v })}>
+
-
-
- ;
- // stacked, the grouping comes first: it decides whether the box below takes genes or a query
- return <>
- {stacked && field("Group cases by", groupBy)}
- {field(isOql ? "OQL query" : "Gene(s)", isOql
- ? set({ source: "alteration", oql })} />
- : {
- // A gene SET (2+) only supports "mutated vs not" — combining several genes'
- // mutation types or copy-number states into one category has no honest reading,
- // so switching to a set falls back to that rather than leaving an axis the
- // engine would refuse.
- const by = next.length > 1 && a.by && a.by !== "mutated" ? "mutated" : a.by;
- set(next.length > 1 ? { ...a, gene: undefined, genes: next, by }
- : { ...a, genes: undefined, gene: next[0] || "", by });
- }} />)}
- {!stacked && groupBy}
- >;
- })()}
+ )}
+ >}
+ {isOql && <>
+ {field("Group cases by",
+ set(v === "statement" ? { source: "alteration", oql, by: "statement" } : { source: "alteration", oql })}>
+
+
+ )}
+ {field("OQL query", set({ source: "alteration", oql: oqlStatements(text).join(", "), ...(a.by === "statement" ? { by: "statement" } : {}) })} />)}
+ >}
);
}
-/**
- * A gene name typed into a plot axis, debounced before it reaches the document/URL. Committing
- * on every keystroke (as the rest of this axis picker does for selects) meant an in-progress
- * edit — including the empty string between deleting the old gene and typing the new one —
- * briefly became the shareable link, which the compiler then rejects ("a molecular axis needs
- * one gene"). A short pause before committing keeps that mid-edit state local to the input.
- */
-export function GeneInput({ gene, busy, onCommit, className }: { gene: string; busy?: boolean; onCommit: (gene: string) => void; className?: string }) {
- const [draft, setDraft] = useState(gene);
- const timer = useRef | null>(null);
- useEffect(() => setDraft(gene), [gene]);
- const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); onCommit(v); };
- return (
- {
- const v = e.target.value.toUpperCase();
- setDraft(v);
- if (timer.current) clearTimeout(timer.current);
- timer.current = setTimeout(() => onCommit(v), 500);
- }}
- onBlur={() => commit(draft)} />
- );
-}
-
-/**
- * One gene, or a gene SET typed as a comma-separated list ("TP53,KRAS,PIK3CA") — an alteration
- * axis's "is ANY of these altered" covariate. Same debounce-before-commit shape as GeneInput, for
- * the same reason: committing every keystroke would briefly make an empty or half-typed gene
- * list the shareable link, which the compiler then rejects.
- */
-export function GeneSetInput({ genes, busy, onCommit, className }: { genes: string[]; busy?: boolean; onCommit: (genes: string[]) => void; className?: string }) {
- const joined = genes.join(",");
- const [draft, setDraft] = useState(joined);
- const timer = useRef | null>(null);
- useEffect(() => setDraft(joined), [joined]);
- const parse = (v: string) => v.split(",").map((g) => g.trim().toUpperCase()).filter(Boolean);
- const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); onCommit(parse(v)); };
- return (
- {
- const v = e.target.value.toUpperCase();
- setDraft(v);
- if (timer.current) clearTimeout(timer.current);
- timer.current = setTimeout(() => onCommit(parse(v)), 500);
- }}
- onBlur={() => commit(draft)} />
- );
-}
-
-/**
- * An OQL query typed into an alteration axis ("KRAS: G12C; STK11: MUT"), committed after a pause,
- * on Enter or on leaving the box — like the gene inputs, so a half-typed query never becomes the
- * shareable link. The engine checks it and the view says what it refused.
- */
-export function OqlInput({ query, busy, onCommit, className }: { query: string; busy?: boolean; onCommit: (oql: string) => void; className?: string }) {
- const [draft, setDraft] = useState(query);
- const timer = useRef | null>(null);
- useEffect(() => setDraft(query), [query]);
- const commit = (v: string) => { if (timer.current) clearTimeout(timer.current); if (v.trim() && v !== query) onCommit(v.trim()); };
- return (
- 2. " +
- "A case is Altered when it matches any of them."}
- onChange={(e) => {
- const v = e.target.value;
- setDraft(v);
- if (timer.current) clearTimeout(timer.current);
- timer.current = setTimeout(() => commit(v), 900);
- }}
- onKeyDown={(e) => { if (e.key === "Enter") commit(draft); }}
- onBlur={() => commit(draft)} />
- );
-}
-
/**
* The "?" in every view's header: a plain-language explanation of what this KIND of view is —
* for a reader who isn't a scientist, not the DSL's own precise documentation (lib/dsl-reference.ts
diff --git a/app/portal/PortalTabs.tsx b/app/portal/PortalTabs.tsx
index 0dab91f5..756cd8fd 100644
--- a/app/portal/PortalTabs.tsx
+++ b/app/portal/PortalTabs.tsx
@@ -148,7 +148,7 @@ export function PortalTabs({ doc, tabs, titles, open, onOpen, onDoc, onCounts, c
{hasKindCard(spec.kind as string) && }
- {spec.kind === "plot" && edit(vi, fn(spec))}
cohortOptions={Object.keys(doc.cohorts || {}).filter((n) => n !== drawn[0]).map((n) => ({ name: n, label: n, depth: 0 }))}>
{figureArea}
diff --git a/app/portal/comparison/page.tsx b/app/portal/comparison/page.tsx
index d3dee03d..6ff2da62 100644
--- a/app/portal/comparison/page.tsx
+++ b/app/portal/comparison/page.tsx
@@ -16,7 +16,7 @@ const CLINICAL_SHOWN = 8;
function comparisonLayout(linked: AnalysisDoc, study: string, known: StudyKnown): PageLayout {
const names = Object.keys(linked.cohorts || {});
- const expr = expressionProfile(known.profiles);
+ const expr = expressionProfile(known.profileCatalog);
const ga = known.gaProfiles.find((p) => /signature/i.test(p) && /contribution/i.test(p)) || known.gaProfiles.find((p) => !/methylation/i.test(p)) || null;
const v = (title: string, view: Record) => ({ title, view: view as unknown as View });
const left = [!known.clinical.length && "Clinical (no attributes)", !(expr && names.length === 2) && "mRNA (" + (expr ? "compares exactly two cohorts" : "no mRNA profile") + ")",
@@ -55,7 +55,7 @@ export default function ComparisonPage() {
{!ready
? <>
>
: setCounts((o) => o || c)} />}
>);
}
diff --git a/app/portal/results/page.tsx b/app/portal/results/page.tsx
index 0a23214a..124288bb 100644
--- a/app/portal/results/page.tsx
+++ b/app/portal/results/page.tsx
@@ -6,7 +6,7 @@ import { useMemo, useState } from "react";
import { useSearchParams } from "next/navigation";
import type { CohortDef, View } from "../../../lib/types";
import { PortalTabs } from "../PortalTabs";
-import { useStudyKnown, expressionProfile, type StudyKnown } from "../study-known";
+import { useStudyKnown, expressionProfile, discreteCnaProfile, type StudyKnown } from "../study-known";
import { arrange, decodeDoc, usePageDoc, type PageLayout } from "../page-doc";
import { ALTERATION_CELLS, cellLabel } from "../../../lib/lollipop";
import { Button } from "../../_ui/button";
@@ -48,7 +48,7 @@ function resultsLayout(study: string, entries: string[], genes: string[], known:
Altered: { all: [{ any: entries.map((q) => ({ kind: "oql", q })) }] } as CohortDef,
Unaltered: { all: entries.map((q) => ({ not: { kind: "oql", q } })) } as CohortDef,
};
- const expr = expressionProfile(known.profiles);
+ const expr = expressionProfile(known.profileCatalog), cna = discreteCnaProfile(known.profileCatalog);
// the portal plots and correlates over every queried sample, not one group
if (expr) cohorts["All patients"] = { all: [] } as CohortDef;
const both = ["Altered", "Unaltered"];
@@ -58,7 +58,8 @@ function resultsLayout(study: string, entries: string[], genes: string[], known:
{ title: "Oncoprint", views: [v("Oncoprint", { kind: "oncoprint", genes, cohorts: both })] },
{ title: "Cancer Types Summary", views: [v("Cancer types summary", { kind: "cancertypes", genes, cohorts: ["Altered"] })] },
...(genes.length >= 2 ? [{ title: "Mutual Exclusivity", views: [v("Mutual exclusivity", { kind: "cooccurrence", genes, cohorts: both })] }] : []),
- ...(expr ? [{ title: "Plots", views: [v("Plot", { kind: "plot", x: { source: "alteration", gene: genes[0], by: "cna" }, y: { source: "molecular", profileType: expr, gene: genes[0] }, cohorts: ["All patients"], points: [{ cohort: "Altered" }] })] }] : []),
+ ...(expr ? [{ title: "Plots", views: [v("Plot", { kind: "plot", x: cna ? { source: "profile", profile: cna, gene: genes[0] } : { source: "alteration", gene: genes[0], by: "cna" },
+ y: { source: "profile", profile: expr, gene: genes[0] }, cohorts: ["All patients"], points: [{ cohort: "Altered" }] })] }] : []),
// the portal's Mutations tab: one gene at a time, its lollipop over its mutation table
{ title: "Mutations", views: [v("Mutation diagram", { kind: "hotspots", genes: [genes[0]], cohorts: ["Altered"] }),
v("Mutation table", { kind: "mutations", gene: genes[0], cohorts: ["Altered"] })],
@@ -119,7 +120,7 @@ export default function ResultsPage() {
{!ready
? <>
>
: setCounts((o) => o || c)} />}
>);
}
diff --git a/app/portal/study-known.ts b/app/portal/study-known.ts
index 43ddae1b..24bbe72d 100644
--- a/app/portal/study-known.ts
+++ b/app/portal/study-known.ts
@@ -3,9 +3,12 @@
// molecular profiles, generic-assay profiles and clinical attributes (most covered first).
import { useEffect, useState } from "react";
import { api } from "../lab/api.ts";
+import type { ProfileInfo } from "../../lib/profile-catalog.ts";
export interface StudyKnown {
profiles: string[]; gaProfiles: string[]; clinical: string[];
+ /** every profile as the study describes it, for a plot axis's Data picker */
+ profileCatalog: ProfileInfo[];
/** the clinical attributes with their kind and coverage, for the oncoprint's Tracks menu */
attrs: { attribute: string; kind: string; patients: number }[];
/** the study's display name and size, for the page header */
@@ -30,6 +33,7 @@ export function useStudyKnown(study: string): { known: StudyKnown | null; error:
const attrs = sorted.map((c) => ({ attribute: c.attribute, kind: c.kind || "cat", patients: c.patients || 0 }));
const meta = ((st.studies || []) as { id: string; name: string }[]).find((x) => x.id === study);
setKnown({ profiles: (j.profiles || []).filter((p: string) => p !== "cna"), gaProfiles: j.gaProfiles || [], clinical, attrs,
+ profileCatalog: j.profileCatalog || [],
name: meta?.name || study, patients: j.totals?.patients || 0, samples: j.totals?.samples || 0 });
})
.catch((e) => setError(String(e)));
@@ -37,8 +41,12 @@ export function useStudyKnown(study: string): { known: StudyKnown | null; error:
return { known, error };
}
-/** an mRNA profile for expression, preferring values over z-scores */
-export function expressionProfile(profiles: string[]): string | null {
- const m = profiles.filter((p) => /mrna|rna_seq/i.test(p));
- return m.find((p) => !/zscore/i.test(p)) || m[0] || null;
+/** an mRNA profile for expression, as the study describes its profiles: values before z-scores */
+export function expressionProfile(catalog: ProfileInfo[]): string | null {
+ const m = catalog.filter((p) => p.type === "MRNA_EXPRESSION").sort((a, b) => a.order - b.order);
+ return (m.find((p) => p.datatype === "CONTINUOUS") || m.find((p) => p.datatype === "Z-SCORE") || m[0])?.profile || null;
+}
+/** the study's discrete copy-number profile (GISTIC, or a panel study's calls) */
+export function discreteCnaProfile(catalog: ProfileInfo[]): string | null {
+ return catalog.find((p) => p.type === "COPY_NUMBER_ALTERATION" && p.datatype === "DISCRETE")?.profile || null;
}
diff --git a/docs/dsl.md b/docs/dsl.md
index 9f3efc2c..ca904690 100644
--- a/docs/dsl.md
+++ b/docs/dsl.md
@@ -275,7 +275,7 @@ patient — so the arm counts summing to more than the base cohort is expected,
| `logistic` | `outcome:axis, outcomeEvent?:"", covariates:[axis, ...(max 8)], cohorts:[names]` | multivariate logistic regression: a BINARY outcome (not time-to-event — no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as `cox`'s. `outcome` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs `outcomeEvent` naming which observed value counts as the event (1) — every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given `outcomeEvent`. Covariates use the exact same dummy-coding and reference-level rules as `cox` (see above): a mutated-vs-not axis defaults to `"Not mutated"`, others default to the most frequent level, `reference` overrides either. Same patient-level-only boundary as `cox` |
| `course` | `eventTypes:[≤6], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin — first recorded event to last contact (the later of their last event and their OS follow-up) — never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) `anchor` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED — read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (`Treatment`/`TREATMENT`) and the fold is named in the caveat; nothing else is merged |
| `sequence` | `labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES — therapies less than `lineGap` days apart are one line, labelled by their whole combination (sorted, so `A + B` and `B + A` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one `lineGap` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days — so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, `lineClasses` `"all"` for every treatment record or a list of class names for your own; `labelKey` only names a line’s therapies (e.g. AGENT), it never decides which records form lines. `exclude` drops records by label value, matched ignoring case. More than two cohorts is refused |
-| `plot` | `x:axis, y:axis, unit?:"sample"\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; `unit:"patient"` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 5,000 (`allPoints: false` turns that off), from a participant that may show cases; `points` lists up to four subsets drawn in colour over it — `{rule:"outliers"}` (the default), `{rule:"extremes",axis,n}`, `{rule:"trend",n}`, `{cohort}` — see **Points** below |
+| `plot` | `x:axis, y:axis, unit?:"sample"\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; `unit:"patient"` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 10,000 (`allPoints: false` turns that off), from a participant that may show cases; `points` lists up to four subsets drawn in colour over it — `{rule:"outliers"}` (the default), `{rule:"extremes",axis,n}`, `{rule:"trend",n}`, `{cohort}` — see **Points** below |
| `attrition` | `cohorts:[names] (≤6)` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its `all`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated |
| `genome` | `binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ `threshold`) or lost (≤ −`threshold`) in every `binMb`-megabase bin (1–50) along chromosomes 1–22 (and X with `includeX` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out |
| `pathways` | `pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"|"any", topGenes?:6, cohorts:[names] (≤4)` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own `sets`, each as the share of the cohort with at least one member altered. `measure` defaults to `"driver"` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or `"any"`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as `compare`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The `topGenes` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named |
@@ -334,13 +334,28 @@ column id or a filter that does not suit its column is refused with the ids acce
### Plot axes
-An axis is one of four things, and the pair decides the shape — the portal's own rule:
+An axis is a clinical attribute, one of the study's own molecular profiles, or a query across genes,
+and the pair decides the shape — the portal's own rule:
| `source` | shape of the axis | fields |
| --- | --- | --- |
| `clinical` | number or category, **as the study declared it** | `attribute` |
-| `molecular` | number | `profileType`, `gene` |
-| `alteration` | category | `gene` **or** `genes` (2-20), `by`: `"mutationType"` (default) \| `"mutated"` \| `"cna"` |
+| `profile` | **as the study describes the profile** (below) | `profile` (its id without the study prefix), `gene`, `by` for a mutation profile, `log` for numbers |
+| `alteration` | category | `oql` (below), or `genes` (2-20, any mutated) |
+
+**A profile axis** reads one gene in one of the study's profiles, and what it is follows from the
+study's own description of that profile (`genetic_profile`: its alteration type and datatype), never
+from its name: a **mutation** profile (MAF) groups cases by mutation type, or with `by: "mutated"`
+mutated or not; a **structural-variant** profile by whether the case has one; a **discrete
+copy-number** profile by its own calls (Deep deletion, Shallow deletion, Diploid, Gain,
+Amplification, drawn in that order); and a **continuous** one (mRNA, z-scores, log2 copy number,
+protein) is a number. A case with no mutation, SV or high-level call in the gene is the wild-type
+answer only when it is profiled for that gene; otherwise it is **Not profiled**. A profile a study
+does not have is refused with the list it does have, and one that two pooled studies describe
+differently is refused, never read as one thing. Generic-assay profiles (methylation, arm-level copy
+number, …) are not plot axes yet. The engine runs a profile axis as the lower-level forms it
+resolves to, which a document may also give directly: `{"source":"molecular","profileType","gene",
+"discrete"?}` and `{"source":"alteration","gene","by":"mutationType"|"mutated"|"cna"|"sv"}`.
number × number draws a **density scatter**, number × category a **box**, category × category a
**contingency table**. Every statistic is computed where the rows are and is therefore exact over
@@ -354,7 +369,7 @@ inventing a second mechanism. Only `by:"mutated"` (the default when `genes` is u
supported for a set — combining several genes' mutation TYPES or copy-number states into one
category has no single honest reading, so that combination is refused rather than guessed at.
-Two things are refused rather than approximated. A molecular axis **must name one gene** — a
+Two things are refused rather than approximated. A profile axis **must name one gene** — a
whole profile is 630 million values in this warehouse, and reading them to draw one picture is
never what was meant. And a categorical axis with more than 150 distinct values is refused,
because a truncated contingency table is a wrong one.
@@ -372,20 +387,20 @@ uses, not yet built for a box); its quartiles are a size-weighted approximation
same one Explore's federated numeric histograms already disclose.
**Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every
-case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the
+case as a point. So does this, up to 10,000 cases, on a scatter as on a box, and only when the
study is one this participant may see cases in — the points carry case ids, so they come from the
same entitlement that lets a patient-level participant list cohort members, and an aggregate-only
participant has no method that could return them. The jitter across a box is fixed per case, so
the figure does not move between the server SVG, the browser and the Python package.
-Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a
+Past 10,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a
hover on it would land on an arbitrary case — and, by default, the **outliers** are a point layer
over it (see **Points**): on a box, the
values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops
at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at
-most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of
+most 3 cases (fewer, if that is what keeps the points under 10,000) is drawn as its cases instead of
as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure
-draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With
+draws, and if a subset is itself past 10,000 none of it is drawn, never the first 10,000. With
`allPoints: false` and `points: []`, or from a participant that may not disclose a case, the
figure is density alone.
The caption always says which it drew, how many points, and why there are none when there are
@@ -407,16 +422,22 @@ bin's width is shared out between the classes in it. That stays an aggregate, so
may not disclose a case can still colour its plot. The key is capped at 12 classes.
**OQL on an axis.** An alteration axis can be an OQL query instead of genes:
-`{"source":"alteration","oql":"KRAS: G12C; STK11: MUT"}`. Each case is **Altered** when it matches
-any entry — the portal's own reading of a multi-gene query — and **Unaltered** otherwise; every
-case of the cohort is one or the other, so the unaltered are counted rather than missing. It is
-the same OQL a cohort's `oql` leaf takes (mutation classes and protein changes, AMP/HOMDEL,
-FUSION, DRIVER against the pinned OncoKB version, EXP/PROT thresholds), and it matches per patient
-or per sample following the plot's `unit`. Because it is a categorical axis, it also works as
-`colorBy`/`ringBy` and as a Cox or logistic covariate, where "Unaltered" is the reference level.
-OQL is for alteration axes only: a molecular axis stays one gene's value.
-
-**Points.** Every case is a point when there are at most 5,000 of them and every study may show
+`{"source":"alteration","oql":"KRAS: G12C, STK11: MUT"}`, its statements separated by a comma, a
+semicolon or a new line (as in a cohort's `oql` leaf). Each case is **Altered** when it matches any
+statement — the portal's own reading of a multi-gene query — and **Unaltered** when it matches none
+and is profiled for every statement's gene in the alteration types that statement asks about; a
+case whose gene panel omits one of them is **Not profiled**, never counted as wild type (the
+panel-aware rule of `docs/PERFORMANCE.md`). Every case of the cohort is one of the three, so none is
+missing. With `"by": "statement"` each statement is its own group, named as written: a case matching
+two is the pair ("KRAS: G12C + STK11: MUT"), three or more "3 or more lines", none Unaltered or Not
+profiled by the same rule (at most 6 statements). It is the same OQL a cohort's `oql` leaf takes
+(mutation classes and protein changes, AMP/HOMDEL, FUSION, DRIVER against the pinned OncoKB version,
+EXP/PROT thresholds), and it matches per patient or per sample following the plot's `unit`.
+Because it is a categorical axis, it also works as `colorBy`/`ringBy` and as a Cox or logistic
+covariate, where "Unaltered" is the reference level.
+OQL is for alteration axes only: a profile axis stays one gene in one profile.
+
+**Points.** Every case is a point when there are at most 10,000 of them and every study may show
cases (`allPoints: false` draws the density alone at any size). Over that the plotted cohort is a
density, and `points` lists the subsets worth seeing one by one, each drawn in its own colour with
the plotted cohort turned grey beneath them (at most four layers, in order):
@@ -428,7 +449,7 @@ grid cell holding at most 3 cases, left unshaded under its points); `{"rule": "e
`{"cohort": "KRAS G12C"}` — the plotted cohort's cases that are also in that cohort (how many of its
patients fall outside the plotted cohort is stated, and they are not drawn). The default is
`[{"rule": "outliers"}]`; `[]` draws none. A case in two layers is drawn by the first, and the
-caption counts what each layer left to an earlier one. Every layer shares one 5,000-point budget,
+caption counts what each layer left to an earlier one. Every layer shares one 10,000-point budget,
and a layer is never cut to its first cases: a cohort layer past the budget, or from a study that
may not show cases, is drawn as an aggregate on the plot's own scale (cells on the scatter's grid,
or a narrow box per group), and a rule layer is not drawn — the caption says which and why.
@@ -2189,7 +2210,7 @@ TCGA lung adenocarcinoma and squamous cell carcinoma pooled (1,053 patients). KR
### Example 20: OQL on a plot axis: STK11 or KEAP1 mutant lung cancer
-Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients.
+Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients. The third gives each statement of `STK11: MUT, KEAP1: MUT, KRAS: G12C` its own group, over samples: 620 KRAS G12C, 459 KEAP1, 377 STK11, 403 with both STK11 and KEAP1 (median TMB 8.6 against 4.1 for the 5,720 with none), the other pairs as their own groups and 109 matching all three. None of these lung samples is on a panel without the three genes, so no group is Not profiled.
```json
{
@@ -2244,12 +2265,78 @@ Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups
"source": "alteration",
"oql": "STK11: MUT; KEAP1: MUT"
}
+ },
+ {
+ "kind": "plot",
+ "unit": "sample",
+ "cohorts": [
+ "NSCLC"
+ ],
+ "x": {
+ "source": "alteration",
+ "oql": "STK11: MUT, KEAP1: MUT, KRAS: G12C",
+ "by": "statement"
+ },
+ "y": {
+ "source": "clinical",
+ "attribute": "TMB_NONSYNONYMOUS"
+ }
+ }
+ ]
+}
+```
+
+### Example 21: Plot axes from the study's own profiles: ERBB2 copy number against expression
+
+TCGA breast cancer (PanCancer Atlas), every sample. The first view groups ERBB2 mRNA z-scores (relative to all samples) by ERBB2's GISTIC copy-number call, from Shallow deletion to Amplification: the 122 amplified samples sit far above the rest. The second groups fraction of genome altered by TP53's mutation type from the study's mutation profile; the 18 samples not in that profile are Not profiled, not counted as unmutated.
+
+```json
+{
+ "study": "brca_tcga_pan_can_atlas_2018",
+ "cohorts": {
+ "all samples": {
+ "all": []
+ }
+ },
+ "views": [
+ {
+ "kind": "plot",
+ "cohorts": [
+ "all samples"
+ ],
+ "unit": "sample",
+ "x": {
+ "source": "profile",
+ "profile": "gistic",
+ "gene": "ERBB2"
+ },
+ "y": {
+ "source": "profile",
+ "profile": "rna_seq_v2_mrna_median_all_sample_Zscores",
+ "gene": "ERBB2"
+ }
+ },
+ {
+ "kind": "plot",
+ "cohorts": [
+ "all samples"
+ ],
+ "unit": "sample",
+ "x": {
+ "source": "profile",
+ "profile": "mutations",
+ "gene": "TP53"
+ },
+ "y": {
+ "source": "clinical",
+ "attribute": "FRACTION_GENOME_ALTERED"
+ }
}
]
}
```
-### Example 21: Merged gene rows: a pathway as one track
+### Example 22: Merged gene rows: a pathway as one track
Pancreatic adenocarcinoma. The first row pools TP53, MDM2 and MDM4: it is altered where any of them is, and its denominator is the union of the three genes' panels — not their intersection, and not the whole cohort. TP53 is drawn again on its own underneath, so the difference the other two make is readable rather than asserted. A row's members are listed in its tooltip and in the contract's `members` field.
@@ -2305,7 +2392,7 @@ Pancreatic adenocarcinoma. The first row pools TP53, MDM2 and MDM4: it is altere
}
```
-### Example 22: A federated z-score reference on an oncoprint
+### Example 23: A federated z-score reference on an oncoprint
A heatmap track with reference:"federation" is scored against a distribution pooled from EVERY participating study, including instances that disclose no patients and therefore contribute no columns — each returns three totals per gene (n, sum, sum of squares), which is enough to define the reference. Run with luad_tcga_pan_can_atlas_2018 aggregate-only, its patients are absent from the columns while its 510 patients still shape what the drawn columns mean.
@@ -2347,7 +2434,7 @@ A heatmap track with reference:"federation" is scored against a distribution poo
}
```
-### Example 23: The clinical course: who is still on treatment, and when
+### Example 24: The clinical course: who is still on treatment, and when
MSK-CHORD, aligned on each patient's FIRST treatment, split by TP53 status. Treatment carries real start/stop dates so it is drawn as a state patients are IN; surgery has no duration, so it is the share with a surgery in that bin — a distinction read from the data's shape, not from the event's name. Note surgery peaking BEFORE time zero: resection typically precedes first systemic therapy. The at-risk row under the axis is the denominator, and it falls steeply — a curve read without it would be badly misleading.
@@ -2401,7 +2488,7 @@ MSK-CHORD, aligned on each patient's FIRST treatment, split by TP53 status. Trea
}
```
-### Example 24: A reusable workup: one template, retargeted in a single edit
+### Example 25: A reusable workup: one template, retargeted in a single edit
The gene list appears once, in `params`, and every view refers to it as `$genes`; the study likewise. Change either and all three figures follow. The rule is only that a value which is exactly "$name" is replaced by that parameter — no interpolation, no expressions — so the resolved document is an ordinary one and compiles to exactly the SQL you would get by typing the values in. A `$name` with no declaration is a loud error naming the path, and a parameter declared but never referenced is reported back, which is what catches a rename that missed a reference. `notes` and each view’s `note` travel with the document.
@@ -2462,7 +2549,7 @@ The gene list appears once, in `params`, and every view refers to it as `$genes`
}
```
-### Example 25: What comes after what: lines of therapy, and what ending a path means
+### Example 26: What comes after what: lines of therapy, and what ending a path means
MSK-CHORD therapy lines by TP53 status, 90-day gap, two panels on one scale. Read the grey blocks first, and read them as TWO things: the darker terminal is patients the study watched for at least a line gap who began no further therapy; the lighter one is patients whose records simply stop, about whom nothing can be concluded. Over half of all paths end censored — collapsing them together (as a plain Sankey does) would turn record-keeping into an apparent clinical decision. The real signal is in first line: TP53 wild-type receives far more hormone therapy than TP53-mutant, which is what hormone-receptor-positive disease looks like. Lines are the one definition every line count shares: systemic-therapy records only, so 'Prior Medications to MSK' (a provenance marker rather than a therapy), radiation and bone treatment are not lines — they co-occur with almost everything, so leaving them in would explode the combination space. The caption says which records were used; a document that wants them anyway says so with lineClasses.
@@ -2509,7 +2596,7 @@ MSK-CHORD therapy lines by TP53 status, 90-day gap, two panels on one scale. Rea
}
```
-### Example 26: Lollipop plot: mutations by protein position, with a suppression floor
+### Example 27: Lollipop plot: mutations by protein position, with a suppression floor
Hotspots in colorectal cancer, drawn the way cBioPortal's mutation tab draws them: one stem per codon, its height the distinct patient + protein change pairs there, with the protein's Pfam domains on the backbone — so KRAS spikes at codons 12-13 and BRAF at V600. minCount is applied by each site before anything is returned — with it set to 3, a change carried by one or two patients never leaves the database holding it, which is where small-cell suppression belongs.
@@ -2548,7 +2635,7 @@ Hotspots in colorectal cancer, drawn the way cBioPortal's mutation tab draws the
}
```
-### Example 27: Co-expression from sums alone
+### Example 28: Co-expression from sums alone
Pearson correlation between gene pairs in TCGA breast cancer. A correlation depends on the data only through six running totals, so each site can contribute without transmitting a single expression value — and because the totals add, the federated answer is exact rather than approximate. ERBB2/GRB7 (adjacent on the 17q12 amplicon) comes out near 0.87.
@@ -2584,7 +2671,7 @@ Pearson correlation between gene pairs in TCGA breast cancer. A correlation depe
}
```
-### Example 28: Clinical attributes compared between cohorts
+### Example 29: Clinical attributes compared between cohorts
How TP53-mutant and wild-type tumors differ clinically. Categorical attributes are compared with a chi-squared over the full value vocabulary; numeric ones with Welch's t computed from three additive totals per site (n, sum, sum of squares) rather than from any transmitted value. An attribute the study does not declare is reported as skipped, not silently dropped.
@@ -2634,7 +2721,7 @@ How TP53-mutant and wild-type tumors differ clinically. Categorical attributes a
}
```
-### Example 29: Co-occurrence and mutual exclusivity in NSCLC
+### Example 30: Co-occurrence and mutual exclusivity in NSCLC
Pairwise associations among lung cancer driver genes. Each site returns four counts per pair — both altered, one, the other, neither — among the patients its own panels profiled for BOTH genes, and nothing else; the coordinator does the inference. Reproduces the textbook pattern: STK11/KEAP1 co-occur, EGFR/KRAS and EGFR/ALK are mutually exclusive.
@@ -2674,7 +2761,7 @@ Pairwise associations among lung cancer driver genes. Each site returns four cou
}
```
-### Example 30: Survival from first chemotherapy, left-truncated
+### Example 31: Survival from first chemotherapy, left-truncated
TP53-mutant vs wild-type, overall survival measured from each patient's first chemotherapy rather than from sequencing. Patients whose first chemotherapy falls after day 0, or who never had chemotherapy, are excluded rather than given negative follow-up. The result reports how long after the anchor patients became observable, so an anchor that is matching earlier events than intended shows up as a long tail.
@@ -2723,7 +2810,7 @@ TP53-mutant vs wild-type, overall survival measured from each patient's first ch
}
```
-### Example 31: Study-stratified survival across two studies
+### Example 32: Study-stratified survival across two studies
TP53-mutant vs wild-type, each cohort spanning two studies. Every site answers with (months, event, n) counts only — no survival time for any individual crosses an instance boundary — and because the log-rank statistic is a sum over strata, those counts are enough to compute the test STRATIFIED by study: each study is compared only against itself. The unstratified pooled p is reported alongside, since the two can disagree when case mix differs between sites.
@@ -2774,7 +2861,7 @@ TP53-mutant vs wild-type, each cohort spanning two studies. Every site answers w
}
```
-### Example 32: extends: a nested cohort without repeating its parent's filter
+### Example 33: extends: a nested cohort without repeating its parent's filter
APC-mutant colorectal cancer, then the same population restricted to liver metastasis — without re-stating the APC clause. `compare` shows both share the same APC-mutant base, so a difference between them is attributable to the liver-mets split, not to two independently-typed cohort definitions quietly drifting apart.
@@ -2820,7 +2907,7 @@ APC-mutant colorectal cancer, then the same population restricted to liver metas
}
```
-### Example 33: Splits: a cohort divided into arms that add up to it
+### Example 34: Splits: a cohort divided into arms that add up to it
Non-small cell lung cancer split by KRAS allele: G12C first, then any other KRAS mutation, then wild-type. Because G12C is tested first, "other KRAS" needs no hand-written "and not G12C". 916 + 1,278 + 5,615 = 7,809, the whole cohort, with nobody unprofiled for KRAS in this study. STK11 and KEAP1 are enriched in both KRAS-mutant arms and TP53 in wild-type — the known co-mutation pattern — while overall survival does not differ significantly across the three arms (k-group log-rank p≈0.1).
@@ -2884,7 +2971,7 @@ Non-small cell lung cancer split by KRAS allele: G12C first, then any other KRAS
}
```
-### Example 34: Median/IQR and a rank-sum test, single-study only
+### Example 35: Median/IQR and a rank-sum test, single-study only
APC-mutant colorectal cancer, split by liver metastasis. Alongside mean/SD, numeric attributes report an exact median/IQR and a Mann-Whitney U — available here only because both cohorts stay within one study (crc_apc_impact_2020): median and rank are not additive across sites the way (n, Σx, Σx²) is, so this is refused rather than approximated the moment a cohort spans more than one study.
@@ -2939,7 +3026,7 @@ APC-mutant colorectal cancer, split by liver metastasis. Alongside mean/SD, nume
}
```
-### Example 35: Cox with a gene-SET covariate: any of three genes, one term
+### Example 36: Cox with a gene-SET covariate: any of three genes, one term
Every patient in the study, one term: mutated in any of TP53/KRAS/PIK3CA versus none of them. Compiles to a single hugo_gene_symbol IN (...) — the same shape as a single-gene covariate, just with a broader gene list, so it costs nothing extra downstream (dummy-coding, the forest plot, the reference-level override all just work).
@@ -2973,7 +3060,7 @@ Every patient in the study, one term: mutated in any of TP53/KRAS/PIK3CA versus
}
```
-### Example 36: Cox stratified by cohort: does a gene's hazard ratio hold up in both arms?
+### Example 37: Cox stratified by cohort: does a gene's hazard ratio hold up in both arms?
APC mutation status regressed on overall survival, separately in the MSI-high and MSS arms of the same study. Pooled, APC reads HR=0.61 (95% CI 0.48-0.78) — a real effect, but stratifying shows it is almost entirely a microsatellite-stable-population effect (MSS: HR=0.54, p<0.0001) that does not reach significance in the small MSI-high arm (HR=0.87, 95% CI 0.31-2.44, n=27) — a pooled fit alone would not have shown that the effect is concentrated in one subgroup.
@@ -3024,7 +3111,7 @@ APC mutation status regressed on overall survival, separately in the MSI-high an
}
```
-### Example 37: Logistic regression: does APC status predict MSI-high status?
+### Example 38: Logistic regression: does APC status predict MSI-high status?
MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do not report) regressed on APC mutation status. APC-mutant patients have substantially lower odds of MSI-high status (OR=0.28, 95% CI 0.13-0.63, p=0.0018) — consistent with APC-driven CRC typically arising through the chromosomal-instability pathway rather than the mismatch-repair-deficient pathway MSI-high status marks.
@@ -3059,7 +3146,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no
}
```
-### Example 38: splitBy: one cohort, auto-split into an arm per receptor subtype
+### Example 39: splitBy: one cohort, auto-split into an arm per receptor subtype
"all patients" split by RECEPTOR_STATUS_PRIMARY gives the same 4 clinically standard subtypes a hand-written 4-cohort document would (HR+/HER2-, Triple Negative, HR+/HER2+, HR-/HER2+), plus a few small ones this study also records (HR+/HER2_Unknown, Unk/ND, HR+/HER2_Equivocal) — all within the default 8-arm cap, so nothing here needs pooling into "Other". TP53 reads 26.8%/90.4%/49.2%/77% across the four main subtypes, matching the published literature on triple-negative breast cancer's markedly higher TP53 mutation rate.
@@ -3087,7 +3174,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no
}
```
-### Example 39: splitBy: gene mutation status, without hand-writing the mutant/wildtype pair
+### Example 40: splitBy: gene mutation status, without hand-writing the mutant/wildtype pair
"all patients" split by APC mutation status gives the same two arms a hand-written {"kind":"oql","q":"APC: MUT"} cohort pair would (325 mutant, 146 wildtype) — APC-mutant patients have better overall survival (median 25.6mo vs 17.9mo, log-rank p=9.1e-5), matching this study's own headline finding (see the crc_apc_impact_2020 Cox example above) from a single view, no separate cohorts to define.
@@ -3111,7 +3198,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no
}
```
-### Example 40: splitBy: a list of OQL queries, arms that overlap and a disclosed "Other"
+### Example 41: splitBy: a list of OQL queries, arms that overlap and a disclosed "Other"
"all patients" split by ["TP53: MUT", "KRAS: MUT"] gives 13,105 TP53-mutant, 7,115 KRAS-mutant, and 9,019 in neither ("Other") — summing to more than the 24,950-patient base cohort because 4,289 patients carry both mutations, which the result discloses as `splitBy.overlapCount` rather than leaving the arm counts to look inconsistent on their own (13,105 + 7,115 − 4,289 + 9,019 = 24,950, exactly the base count).
@@ -3142,7 +3229,7 @@ MSI_TYPE = Instable (vs every other observed value: Stable, Indeterminate, Do no
}
```
-### Example 41: Survival split three ways: the k-group log-rank
+### Example 42: Survival split three ways: the k-group log-rank
MSI status in three groups (stable / instable / indeterminate) instead of the usual two-arm split. The test asks whether the three curves differ AT ALL, not which pair does — a different question from running the 2-arm test three times pairwise, which would need its own multiple-testing correction.
@@ -3201,7 +3288,7 @@ MSI status in three groups (stable / instable / indeterminate) instead of the us
}
```
-### Example 42: A missing alteration type is disclosed, never a false negative
+### Example 43: A missing alteration type is disclosed, never a false negative
TP53 (a real mutation hotspot here) and MDM2 (never assayed for copy-number in this study) side by side. Both show 0%-or-real altered figures, but only the view's note — and each cell's per-type breakdown (mutation/CNA/structural-variant counts, visible on hover in the app) — reveals that MDM2's 0% means "never checked" while TP53's is a real, profiled result.
@@ -3237,7 +3324,7 @@ TP53 (a real mutation hotspot here) and MDM2 (never assayed for copy-number in t
}
```
-### Example 43: Four receptor subtypes, one real test: Welch's ANOVA
+### Example 44: Four receptor subtypes, one real test: Welch's ANOVA
Breast cancer's four receptor subtypes, compared on genomic instability (FRACTION_GENOME_ALTERED) and mutation burden (TMB_NONSYNONYMOUS). Triple-negative's markedly higher genomic instability is exactly the known biology, and both the parametric (ANOVA) and rank-based (Kruskal-Wallis) tests agree emphatically; TMB does not differ significantly by subtype here, and again both tests agree.
@@ -3312,7 +3399,7 @@ Breast cancer's four receptor subtypes, compared on genomic instability (FRACTIO
}
```
-### Example 44: A non-standard survival endpoint, named directly
+### Example 45: A non-standard survival endpoint, named directly
MSI-H vs MSS endometrial cancer, progression-free survival from first-line treatment — the study's actual recorded outcome, reachable only because the exact column names are given rather than assumed from an OS/PFS/DFS-style prefix.
@@ -3360,7 +3447,7 @@ MSI-H vs MSS endometrial cancer, progression-free survival from first-line treat
}
```
-### Example 45: Top 20% TMB, within one cancer type: a percentile cutoff
+### Example 46: Top 20% TMB, within one cancer type: a percentile cutoff
tmb_mskcc_2018 is pan-cancer, so a plain percentile would mix histologies with very different baseline TMB. "NSCLC high-TMB"/"NSCLC low-TMB" both restrict to NSCLC first, then split on whether TMB is at or above the 80th percentile COMPUTED WITHIN THAT NSCLC POPULATION (percentileOf:"NSCLC") — the readback shows the resolved number and what it was a percentile of, not just a bare threshold.
@@ -3433,7 +3520,7 @@ tmb_mskcc_2018 is pan-cancer, so a plain percentile would mix histologies with v
}
```
-### Example 46: Cohort attrition: which criterion removed whom
+### Example 47: Cohort attrition: which criterion removed whom
KRAS G12C lung adenocarcinoma treated with immunotherapy, with at least six months of follow-up, beside KRAS wild-type lung adenocarcinoma. The G12C arm loses most of its patients to the genotype, and a further 78 to the follow-up requirement — the kind of step a reader needs to see before trusting a survival curve drawn from what is left.
@@ -3510,7 +3597,7 @@ KRAS G12C lung adenocarcinoma treated with immunotherapy, with at least six mont
}
```
-### Example 47: Genome-wide copy number: colorectal against breast
+### Example 48: Genome-wide copy number: colorectal against breast
Colorectal and breast cancers in MSK-IMPACT, 2 Mb bins. The textbook colorectal pattern is all there — 20q gained in 60%, 13q in 42%, 8q in 32%, 18q lost in 54% — beside breast cancer's 1q and 8q gains and 16q loss. The strip underneath marks every bin where the two differ at q < 0.05, coloured by which cohort is higher.
@@ -3556,7 +3643,7 @@ Colorectal and breast cancers in MSK-IMPACT, 2 Mb bins. The textbook colorectal
}
```
-### Example 48: Oncogenic signalling pathways: colorectal against pancreatic
+### Example 49: Oncogenic signalling pathways: colorectal against pancreatic
Colorectal and pancreatic cancer in MSK-CHORD. WNT separates them almost completely (82% against 8%, nearly all APC); RTK-RAS is higher in pancreatic cancer (84%, KRAS 81%), and so is the cell cycle (39%, CDKN2A 33%); PI3K and NOTCH (FBXW7) are colorectal. The canonical pathways also show how much a targeted panel misses: 56 of WNT's 68 genes are on no panel here.
@@ -3601,7 +3688,7 @@ Colorectal and pancreatic cancer in MSK-CHORD. WNT separates them almost complet
}
```
-### Example 49: Differential expression and enrichment: basal-like against luminal A breast cancer
+### Example 50: Differential expression and enrichment: basal-like against luminal A breast cancer
TCGA PanCancer Atlas breast cancer, PAM50 basal-like against luminal A, on mRNA z-scores. The luminal genes (ESR1, PGR, FOXA1, GATA3) and the basal ones (FOXC1, and proliferation — MKI67, CDC20, AURKB) come out as expected, and so does the enrichment: E2F targets, the G2M checkpoint and MYC targets in basal-like tumours, early estrogen response in luminal A.
@@ -3645,7 +3732,7 @@ TCGA PanCancer Atlas breast cancer, PAM50 basal-like against luminal A, on mRNA
}
```
-### Example 50: Paired samples: what differs between a patient's primary and metastasis
+### Example 51: Paired samples: what differs between a patient's primary and metastasis
MSK-IMPACT, the 2,278 patients sequenced at both a primary and a metastatic site. Truncal drivers agree (TP53 78% concordant); what the metastasis alone carries is the resistance biology — ESR1 mutations (49 metastasis-only against 7 primary-only) and AR alterations (71 against 1), both selected under hormone therapy — and MYC amplification and CDKN2A loss.
@@ -3678,7 +3765,7 @@ MSK-IMPACT, the 2,278 patients sequenced at both a primary and a metastatic site
}
```
-### Example 51: Paired samples, every gene: what the metastasis carries that the primary does not
+### Example 52: Paired samples, every gene: what the metastasis carries that the primary does not
MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list. Of 347 genes with a discordant patient, the scan puts AR (71 metastasis-only against 1 primary-only) and ESR1 (49 against 7) first — the hormone-therapy resistance genes the hand-picked version above had to be told to look at — followed by MYC, NKX3-1 and PTEN.
@@ -3702,7 +3789,7 @@ MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list
}
```
-### Example 52: Before and after treatment, pooled across studies that label it differently
+### Example 53: Before and after treatment, pooled across studies that label it differently
111 patients with a sample on each side across gastroesophageal (two studies), breast (alpelisib, and a second cohort) and bladder cancer. At this size nothing survives correction across the 278 genes tested — the honest answer — and the most one-sided genes run the other way: CCND1 and MYC amplifications seen before treatment and not after (10 and 10 patients against 1 and 2), as a lower-purity post-treatment sample would also show. Read it as a lead, not a finding; the same view over a large serially sampled cohort is where acquired resistance would show.
@@ -3766,7 +3853,7 @@ MSK-IMPACT again, the 2,278 patients with both kinds of sample, but no gene list
}
```
-### Example 53: Clonality, LOH and genome doubling from FACETS
+### Example 54: Clonality, LOH and genome doubling from FACETS
Colorectal against breast cancer. Tumour suppressors show the classic second hit — TP53 is ~88% clonal with its other allele lost 77–89% of the time, CDH1 in breast 95% — while the oncogene KRAS rarely loses its partner. PIK3CA is markedly more often subclonal in colorectal cancer (68% clonal) than in breast (84%), in keeping with a later event there. Around a third of both cohorts' tumours have doubled their genome.
@@ -3820,7 +3907,7 @@ Colorectal against breast cancer. Tumour suppressors show the classic second hit
}
```
-### Example 54: Brain metastasis, with death as a competing risk
+### Example 55: Brain metastasis, with death as a competing risk
MSK-CHORD, from each patient's day 0, among those without a brain metastasis by then. At five years 24.7% of NSCLC patients have one — where 1 − Kaplan–Meier would have said 30.4% — against 20.6% in breast and 10.8% in colorectal cancer (1 − KM: 14.4%, a third too high, because so many colorectal patients die first).
@@ -3888,7 +3975,7 @@ MSK-CHORD, from each patient's day 0, among those without a brain metastasis by
}
```
-### Example 55: PSA response to hormone therapy, by TP53 status
+### Example 56: PSA response to hormone therapy, by TP53 status
Prostate cancer in MSK-CHORD, from the first hormone therapy. PSA falls by a median of more than 95% within three months in both groups, but 93.0% of TP53 wild-type patients reach PSA50 against 85.8% of TP53-mutant ones (Fisher p = 1.4e-6), and the mutant group's upper quartile turns back upward within a year.
@@ -3949,7 +4036,7 @@ Prostate cancer in MSK-CHORD, from the first hormone therapy. PSA falls by a med
}
```
-### Example 56: A survival risk model from mutated genes in lung cancer
+### Example 57: A survival risk model from mutated genes in lung cancer
Non-small cell lung cancer in MSK-CHORD. The model keeps KEAP1, STK11, SMARCA4 and TP53 as adverse and EGFR as favourable, as the literature does, and its risk thirds separate clearly; a held-out C-index near 0.61 says mutations alone predict survival only modestly. Five runs keep the example quick; OncoCast recommends fifty. Its Table 1 (`describe`) shows who is in each third: the high-risk third is more often male, a former or current smoker and at stage 4 than the low-risk third. KEAP1 and TP53 are marked “in the model” — the score is built from them, so their enrichment is by construction and gets no q-value — and ZFHX3, which not every panel covers, is counted over each group's patients whose panel does.
@@ -3995,7 +4082,7 @@ Non-small cell lung cancer in MSK-CHORD. The model keeps KEAP1, STK11, SMARCA4 a
}
```
-### Example 57: Custom data: a curated biopsy site on every sample
+### Example 58: Custom data: a curated biopsy site on every sample
luad_mskimpact_2021 is 93 EGFR-mutant lung adenocarcinomas with 186 primary and metastatic samples. The demo upload gives every sample a BIOPSY_SITE, grouped from the study's own SAMPLE_TYPE and free-text METASTATIC_SITE the way a reviewer would (scripts/gen-custom-demo.ts), standing in for a clinician-curated annotation. "CNS met" filters on it: 8 patients. Ordered by site, the oncoprint puts each patient's primary and metastases in their own groups. Fraction of the genome altered has a median of 0.47 in the 11 CNS samples and 0.36 in liver, against 0.21 in the 76 primaries. Split by site, TP53 is altered in 7 of 8 CNS-met patients (87.5%) against 65.6% of the patients with a primary sample. The split arms overlap, because a patient whose primary and metastasis were both sequenced belongs to both arms. With 8 to 61 patients per arm, these are illustrations, not findings.
@@ -4078,7 +4165,7 @@ Runs on custom data: `luad-biopsy-site` (LUAD biopsy site (demo), 186 samples, c
}
```
-### Example 58: Custom data: single-cell features of ovarian tumours (MSK SPECTRUM)
+### Example 59: Custom data: single-cell features of ovarian tumours (MSK SPECTRUM)
msk_spectrum_tme_2022 is 42 untreated high-grade serous ovarian cancers; 41 had scRNA-seq at several anatomical sites. scripts/gen-spectrum-sc-demo.py reads the public CELLxGENE cell annotations (cell states, per-cell pathway scores, per-cell 6p allele fraction) and summarises them per patient (solid sites pooled; ADX_ adnexal, NADX_ non-adnexal) and per cBioPortal sample (the scRNA-seq site of the same patient, 61 of 82 samples). Dysfunctional T/NK cells (TNK_DYSFUNC) have a median of 10.8% of T/NK cells in the 16 HRD-Dup tumours against 6.0% in the 14 FBI tumours, and naive/central-memory cells the reverse (7.5% against 11.1%), as in the paper's Fig. 4b. In adnexal cancer cells, JAK-STAT pathway activity has a median of 0.03 in HRD-Dup against -0.57 in FBI (Fig. 3e), and cancer-cell and T/NK-cell JAK-STAT correlate across patients (Spearman 0.74, Fig. 4e). Clonal 6p loss of heterozygosity (over 80% of cancer cells) is called in 4 patients, 3 of them HRD-Dup — the paper reports 4 of 41. By site, T cells are a median 43% of immune cells in the 22 adnexal samples against 54% in the 18 omental ones. These are summaries of public cell metadata, not the paper's statistical models.
@@ -4195,7 +4282,7 @@ Runs on custom data: `spectrum-sc-composition` (SPECTRUM scRNA: cell-type compos
}
```
-### Example 59: Composition: T and NK cell states by mutational signature (MSK SPECTRUM)
+### Example 60: Composition: T and NK cell states by mutational signature (MSK SPECTRUM)
The 36 SPECTRUM patients with a consensus signature other than Undetermined, split as the paper splits them. Dysfunctional CXCL13+ CD8 T cells are a mean 7.0% of T/NK cells in the 16 HRD-Dup patients against 4.9% in HRD-Del and 4.4% in FBI, and naive CD4 cells the reverse (15.2% against 19.6% and 20.1%) — the direction of the paper's Fig. 4b. The clearest single difference is cycling T/NK cells, higher in HRD-Dup (8.6% against 5.7%; Welch p = 0.003, q = 0.08 across the 30 comparisons); none reaches q < 0.05 at this size, which the matrix shows as no outlined dot. Per sample, from the study's cell-type fractions, T cells are 24% of HRD-Dup samples' cells against 32% of FBI's. The paper fits a binomial model on cell counts with site and sort as covariates; this view compares shares between groups and says so.
@@ -4275,7 +4362,7 @@ Runs on custom data: `spectrum-sc-composition` (SPECTRUM scRNA: cell-type compos
}
```
-### Example 60: Composition: adenomas and serrated polyps, from a generic single-cell summary (HTAN Vanderbilt)
+### Example 61: Composition: adenomas and serrated polyps, from a generic single-cell summary (HTAN Vanderbilt)
crc_hta11_htan_2021 is 52 patients with colorectal polyps (Chen et al., Cell 2021). The summary covers 55 of its 61 samples; its counts equal the study's own absolute_counts profile for all 670 sample and cell-type pairs it reports. Per sample, adenoma-specific cells (ASC) are 32.1% of the epithelium in the 30 conventional adenomas against 4.1% in the 20 serrated polyps (q = 2e-5), and serrated-specific cells (SSC) 1.5% against 19.7% (q = 0.003): the two paths to malignancy the paper describes. Stem cells (STM) differ little here (3.1% against 2.5%). The study's own fractions, over every cell type, give the same picture (ASC 29.9% against 3.7%). Cohorts are patients, so a patient with polyps of both types counts in both; 5 of the 52 have both.
@@ -4344,7 +4431,7 @@ Runs on custom data: `hta11-sc-p` (HTAN Vanderbilt colorectal polyp scRNA-seq (C
}
```
-### Example 61: Cancer types summary: RAS genes across 50,000 tumours
+### Example 62: Cancer types summary: RAS genes across 50,000 tumours
KRAS, NRAS and HRAS across MSK-IMPACT 50K. Pancreatic cancer leads at 82%, appendiceal and colorectal follow, and the bars show where RAS is amplified rather than mutated — germ cell tumours and esophagogastric cancer.
@@ -4373,7 +4460,7 @@ KRAS, NRAS and HRAS across MSK-IMPACT 50K. Pancreatic cancer leads at 82%, appen
}
```
-### Example 62: Cohort overlap: how four groups share their patients
+### Example 63: Cohort overlap: how four groups share their patients
TP53-mutant, KRAS-mutant, immunotherapy-treated and NSCLC patients in MSK-CHORD. 5,314 TP53-mutant patients belong to none of the other three groups; 435 are in all four.
@@ -4443,7 +4530,7 @@ TP53-mutant, KRAS-mutant, immunotherapy-treated and NSCLC patients in MSK-CHORD.
}
```
-### Example 63: Alteration enrichment: what separates colorectal from pancreatic tumours
+### Example 64: Alteration enrichment: what separates colorectal from pancreatic tumours
Microsatellite-stable colorectal against pancreatic cancer in MSK-CHORD. APC, TCF7L2, PIK3CA, FBXW7 and SOX9 mark colorectal cancer; KRAS and CDKN2A mark pancreatic cancer.
@@ -4508,7 +4595,7 @@ Microsatellite-stable colorectal against pancreatic cancer in MSK-CHORD. APC, TC
}
```
-### Example 64: Mutational signatures: what UV light and tobacco leave behind
+### Example 65: Mutational signatures: what UV light and tobacco leave behind
Melanoma against non-small cell lung cancer in MSK-IMPACT 50K. The UV signature (7) makes up about half of a melanoma's mutations and almost none of a lung cancer's; the smoking signature (4) is the reverse.
@@ -4554,7 +4641,7 @@ Melanoma against non-small cell lung cancer in MSK-IMPACT 50K. The UV signature
}
```
-### Example 65: Co-expression: the genes that rise and fall with ESR1
+### Example 66: Co-expression: the genes that rise and fall with ESR1
TCGA PanCancer Atlas breast cancer. The oestrogen-receptor programme lines up behind ESR1 — CCDC170 (its neighbour on 6q25, often co-amplified), THSD4, FOXA1, TBC1D9 — and the genes of basal-like tumours fall away from it.
@@ -4579,7 +4666,7 @@ TCGA PanCancer Atlas breast cancer. The oestrogen-receptor programme lines up be
}
```
-### Example 66: Clinical data: every pancreatic cancer sample, oldest patients first
+### Example 67: Clinical data: every pancreatic cancer sample, oldest patients first
Pancreatic cancer in MSK-CHORD, sorted by age, highest first.
@@ -4616,7 +4703,7 @@ Pancreatic cancer in MSK-CHORD, sorted by age, highest first.
}
```
-### Example 67: A patient's timeline: treatments, diagnoses, samples and lab values from diagnosis
+### Example 68: A patient's timeline: treatments, diagnoses, samples and lab values from diagnosis
MSK-CHORD breast cancer patients with the most timeline events first; P-0000012 drawn, with chemotherapy folded into one row, and P-0000012's alterations beside it.
@@ -4669,7 +4756,7 @@ MSK-CHORD breast cancer patients with the most timeline events first; P-0000012
}
```
-### Example 68: A patient's mutations, copy-number alterations and structural variants
+### Example 69: A patient's mutations, copy-number alterations and structural variants
MSK-CHORD prostate cancer patients, those with the most structural variants first; P-0024249's tables, mutations kept to OncoKB-oncogenic calls.
@@ -4717,7 +4804,7 @@ MSK-CHORD prostate cancer patients, those with the most structural variants firs
}
```
-### Example 69: KRAS codon 12/13 mutations in colorectal cancer, clonal-looking calls first
+### Example 70: KRAS codon 12/13 mutations in colorectal cancer, clonal-looking calls first
MSK-CHORD patients with a colorectal cancer sample: their KRAS G12/G13 calls at an allele frequency of 0.2 or more, highest first.
diff --git a/docs/plot-case-level.md b/docs/plot-case-level.md
index bc2f1067..e33db78d 100644
--- a/docs/plot-case-level.md
+++ b/docs/plot-case-level.md
@@ -35,7 +35,7 @@ a hover lands on an arbitrary one. So:
combines the patient's samples by a stated rule: a number is their mean, a category their common
value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption
says how many patients had more than one sample.
-- **At or under `POINTS_CAP` (5,000), every case is a point** (`allPoints`, default true), on a
+- **At or under `POINTS_CAP` (10,000), every case is a point** (`allPoints`, default true), on a
scatter as on a box — the portal's picture, and what most single studies get.
- **Over the cap, the crowd is density, and point layers draw chosen subsets in colour over it.**
`points` is an ordered list of at most four layers (`lib/plot-layers.ts`):
@@ -75,23 +75,43 @@ One commit per item; `npm test` and a focused check per item; the gate before ea
the figure beside it (`PlotControls`, shared by the lab card and the Results page).
3. **Done.** A point is a sample by default; patient mode combines samples by a stated rule.
4. **Done.** Point layers (above), replacing `highlight`, with the sidebar's Points section.
-5. **Measure the cap.** SVG draw time and payload at 5k, 10k, 20k and 50k points on msk_chord;
- set `POINTS_CAP` from it.
-6. **Multiple genes.** An alteration axis whose OQL has several statements gets one category per
- statement (overlaps as their own category, never assigned arbitrarily), and commas separate
- statements as well as `;` and newlines.
-7. **Colour by gene per case** (phase 2). `colorBy: { source: "gene", gene }`: fill from the case's
+5. **Done.** Show one layer alone: each layer is a group in the SVG tagged with its place in the
+ list (its cases an earlier layer drew are a hidden group of their own), and a row of buttons
+ above the figure steps the rest back with one CSS rule; Esc restores. Browser state only.
+6. **Done.** The cap, measured. The plot renderer drew a scatter and a box at n points
+ (synthetic values, real-shaped ids), and headless Chrome on the dev server (6 cores, no GPU)
+ inserted and painted each SVG, median of 3:
+
+ | points | paint, scatter / box | one hover hit-test | JSON (raw / gzip) |
+ |---|---|---|---|
+ | 5,000 | 84 / 117 ms | 0.4 ms | 220 / 35 KB |
+ | 10,000 | 180 / 190 ms | 1.7 ms | 440 / 70 KB |
+ | 20,000 | 352 / 337 ms | 3.2 ms | 880 / 140 KB |
+ | 50,000 | 904 / 752 ms | 7 ms | 2.2 MB / 350 KB |
+
+ Building the SVG string is 25–190 ms and not the limit; painting is, at about 18 ms per
+ thousand points, paid again on every redraw (a resize, a re-run). `POINTS_CAP` is 10,000, which
+ keeps a redraw near 200 ms on this machine (a laptop with a GPU is faster). Hover will go
+ through one overlay (item 9), so per-circle hit-testing does not set the cap.
+7. **Done.** Multiple genes. An OQL axis's statements are separated by ",", ";" or a new line
+ (everywhere OQL is read). `by: "statement"` gives each statement its own group, a case matching
+ two the pair, three or more "3 or more lines". Either way a case matching nothing is
+ "Unaltered" only when it is profiled for every statement's gene (panel-aware), otherwise "Not
+ profiled". On msk_chord, KRAS G12C and RRAGC (on 20,048 of 25,040 samples' panels): 1,111 /
+ 61 / 3 both / 19,115 Unaltered / 4,750 Not profiled, equal to a direct query; the 4,750 were
+ counted as Unaltered before.
+8. **Colour by gene per case** (phase 2). `colorBy: { source: "gene", gene }`: fill from the case's
highest-priority mutation (oncoprint priority, driver/VUS by the pinned OncoKB), ring from
CNA/SV, "not profiled" its own class (panel coverage, reuse the oncoprint's). On scatters too;
the rare classes become a point layer over the cap. Check per-class counts against the portal's
Plots tab for one study and gene.
-8. **Interaction** (phase 3): one overlay over the SVG maps the cursor to the nearest point (a grid
+9. **Interaction** (phase 3): one overlay over the SVG maps the cursor to the nearest point (a grid
index over the drawn points); hover tooltip (sample id linking to the patient view, values,
- alterations), legend-click highlight, case-id / protein-change search whose matches come back
+ alterations), case-id / protein-change search whose matches come back
from the server as a layer.
-9. **Brush and drill-down** (phase 4): a region or a cell returns its cases as a layer; "N cases →
+10. **Brush and drill-down** (phase 4): a region or a cell returns its cases as a layer; "N cases →
study view" built from the full list on the server.
-10. **Sidebar parity** (phase 5): "same gene", Driver vs VUS and VAF axes, box category filter and
+11. **Sidebar parity** (phase 5): "same gene", Driver vs VUS and VAF axes, box category filter and
sort-by-median, regression line toggle, R² and the equation, connect samples from the same
patient.
diff --git a/e2e.cjs b/e2e.cjs
index f178f942..e7a22316 100644
--- a/e2e.cjs
+++ b/e2e.cjs
@@ -568,7 +568,7 @@ async function main() {
try { const v = JSON.parse(document.querySelector("#docjson").value).views[0];
return v.x.attribute === "FRACTION_GENOME_ALTERED" && v.y.attribute === "MUTATION_COUNT"; } catch { return false; }
}, null, { timeout: 30000 });
- sideOk = blocks.join(",") === "Horizontal axis,Vertical axis" && names.join(",") === "Data type,Clinical attribute" && !!beside && heights.length === 1;
+ sideOk = blocks.join(",") === "Horizontal axis,Vertical axis" && names.join(",") === "Data,Clinical attribute" && !!beside && heights.length === 1;
sideWhat = "blocks " + blocks.join(",") + "; fields " + names.join(",") + "; figure beside: " + beside + "; select heights " + heights.join(",") + "; swap rewrote x/y";
} catch (e) { sideWhat = String(e.message).split("\n")[0]; }
ok("a plot's axes are defined in a sidebar beside the figure, and Swap axes swaps them", sideOk, sideWhat);
@@ -592,6 +592,57 @@ async function main() {
} catch (e) { layerWhat = String(e.message).split("\n")[0]; }
ok("a plot's point layers are listed in the sidebar, and adding one draws it with a key entry", layerOk, layerWhat);
+ // Hiding the plotted cases or a layer is instant and the browser's own: its group steps back,
+ // a case it drew reappears in the next layer holding it, Esc shows everything.
+ let soloWhat = "no layer buttons", soloOk = false;
+ try {
+ const groups = () => page.evaluate(() => [...document.querySelectorAll(".plot-figure g[data-layer]")]
+ .map((g) => [g.getAttribute("data-layer") + (g.hasAttribute("data-shared") ? "<" + g.getAttribute("data-owner") : ""), getComputedStyle(g).opacity, getComputedStyle(g).display]));
+ const docBefore = await page.locator("#docjson").inputValue();
+ const chips = page.locator(".plot-solo button");
+ const names = await chips.allInnerTexts();
+ await chips.nth(1).click(); // the first layer
+ const hid = await groups();
+ await chips.first().click(); // the plotted cases too
+ const both = await groups();
+ await page.keyboard.press("Escape");
+ const back = await groups();
+ const of = (gs, k) => gs.filter(([key]) => key === k);
+ soloOk = /^all samples/.test(names[0]) && names.length >= 2 &&
+ of(hid, "0").every(([, o]) => o !== "1") && of(hid, "base").every(([, o]) => o === "1") &&
+ hid.filter(([k]) => /<0$/.test(k)).every(([, o, d]) => o === "1" && d !== "none") &&
+ of(both, "base").every(([, o]) => o !== "1") &&
+ back.filter(([k]) => !k.includes("<")).every(([, o]) => o === "1") &&
+ (await page.locator("#docjson").inputValue()) === docBefore;
+ soloWhat = "chips " + JSON.stringify(names) + "; first layer hidden " + JSON.stringify(hid) + "; after Esc " + JSON.stringify(back) + "; document unchanged";
+ } catch (e) { soloWhat = String(e.message).split("\n")[0]; }
+ ok("the plotted cases and each layer can be hidden from their chips, shared cases follow, Esc shows everything, and the document is untouched", soloOk, soloWhat);
+
+ // An OQL axis is edited through the app's gene query dialog: it opens with one line per
+ // statement, and what it applies is the axis's query.
+ let oqlWhat = "no OQL axis", oqlOk = false;
+ try {
+ await page.goto(BASE + "/lab?tab=analysis&doc=" + encodeURIComponent(Buffer.from(JSON.stringify({
+ study: "msk_chord_2024", cohorts: { All: { all: [] } },
+ views: [{ kind: "plot", cohorts: ["All"], x: { source: "alteration", oql: "KRAS: G12C, STK11: MUT", by: "statement" },
+ y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" } }]
+ })).toString("base64")));
+ await page.waitForSelector(".plot-side", { timeout: 300000 });
+ // genes and OQL are chosen in the dialog only: the axis holds no free-text field
+ const freeText = await page.locator(".plot-side .plot-axis input").count();
+ await page.getByRole("button", { name: "OQL for this axis" }).click();
+ const dlg = page.locator("[role=dialog]");
+ await dlg.waitFor({ timeout: 30000 });
+ const opened = await dlg.locator("textarea").inputValue();
+ await dlg.locator("textarea").fill("KRAS: G12C\nSTK11: MUT\nKEAP1: MUT");
+ await dlg.getByRole("button", { name: /^Apply/ }).click({ timeout: 30000 });
+ await page.waitForFunction(() => /KEAP1/.test(JSON.parse(document.querySelector("#docjson").value).views[0].x.oql), null, { timeout: 30000 });
+ const x = await page.evaluate(() => JSON.parse(document.querySelector("#docjson").value).views[0].x);
+ oqlOk = freeText === 0 && opened === "KRAS: G12C\nSTK11: MUT" && x.oql === "KRAS: G12C, STK11: MUT, KEAP1: MUT" && x.by === "statement";
+ oqlWhat = "free-text fields " + freeText + "; opened " + JSON.stringify(opened) + " -> axis " + JSON.stringify(x);
+ } catch (e) { oqlWhat = String(e.message).split("\n")[0]; }
+ ok("a plot's OQL axis is edited in the gene query dialog, and its lines become the axis's statements", oqlOk, oqlWhat);
+
// The oncoprint's gear keeps ONLY the link out to cBioPortal; genes moved
// to a named button and a modal, so the gear is not where editing hides.
let gearSplit = "no oncoprint rendered", gearOk = false;
diff --git a/lib/analysis.ts b/lib/analysis.ts
index 2f70140b..98f28a6a 100644
--- a/lib/analysis.ts
+++ b/lib/analysis.ts
@@ -9,6 +9,7 @@
import { applyLocks, lockDate, lockSamples } from "./cohort-lock.ts";
import { chQuery, STUDY, PRIMARY, type ChConn } from "./ch.ts";
import { sq } from "./compiler.ts";
+import type { ProfileInfo } from "./profile-catalog.ts";
import { describeCohortDef } from "./cohort2.ts";
import type { AnalysisDoc, AnalysisResult, CohortDef, PartitionDef, ViewResult, ViewParticipation, TemplateDoc, SplitBy, LockDrift } from "./types.ts";
import { resolveSplitBy, type AttributeDistribution } from "./cohort-splitby.ts";
@@ -34,11 +35,16 @@ async function oncokbPin(): Promise {
} catch { return null; }
}
-const _profileCache = new Map(); // per-study
-export async function studyProfiles(study: string): Promise<{ profiles: string[]; gaProfiles: string[] }> {
+const _profileCache = new Map(); // per-study
+export interface StudyProfiles {
+ profiles: string[]; gaProfiles: string[];
+ /** every profile as the study describes it (genetic_profile), for what a plot axis may read */
+ catalog: ProfileInfo[];
+}
+export async function studyProfiles(study: string): Promise {
if (_profileCache.has(study)) return _profileCache.get(study)!;
const { peerForStudy } = await import("./federation-remote.ts");
- if (await peerForStudy(study)) return { profiles: [], gaProfiles: [] }; // the peer owns its own profile inventory
+ if (await peerForStudy(study)) return { profiles: [], gaProfiles: [], catalog: [] }; // the peer owns its own profile inventory
const conn = (await studyRouting()).get(study) || PRIMARY;
const profiles = (await chQuery(
"SELECT DISTINCT profile_type FROM " + conn.db + ".genetic_alteration_derived WHERE cancer_study_identifier = " + sq(study), conn)).data.map((r) => r.profile_type);
@@ -47,7 +53,15 @@ export async function studyProfiles(study: string): Promise<{ profiles: string[]
"SELECT DISTINCT replaceOne(gp.stable_id, " + sq(study + "_") + ", '') AS pt FROM " + conn.db + ".genetic_profile gp\n" +
"JOIN " + conn.db + ".cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id\n" +
"WHERE cs.cancer_study_identifier = " + sq(study) + " AND gp.genetic_alteration_type = 'GENERIC_ASSAY'", conn)).data.map((r) => r.pt);
- const out = { profiles, gaProfiles };
+ const catalog = (await chQuery(
+ "SELECT replaceOne(gp.stable_id, " + sq(study + "_") + ", '') AS profile, gp.genetic_alteration_type AS type, gp.datatype AS datatype,\n" +
+ " gp.name AS name, ifNull(toString(gp.generic_assay_type), '') AS assay, toInt32OrZero(ifNull(toString(gp.sort_order), '')) AS ord\n" +
+ "FROM " + conn.db + ".genetic_profile gp\n" +
+ "JOIN " + conn.db + ".cancer_study cs ON cs.cancer_study_id = gp.cancer_study_id\n" +
+ "WHERE cs.cancer_study_identifier = " + sq(study), conn)).data.map((r): ProfileInfo => ({
+ profile: String(r.profile), type: String(r.type), datatype: String(r.datatype), name: String(r.name),
+ ...(r.assay ? { assayType: String(r.assay) } : {}), order: Number(r.ord) || 0 }));
+ const out = { profiles, gaProfiles, catalog };
_profileCache.set(study, out);
return out;
}
diff --git a/lib/cohort2.ts b/lib/cohort2.ts
index 94d00247..d6af877b 100644
--- a/lib/cohort2.ts
+++ b/lib/cohort2.ts
@@ -232,6 +232,24 @@ function derivedLeafSql(leaf: any, ctx: Ctx): string {
" WHERE " + conds.join(" AND ");
}
+/**
+ * The cases (samples, or patients with a profiled sample) profiled for `gene` in one alteration
+ * type, as bare ids in the column `profiled_id`: the same panel tables, and the same whole-exome rule, as the `profiled` clause
+ * and the gene-table denominator. What an OQL plot axis needs to tell "not altered" from "not
+ * profiled", per sample or per patient.
+ */
+export function profiledCasesSql(gene: string, type: "MUTATION" | "CNA" | "SV", o: { study: string; db: string; unit: "patient" | "sample" }): string {
+ const g = String(gene).toUpperCase().trim();
+ if (!/^[A-Z0-9][A-Z0-9.-]{0,19}$/.test(g)) throw new Error("profiled: invalid gene symbol");
+ const col = o.unit === "sample" ? "sd.sample_unique_id" : "sd.patient_unique_id";
+ return "SELECT DISTINCT " + PID(col, o.study) + " AS profiled_id\n" +
+ " FROM " + o.db + ".sample_to_gene_panel_derived sp\n" +
+ " JOIN " + o.db + ".sample_derived sd ON sd.sample_unique_id = sp.sample_unique_id\n" +
+ " WHERE sp.cancer_study_identifier = " + sq(o.study) + " AND sp.alteration_type = " + sq(PROFILED_TYPE[type]) +
+ "\n AND (sp.gene_panel_id = 'WES' OR sp.gene_panel_id IN (\n" +
+ " SELECT gene_panel_id FROM " + o.db + ".gene_panel_to_gene_derived WHERE gene = " + sq(g) + "))";
+}
+
function profiledLeafSql(leaf: ProfiledLeaf, ctx: Ctx): string {
const t = PROFILED_TYPE[String(leaf.type || "").toUpperCase()];
if (!t) throw new Error("profiled clause: type must be MUTATION, CNA, or SV");
diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts
index af3aa848..0dae5cd1 100644
--- a/lib/dsl-reference.ts
+++ b/lib/dsl-reference.ts
@@ -17,12 +17,17 @@
// ============================================================================
import { DOC_VERSION } from "./doc-version.ts";
+import { POINTS_CAP } from "./plot-layers.ts";
+import { OQL_STATEMENT_CAP } from "./plot-oql.ts";
import type { TemplateDoc } from "./types.ts";
import type { CustomDataset } from "./custom-data-names.ts";
import { LUAD_BIOPSY_SITE_DEMO } from "./custom-data-demo.ts";
import { SPECTRUM_SC_DEMO } from "./spectrum-sc-demo.ts";
import { HTA11_SC_DEMO } from "./hta11-sc-demo.ts";
+/** the plot's points cap as the docs print it */
+const POINTS_CAP_TEXT = POINTS_CAP.toLocaleString("en-US");
+
export const DSL_VERSION = DOC_VERSION;
export interface DslExample {
@@ -459,9 +464,9 @@ export const EXAMPLES: DslExample[] = [
},
{
title: "OQL on a plot axis: STK11 or KEAP1 mutant lung cancer",
- demonstrates: "An alteration axis written as an OQL query — several genes, Altered when a case matches any of them — used as a box plot's groups and as a colour.",
+ demonstrates: "An alteration axis written as an OQL query — several genes, Altered when a case matches any of them, or one group per statement — used as a box plot's groups and as a colour.",
tags: ["OQL"],
- note: "Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients.",
+ note: "Non-small cell lung cancer in MSK-CHORD (7,809 patients). The first view groups tumour mutational burden by the OQL query `STK11: MUT; KEAP1: MUT`: 1,535 patients are Altered (a mutation in either gene), the same count as a cohort defined by that OQL, and 6,274 are Unaltered. The second view groups by sample type and colours each violin by the same query, counting samples rather than patients. The third gives each statement of `STK11: MUT, KEAP1: MUT, KRAS: G12C` its own group, over samples: 620 KRAS G12C, 459 KEAP1, 377 STK11, 403 with both STK11 and KEAP1 (median TMB 8.6 against 4.1 for the 5,720 with none), the other pairs as their own groups and 109 matching all three. None of these lung samples is on a panel without the three genes, so no group is Not profiled.",
doc: {
study: "msk_chord_2024",
cohorts: {
@@ -474,7 +479,28 @@ export const EXAMPLES: DslExample[] = [
{ kind: "plot", unit: "sample", cohorts: ["NSCLC"], allPoints: false, points: [],
x: { source: "clinical", attribute: "SAMPLE_TYPE" },
y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" },
- colorBy: { source: "alteration", oql: "STK11: MUT; KEAP1: MUT" } }
+ colorBy: { source: "alteration", oql: "STK11: MUT; KEAP1: MUT" } },
+ { kind: "plot", unit: "sample", cohorts: ["NSCLC"],
+ x: { source: "alteration", oql: "STK11: MUT, KEAP1: MUT, KRAS: G12C", by: "statement" },
+ y: { source: "clinical", attribute: "TMB_NONSYNONYMOUS" } }
+ ]
+ }
+ },
+ {
+ title: "Plot axes from the study's own profiles: ERBB2 copy number against expression",
+ demonstrates: "A `profile` axis: one gene in one of the study's molecular profiles, read as the study describes the profile — discrete copy number as its levels, expression as a number, mutations by type.",
+ tags: ["profiles"],
+ note: "TCGA breast cancer (PanCancer Atlas), every sample. The first view groups ERBB2 mRNA z-scores (relative to all samples) by ERBB2's GISTIC copy-number call, from Shallow deletion to Amplification: the 122 amplified samples sit far above the rest. The second groups fraction of genome altered by TP53's mutation type from the study's mutation profile; the 18 samples not in that profile are Not profiled, not counted as unmutated.",
+ doc: {
+ study: "brca_tcga_pan_can_atlas_2018",
+ cohorts: { "all samples": { all: [] } },
+ views: [
+ { kind: "plot", cohorts: ["all samples"], unit: "sample",
+ x: { source: "profile", profile: "gistic", gene: "ERBB2" },
+ y: { source: "profile", profile: "rna_seq_v2_mrna_median_all_sample_Zscores", gene: "ERBB2" } },
+ { kind: "plot", cohorts: ["all samples"], unit: "sample",
+ x: { source: "profile", profile: "mutations", gene: "TP53" },
+ y: { source: "clinical", attribute: "FRACTION_GENOME_ALTERED" } }
]
}
},
@@ -1522,7 +1548,7 @@ patient — so the arm counts summing to more than the base cohort is expected,
| \`logistic\` | \`outcome:axis, outcomeEvent?:"", covariates:[axis, ...(max 8)], cohorts:[names]\` | multivariate logistic regression: a BINARY outcome (not time-to-event \u2014 no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as \`cox\`'s. \`outcome\` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs \`outcomeEvent\` naming which observed value counts as the event (1) \u2014 every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given \`outcomeEvent\`. Covariates use the exact same dummy-coding and reference-level rules as \`cox\` (see above): a mutated-vs-not axis defaults to \`\"Not mutated\"\`, others default to the most frequent level, \`reference\` overrides either. Same patient-level-only boundary as \`cox\` |
| \`course\` | \`eventTypes:[\u22646], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]\` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin \u2014 first recorded event to last contact (the later of their last event and their OS follow-up) \u2014 never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) \`anchor\` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED \u2014 read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (\`Treatment\`/\`TREATMENT\`) and the fold is named in the caveat; nothing else is merged |
| \`sequence\` | \`labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]\` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES \u2014 therapies less than \`lineGap\` days apart are one line, labelled by their whole combination (sorted, so \`A + B\` and \`B + A\` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one \`lineGap\` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days \u2014 so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, \`lineClasses\` \`"all"\` for every treatment record or a list of class names for your own; \`labelKey\` only names a line\u2019s therapies (e.g. AGENT), it never decides which records form lines. \`exclude\` drops records by label value, matched ignoring case. More than two cohorts is refused |
-| \`plot\` | \`x:axis, y:axis, unit?:"sample"\\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; \`unit:"patient"\` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to 5,000 (\`allPoints: false\` turns that off), from a participant that may show cases; \`points\` lists up to four subsets drawn in colour over it — \`{rule:"outliers"}\` (the default), \`{rule:"extremes",axis,n}\`, \`{rule:"trend",n}\`, \`{cohort}\` — see **Points** below |
+| \`plot\` | \`x:axis, y:axis, unit?:"sample"\\|"patient", bins?:[240,160], allPoints?:true, points?:[layer(max 4)], colorBy?:axis, ringBy?:axis, cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. A point is a sample, as on cBioPortal's Plots tab; \`unit:"patient"\` makes it a patient, combining a patient's samples: a numeric value is their mean, a category their common value or "Multiple" where they differ, an alteration counts if any sample has it, and the caption says how many patients had more than one sample. Every case is a point up to ${POINTS_CAP_TEXT} (\`allPoints: false\` turns that off), from a participant that may show cases; \`points\` lists up to four subsets drawn in colour over it — \`{rule:"outliers"}\` (the default), \`{rule:"extremes",axis,n}\`, \`{rule:"trend",n}\`, \`{cohort}\` — see **Points** below |
| \`attrition\` | \`cohorts:[names] (≤6)\` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its \`all\`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated |
| \`genome\` | \`binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)\` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ \`threshold\`) or lost (≤ −\`threshold\`) in every \`binMb\`-megabase bin (1–50) along chromosomes 1–22 (and X with \`includeX\` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out |
| \`pathways\` | \`pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"\|"any", topGenes?:6, cohorts:[names] (≤4)\` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own \`sets\`, each as the share of the cohort with at least one member altered. \`measure\` defaults to \`"driver"\` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or \`"any"\`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as \`compare\`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The \`topGenes\` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named |
@@ -1581,13 +1607,28 @@ column id or a filter that does not suit its column is refused with the ids acce
### Plot axes
-An axis is one of four things, and the pair decides the shape — the portal's own rule:
+An axis is a clinical attribute, one of the study's own molecular profiles, or a query across genes,
+and the pair decides the shape — the portal's own rule:
| \`source\` | shape of the axis | fields |
| --- | --- | --- |
| \`clinical\` | number or category, **as the study declared it** | \`attribute\` |
-| \`molecular\` | number | \`profileType\`, \`gene\` |
-| \`alteration\` | category | \`gene\` **or** \`genes\` (2-20), \`by\`: \`"mutationType"\` (default) \\| \`"mutated"\` \\| \`"cna"\` |
+| \`profile\` | **as the study describes the profile** (below) | \`profile\` (its id without the study prefix), \`gene\`, \`by\` for a mutation profile, \`log\` for numbers |
+| \`alteration\` | category | \`oql\` (below), or \`genes\` (2-20, any mutated) |
+
+**A profile axis** reads one gene in one of the study's profiles, and what it is follows from the
+study's own description of that profile (\`genetic_profile\`: its alteration type and datatype), never
+from its name: a **mutation** profile (MAF) groups cases by mutation type, or with \`by: "mutated"\`
+mutated or not; a **structural-variant** profile by whether the case has one; a **discrete
+copy-number** profile by its own calls (Deep deletion, Shallow deletion, Diploid, Gain,
+Amplification, drawn in that order); and a **continuous** one (mRNA, z-scores, log2 copy number,
+protein) is a number. A case with no mutation, SV or high-level call in the gene is the wild-type
+answer only when it is profiled for that gene; otherwise it is **Not profiled**. A profile a study
+does not have is refused with the list it does have, and one that two pooled studies describe
+differently is refused, never read as one thing. Generic-assay profiles (methylation, arm-level copy
+number, …) are not plot axes yet. The engine runs a profile axis as the lower-level forms it
+resolves to, which a document may also give directly: \`{"source":"molecular","profileType","gene",
+"discrete"?}\` and \`{"source":"alteration","gene","by":"mutationType"|"mutated"|"cna"|"sv"}\`.
number × number draws a **density scatter**, number × category a **box**, category × category a
**contingency table**. Every statistic is computed where the rows are and is therefore exact over
@@ -1601,7 +1642,7 @@ inventing a second mechanism. Only \`by:"mutated"\` (the default when \`genes\`
supported for a set — combining several genes' mutation TYPES or copy-number states into one
category has no single honest reading, so that combination is refused rather than guessed at.
-Two things are refused rather than approximated. A molecular axis **must name one gene** — a
+Two things are refused rather than approximated. A profile axis **must name one gene** — a
whole profile is 630 million values in this warehouse, and reading them to draw one picture is
never what was meant. And a categorical axis with more than 150 distinct values is refused,
because a truncated contingency table is a wrong one.
@@ -1619,20 +1660,20 @@ uses, not yet built for a box); its quartiles are a size-weighted approximation
same one Explore's federated numeric histograms already disclose.
**Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every
-case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the
+case as a point. So does this, up to ${POINTS_CAP_TEXT} cases, on a scatter as on a box, and only when the
study is one this participant may see cases in — the points carry case ids, so they come from the
same entitlement that lets a patient-level participant list cohort members, and an aggregate-only
participant has no method that could return them. The jitter across a box is fixed per case, so
the figure does not move between the server SVG, the browser and the Python package.
-Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a
+Past ${POINTS_CAP_TEXT} the crowd is drawn as density — a quarter of a million dots is a solid block, and a
hover on it would land on an arbitrary case — and, by default, the **outliers** are a point layer
over it (see **Points**): on a box, the
values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops
at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at
-most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of
+most 3 cases (fewer, if that is what keeps the points under ${POINTS_CAP_TEXT}) is drawn as its cases instead of
as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure
-draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With
+draws, and if a subset is itself past ${POINTS_CAP_TEXT} none of it is drawn, never the first ${POINTS_CAP_TEXT}. With
\`allPoints: false\` and \`points: []\`, or from a participant that may not disclose a case, the
figure is density alone.
The caption always says which it drew, how many points, and why there are none when there are
@@ -1654,16 +1695,22 @@ bin's width is shared out between the classes in it. That stays an aggregate, so
may not disclose a case can still colour its plot. The key is capped at 12 classes.
**OQL on an axis.** An alteration axis can be an OQL query instead of genes:
-\`{"source":"alteration","oql":"KRAS: G12C; STK11: MUT"}\`. Each case is **Altered** when it matches
-any entry — the portal's own reading of a multi-gene query — and **Unaltered** otherwise; every
-case of the cohort is one or the other, so the unaltered are counted rather than missing. It is
-the same OQL a cohort's \`oql\` leaf takes (mutation classes and protein changes, AMP/HOMDEL,
-FUSION, DRIVER against the pinned OncoKB version, EXP/PROT thresholds), and it matches per patient
-or per sample following the plot's \`unit\`. Because it is a categorical axis, it also works as
-\`colorBy\`/\`ringBy\` and as a Cox or logistic covariate, where "Unaltered" is the reference level.
-OQL is for alteration axes only: a molecular axis stays one gene's value.
-
-**Points.** Every case is a point when there are at most 5,000 of them and every study may show
+\`{"source":"alteration","oql":"KRAS: G12C, STK11: MUT"}\`, its statements separated by a comma, a
+semicolon or a new line (as in a cohort's \`oql\` leaf). Each case is **Altered** when it matches any
+statement — the portal's own reading of a multi-gene query — and **Unaltered** when it matches none
+and is profiled for every statement's gene in the alteration types that statement asks about; a
+case whose gene panel omits one of them is **Not profiled**, never counted as wild type (the
+panel-aware rule of \`docs/PERFORMANCE.md\`). Every case of the cohort is one of the three, so none is
+missing. With \`"by": "statement"\` each statement is its own group, named as written: a case matching
+two is the pair ("KRAS: G12C + STK11: MUT"), three or more "3 or more lines", none Unaltered or Not
+profiled by the same rule (at most ${OQL_STATEMENT_CAP} statements). It is the same OQL a cohort's \`oql\` leaf takes
+(mutation classes and protein changes, AMP/HOMDEL, FUSION, DRIVER against the pinned OncoKB version,
+EXP/PROT thresholds), and it matches per patient or per sample following the plot's \`unit\`.
+Because it is a categorical axis, it also works as \`colorBy\`/\`ringBy\` and as a Cox or logistic
+covariate, where "Unaltered" is the reference level.
+OQL is for alteration axes only: a profile axis stays one gene in one profile.
+
+**Points.** Every case is a point when there are at most ${POINTS_CAP_TEXT} of them and every study may show
cases (\`allPoints: false\` draws the density alone at any size). Over that the plotted cohort is a
density, and \`points\` lists the subsets worth seeing one by one, each drawn in its own colour with
the plotted cohort turned grey beneath them (at most four layers, in order):
@@ -1675,7 +1722,7 @@ grid cell holding at most 3 cases, left unshaded under its points); \`{"rule": "
\`{"cohort": "KRAS G12C"}\` — the plotted cohort's cases that are also in that cohort (how many of its
patients fall outside the plotted cohort is stated, and they are not drawn). The default is
\`[{"rule": "outliers"}]\`; \`[]\` draws none. A case in two layers is drawn by the first, and the
-caption counts what each layer left to an earlier one. Every layer shares one 5,000-point budget,
+caption counts what each layer left to an earlier one. Every layer shares one ${POINTS_CAP_TEXT}-point budget,
and a layer is never cut to its first cases: a cohort layer past the budget, or from a study that
may not show cases, is drawn as an aggregate on the plot's own scale (cells on the scatter's grid,
or a narrow box per group), and a rule layer is not drawn — the caption says which and why.
@@ -1898,11 +1945,14 @@ export const AGENT_GRAMMAR = `Document shape:
// = {"rule":"outliers"} | {"rule":"extremes","axis":"x|y","n":50?} | {"rule":"trend","n":50?} | {"cohort":C}
// default [{"rule":"outliers"}]; [] draws none; the first layer draws a case two share
// = {"source":"clinical","attribute":A,"log":?}
- // | {"source":"molecular","profileType":P,"gene":G,"log":?}
- // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|driver"?}
+ // | {"source":"profile","profile":P,"gene":G,"by":"mutationType|mutated"?,"log":?}
+ // // P is one of the study's profiles; what it is (mutations, SVs, discrete or
+ // // continuous copy number, mRNA, protein) comes from the study's own description
+ // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|sv"?} // what a profile axis resolves to
// | {"source":"alteration","genes":[G,...(2-20)],"by":"mutated"?} // any-of-N as one category; only "mutated"
- // | {"source":"alteration","oql":"KRAS: G12C; STK11: MUT"} // Altered (matches any entry) vs Unaltered
- // two numeric axes -> scatter (binned; every case a point up to 5,000, else sparse-cell outliers); one categorical -> box;
+ // | {"source":"alteration","oql":"KRAS: G12C, STK11: MUT","by":"any|statement"?}
+ // // any: Altered / Unaltered / Not profiled; statement: one group per statement (max 6)
+ // two numeric axes -> scatter (binned; every case a point up to ${POINTS_CAP_TEXT}, else sparse-cell outliers); one categorical -> box;
// two categorical -> a contingency table
{"kind":"attrition","cohorts":[names?(max 6)]},
{"kind":"genome","binMb":2,"threshold":0.2,"includeX":false,"cohorts":[names?(max 4)]},
diff --git a/lib/federation.ts b/lib/federation.ts
index 63c0c2e2..2c33ae70 100644
--- a/lib/federation.ts
+++ b/lib/federation.ts
@@ -27,7 +27,7 @@ import { compileCohortDef, eventTypeMatchSql, memberSamplesSql, lockDriftSql } f
import { axisKind, axisLabel, axisSelect, corrPValue, plotShape, tukey, validateAxis,
categoryColor, CATEGORY_CAP, COLOR_CAP, DEFAULT_BINS, POINTS_CAP, VIOLIN_BINS,
plotBinsSql, plotBoxSql, plotCountSql, plotCoverageSql, plotMomentsSql, plotPointsSql, plotStatsSql, plotTableSql, plotViolinSql,
- highlightCohortSql, plotHighlightCountSql, plotScatterPointsSql, plotRankedPointsSql,
+ highlightCohortSql, plotHighlightCountSql, plotScatterPointsSql, plotRankedPointsSql, OQL_NOT_PROFILED,
type AxisKind, type HighlightPoint, type PlotAxis, type PlotData, type PlotPoint, type PlotRequest, type PlotShape,
type PlotXYPoint, type SparseCells, type Whiskers, type PointRank } from "./plot.ts";
// re-exported so test/boundary.test.ts sweeps them with the other aggregate SQL builders
@@ -807,6 +807,24 @@ export function anchoredSurvivalSql(
* patient-level only (see PatientLevelResolver.coxRows) rather than something every
* participant can answer.
*/
+/** an axis that can answer "Not profiled": a gene's alterations, or a discrete copy-number profile */
+function undecidable(a: PlotAxis): boolean {
+ return a.source === "alteration" || (a.source === "molecular" && !!a.discrete);
+}
+/** an axis's rows without its "Not profiled" cases */
+function decidedOnly(sel: string): string {
+ return "SELECT id, v FROM (\n" + sel + "\n) WHERE v != " + sq(OQL_NOT_PROFILED);
+}
+/**
+ * The SQL counting the rows of a model's query (before "Not profiled" is left out) that are not
+ * profiled on an undecidable axis: the patients a fit leaves out for that reason. null when no axis
+ * of the model can be undecided.
+ */
+function notProfiledCountSql(unfiltered: string, sels: { label: string }[], axes: PlotAxis[]): string | null {
+ const cols = sels.filter((_, i) => undecidable(axes[i])).map((s) => s.label + " = " + sq(OQL_NOT_PROFILED));
+ return cols.length ? "SELECT countIf(" + cols.join(" OR ") + ") AS n FROM (\n" + unfiltered + "\n)" : null;
+}
+
export function coxRowsSql(
cohortSql: string, study: string, db: string, endpoint: string, covariates: { label: string; sel: string }[],
columns?: EndpointColumns
@@ -1289,10 +1307,10 @@ export interface PatientLevelCapable extends AggregateCapable {
members(def: CohortDef | null | undefined): Promise;
/** per-patient rows for a Cox fit — see PatientLevelResolver.coxRows for why this is here and not on AggregateCapable */
coxRows(def: CohortDef | null | undefined, endpoint: string, covariates: PlotAxis[], columns?: EndpointColumns):
- Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[] }>;
+ Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[]; notProfiled: number }>;
/** per-patient rows for a logistic fit — same reason as coxRows */
logisticRows(def: CohortDef | null | undefined, outcome: PlotAxis, covariates: PlotAxis[]):
- Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[] }>;
+ Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[]; notProfiled: number }>;
/** one row per patient (never an identifier) for a view in lib/views/ (a lab waterfall) — see PatientLevelResolver.viewRows */
viewRows(label: string, def: CohortDef | null | undefined, build: (cohortSql: string, study: string, db: string) => string): Promise;
/** a table view's rows (lib/table-query.ts) in this instance, with their ids — see PatientLevelResolver.rowsPage */
@@ -2061,21 +2079,33 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa
* single row carries an identifier. `kinds` says which covariates came back numeric so the
* caller (which dummy-codes categoricals) doesn't have to re-derive it.
*/
+ /** a count query's answer, or 0 when there is nothing to count */
+ private async countIfAny(label: string, sql: string | null, conn: ChConn): Promise {
+ if (!sql) return 0;
+ const r = await this.runLogged(label, sql, conn);
+ return Number(r.data[0]?.n || 0);
+ }
+
async coxRows(def: CohortDef | null | undefined, endpoint: string, covariates: PlotAxis[], columns?: EndpointColumns):
- Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[] }> {
+ Promise<{ rows: { t: number; event: number; values: (number | string)[] }[]; kinds: AxisKind[]; notProfiled: number }> {
covariates.forEach((c, i) => validateAxis(c, "covariate " + (i + 1)));
const db = this.instance.conn.db, conn = this.instance.conn;
const kindsMap = await this.clinicalKinds(covariates);
const kinds = covariates.map((c) => axisKind(c, (a) => kindsMap.get(a)));
const ctx = { study: this.study, db, unit: "patient" as const, cohortSql: this.cohortSql(def), ...this.oqlCtx() };
const sels = covariates.map((c, i) => ({ label: "c" + i, sel: axisSelect(c, ctx, kinds[i]) }));
- const sql = coxRowsSql(ctx.cohortSql, this.study, db, endpoint, sels, columns);
- const r = await this.runLogged("coxRows:" + endpoint, sql, conn);
+ // a patient not profiled for a covariate's gene has no value to fit, so the fit leaves them
+ // out (in the query the method record re-runs) and counts them, rather than coding "unknown"
+ const decided = sels.map((s, i) => undecidable(covariates[i]) ? { ...s, sel: decidedOnly(s.sel) } : s);
+ const sql = coxRowsSql(ctx.cohortSql, this.study, db, endpoint, decided, columns);
+ const [r, np] = await Promise.all([
+ this.runLogged("coxRows:" + endpoint, sql, conn),
+ this.countIfAny("coxNotProfiled:" + endpoint, notProfiledCountSql(coxRowsSql(ctx.cohortSql, this.study, db, endpoint, sels, columns), sels, covariates), conn)]);
const rows = r.data.map((row) => ({
t: Number(row.m), event: Number(row.ev),
values: covariates.map((_, i) => kinds[i] === "num" ? Number(row["c" + i]) : String(row["c" + i]))
}));
- return { rows, kinds };
+ return { rows, kinds, notProfiled: np };
}
/**
@@ -2145,7 +2175,7 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa
* level, or which already-0/1 numeric value, counts as the event.
*/
async logisticRows(def: CohortDef | null | undefined, outcome: PlotAxis, covariates: PlotAxis[]):
- Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[] }> {
+ Promise<{ rows: { y: number | string; values: (number | string)[] }[]; outcomeKind: AxisKind; kinds: AxisKind[]; notProfiled: number }> {
validateAxis(outcome, "outcome");
covariates.forEach((c, i) => validateAxis(c, "covariate " + (i + 1)));
const db = this.instance.conn.db, conn = this.instance.conn;
@@ -2155,13 +2185,19 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa
const ctx = { study: this.study, db, unit: "patient" as const, cohortSql: this.cohortSql(def), ...this.oqlCtx() };
const outcomeSel = { label: "y0", sel: axisSelect(outcome, ctx, outcomeKind) };
const sels = covariates.map((c, i) => ({ label: "c" + i, sel: axisSelect(c, ctx, kinds[i]) }));
- const sql = logisticRowsSql(ctx.cohortSql, outcomeSel, sels);
- const r = await this.runLogged("logisticRows", sql, conn);
+ // as for Cox: not profiled for the outcome's or a covariate's gene is left out and counted
+ const all = [outcomeSel, ...sels], axes = [outcome, ...covariates];
+ const [dOut, ...dSels] = all.map((s, i) => undecidable(axes[i]) ? { ...s, sel: decidedOnly(s.sel) } : s);
+ const sql = logisticRowsSql(ctx.cohortSql, dOut, dSels);
+ const counted = all.map((s, i) => ({ ...s, label: i === 0 ? "y" : s.label }));
+ const [r, np] = await Promise.all([
+ this.runLogged("logisticRows", sql, conn),
+ this.countIfAny("logisticNotProfiled", notProfiledCountSql(logisticRowsSql(ctx.cohortSql, outcomeSel, sels), counted, axes), conn)]);
const rows = r.data.map((row) => ({
y: outcomeKind === "num" ? Number(row.y) : String(row.y),
values: covariates.map((_, i) => kinds[i] === "num" ? Number(row["c" + i]) : String(row["c" + i]))
}));
- return { rows, outcomeKind, kinds };
+ return { rows, outcomeKind, kinds, notProfiled: np };
}
}
diff --git a/lib/figures/plot.ts b/lib/figures/plot.ts
index e49fd3a7..4d8f8995 100644
--- a/lib/figures/plot.ts
+++ b/lib/figures/plot.ts
@@ -5,7 +5,7 @@
// picture draws for a hundred cases or four hundred thousand. The only cases drawn one by one are
// the points the engine chose (every case under the cap, the outliers over it), at most POINTS_CAP.
import type { ViewResult } from "../types.ts";
-import type { PlotData } from "../plot.ts";
+import type { HighlightPoint, PlotData, PlotLayer } from "../plot.ts";
import { isAbsent, jitterFor } from "../plot-draw.ts";
import { LAYER_BASE } from "../plot-layers.ts";
import {
@@ -66,6 +66,19 @@ function layerItems(d: PlotData): LegendItem[] {
}
/** the layers drawn at all: with any, the plotted cohort steps back to a grey so their colours read */
const drawnLayers = (d: PlotData) => (d.layers || []).filter((l) => l.shown !== "none");
+/**
+ * One layer's marks as a group tagged with its place in the view's list, so a page can hide it;
+ * `base` is the plotted cohort. A layer's cases an earlier layer drew are hidden groups, one per
+ * layer that drew them (`owner`), which the page shows when that owner is hidden.
+ */
+const layerGroup = (key: number | "base", marks: string[], owner?: number) =>
+ marks.length ? `${marks.join("")}` : "";
+/** a layer's shared cases, grouped by the layer that drew each */
+function sharedGroups(h: PlotLayer, key: number, mark: (pt: HighlightPoint) => string[]): string {
+ const by = new Map();
+ (h.shared || []).forEach((pt, i) => { const o = h.sharedOwners?.[i] ?? 0; (by.get(o) ?? by.set(o, []).get(o)!).push(...mark(pt)); });
+ return [...by.entries()].map(([o, marks]) => layerGroup(key, marks, o)).join("");
+}
function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string {
const { x0, x1, y0, y1 } = d.extent!, [gx, gy] = d.grid!;
@@ -100,17 +113,24 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string {
const shaded = d.pointSet?.rule === "all" && d.xyPoints?.length ? []
: cellMax ? d.bins!.filter((b) => b.n > cellMax) : d.bins!;
// under point layers the plotted cohort is a floor-raised grey, so its faintest cells still read
- for (const b of shaded) out.push(cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n)));
const own = d.xyPoints || [];
- for (const pt of own)
- out.push(``);
- // a cohort layer too large to draw as points is cells on the same grid, shaded by its own densest cell
- for (const h of hs) if (h.shown === "density" && h.bins) {
- const hMax = Math.max(1, ...h.bins.map((b) => b.n));
- for (const b of h.bins) out.push(cell(b, h.color, 0.3 + 0.7 * Math.log1p(b.n) / Math.log1p(hMax)));
+ out.push(layerGroup("base", [
+ ...shaded.map((b) => cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))),
+ ...own.map((pt) => ``)]));
+ const dot = (pt: { x: number; y: number }, color: string) =>
+ ``;
+ for (const h of hs) {
+ const key = d.layers!.indexOf(h);
+ // a cohort layer too large to draw as points is cells on the same grid, shaded by its own densest cell
+ if (h.shown === "density" && h.bins) {
+ const hMax = Math.max(1, ...h.bins.map((b) => b.n));
+ out.push(layerGroup(key, h.bins.map((b) => cell(b, h.color, 0.3 + 0.7 * Math.log1p(b.n) / Math.log1p(hMax)))));
+ }
+ if (h.shown === "points") {
+ out.push(layerGroup(key, h.points!.flatMap((pt) => "x" in pt ? [dot(pt, h.color)] : [])));
+ out.push(sharedGroups(h, key, (pt) => "x" in pt ? [dot(pt, h.color)] : []));
+ }
}
- for (const h of hs) if (h.shown === "points") for (const pt of h.points!) if ("x" in pt)
- out.push(``);
// the fitted line, over every case rather than over what is visible, clipped to the frame
if (st && st.slope != null && st.intercept != null && x1 > x0) {
const yAt = (xv: number) => st.intercept! + st.slope! * xv;
@@ -209,6 +229,8 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string {
gs.forEach((g, i) => {
const cx = L + bw * (i + 0.5), half = Math.min(24, bw * 0.32);
+ // the plotted cohort's violin and points, as one group a page can step back from
+ const baseMarks: string[] = [];
if (d.overlay === "violin" && vio.has(g.g)) {
const cells = vio.get(g.g)!;
// a bin stands for a range of VALUES, mapped through the extent the bins were cut over
@@ -224,7 +246,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string {
const yTop = ys(Math.min(cHi, vHi));
const step = Math.max(1.2, ys(Math.max(cLo, vLo)) - yTop);
if (!colorKey.length) {
- out.push(``);
+ baseMarks.push(``);
continue;
}
let x = cx - w;
@@ -233,7 +255,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string {
for (const sg of segs) {
if (!sg.n) continue;
const seg = 2 * w * (sg.n / total);
- out.push(``);
+ baseMarks.push(``);
x += seg;
}
}
@@ -245,14 +267,15 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string {
const ring = pt.r != null && !isAbsent(pt.r) ? ringOf(pt.r) : null;
if (pt.v > vhi || pt.v < vlo) {
ptOffScale++;
- out.push(edgeMark(cx + jitterFor(pt.id) * half * 0.9, pt.v > vhi, rBase, fill));
+ baseMarks.push(edgeMark(cx + jitterFor(pt.id) * half * 0.9, pt.v > vhi, rBase, fill));
continue;
}
- out.push(``);
}
}
+ out.push(layerGroup("base", baseMarks));
out.push(``);
out.push(`${esc(g.g)}: median ${fmtNum(g.median)}, ` +
@@ -265,26 +288,29 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string {
const hg = h.groups!.find((x) => x.g === g.g);
if (!hg) return;
const hw = Math.max(1.5, half * 0.42), hx = cx + (j - (boxed.length - 1) / 2) * hw * 1.25;
- out.push(`` +
+ out.push(layerGroup(d.layers!.indexOf(h), [`` +
`` +
- ``);
+ ``]));
});
// A layer's value past the axis is pinned to the frame's edge as a triangle pointing off it, and
// counted in the caption, rather than drawn outside the figure. With the cases coloured by a
// category, a layer is a ring round the case, so fill and ring each keep one meaning.
for (const h of hs) if (h.shown === "points") {
const rH = PLOT_POINT_R;
- for (const pt of h.points!) if ("g" in pt && pt.g === g.g) {
+ const marks = (pts: HighlightPoint[], counted: boolean) => pts.flatMap((pt) => {
+ if (!("g" in pt) || pt.g !== g.g) return [];
const px = cx + jitterFor(pt.id) * half * 0.9;
if (pt.v > vhi || pt.v < vlo) {
- offScale++;
- out.push(edgeMark(px, pt.v > vhi, rH, h.color));
- continue;
+ if (counted) offScale++;
+ return [edgeMark(px, pt.v > vhi, rH, h.color)];
}
- out.push(categorised
+ return [categorised
? ``
- : ``);
- }
+ : ``];
+ });
+ const key = d.layers!.indexOf(h);
+ out.push(layerGroup(key, marks(h.points!, true)));
+ out.push(sharedGroups(h, key, (pt) => marks([pt], false)));
}
});
out.push(``);
diff --git a/lib/gene-query.ts b/lib/gene-query.ts
index 52362961..512ca7d7 100644
--- a/lib/gene-query.ts
+++ b/lib/gene-query.ts
@@ -2,7 +2,7 @@
// (each line or ";"-separated entry a gene and what counts as altered in it) or a plain gene list
// (cBioPortal's, where `[A B]` is one merged row). Pure and browser-safe; the symbols it finds are
// checked against the warehouse's gene table separately (POST /api/genes).
-import { parseOql, type OqlSpec } from "./oql-parse.ts";
+import { parseOql, OQL_SEPARATOR, type OqlSpec } from "./oql-parse.ts";
import { parseGeneList, geneSymbols, normalizeGeneTracks, MAX_GENE_TRACKS, type GeneTrackSpec } from "./gene-tracks.ts";
export type GeneQueryMode = "oql" | "genes";
@@ -64,7 +64,7 @@ export function checkGeneQuery(text: string, mode: GeneQueryMode): QueryCheck {
return { entries: [], genes: [], error: String((e as Error).message || e) };
}
}
- const parts = text.split(/[;\n]/).flatMap(splitBareGenes);
+ const parts = text.split(OQL_SEPARATOR).flatMap(splitBareGenes);
const entries: QueryEntry[] = parts.map((p) => {
const head = (p.match(/^([A-Za-z0-9.-]+)/)?.[1] || "").toUpperCase();
try {
diff --git a/lib/method.ts b/lib/method.ts
index 43b09d6a..0d676f63 100644
--- a/lib/method.ts
+++ b/lib/method.ts
@@ -115,7 +115,8 @@ export interface FitTrace {
/** the cohort this fit belongs to, when each cohort is fitted separately */
scope?: string;
/** per study: patients in the cohort there, and rows its query returned */
- perStudy: { study: string; cohortPatients: number; rows: number }[];
+ /** each study's cohort patients, rows fetched, and patients left out as not profiled for an axis's gene */
+ perStudy: { study: string; cohortPatients: number; rows: number; notProfiled?: number }[];
covariates: CoxCovariate[];
kinds: AxisKind[];
columns: MethodColumn[];
@@ -160,7 +161,9 @@ function fetchSteps(t: FitTrace, query: string, keep: string): MethodStep[] {
kind: "fetch" as const,
title: "Fetch one row per patient",
detail: "The " + query + " query above. " + keep + " " +
- (s.cohortPatients > s.rows ? plural(s.cohortPatients - s.rows, "cohort patient") + " had no row." : "Every cohort patient had a row."),
+ (s.cohortPatients > s.rows ? plural(s.cohortPatients - s.rows, "cohort patient") + " had no row" +
+ (s.notProfiled ? ", " + s.notProfiled.toLocaleString("en-US") + " of them because " + (s.notProfiled === 1 ? "it was" : "they were") +
+ " not profiled for the gene of an axis (a value of “Not profiled” is left out, never fitted as a level)" : "") + "." : "Every cohort patient had a row."),
scope: s.study,
rowsIn: s.cohortPatients,
rowsOut: s.rows,
diff --git a/lib/oql-parse.ts b/lib/oql-parse.ts
index 1fa83402..dc99b8e7 100644
--- a/lib/oql-parse.ts
+++ b/lib/oql-parse.ts
@@ -27,11 +27,18 @@ const REJECT: Record = {
SOMATIC: "SOMATIC modifier is not supported (all stored mutations are somatic/unknown)"
};
+/** what separates one OQL statement from the next: a semicolon, a new line or a comma */
+export const OQL_SEPARATOR = /[;\n,]/;
+/** a query's statements as typed, trimmed, empty ones dropped */
+export function oqlStatements(q: string): string[] {
+ return q.split(OQL_SEPARATOR).map((s) => s.trim()).filter(Boolean);
+}
+
// -> [{gene, specs: [{type:'MUT'|'CNA'|'SV'|'ANY', pchange?, cls?, cna?, driver}]}]
export function parseOql(q: string): OqlEntry[] {
if (typeof q !== "string" || !q.trim()) throw new Error("empty OQL");
if (q.length > 1000) throw new Error("OQL too long");
- const entries = q.split(/[;\n]/).map(s => s.trim()).filter(Boolean);
+ const entries = oqlStatements(q);
// Each entry compiles to its own UNION-DISTINCT'd subquery (see below), so cost scales
// roughly linearly rather than sitting at a wall near any particular number — 30 covers a
// full pathway gene set (e.g. a 20-30 gene DDR panel) with room to spare, not a measured
diff --git a/lib/plot-draw.ts b/lib/plot-draw.ts
index eaf775cc..ffdddd9a 100644
--- a/lib/plot-draw.ts
+++ b/lib/plot-draw.ts
@@ -3,7 +3,7 @@
/** categories that mean "this case has none of that", which draw as neutral and never as a ring */
export function isAbsent(label: string): boolean {
- return /^(not mutated|unaltered|neutral|no data|wild type|none|)$/i.test(String(label).trim());
+ return /^(not mutated|unaltered|neutral|diploid|no structural variant|not profiled|no data|wild type|none|)$/i.test(String(label).trim());
}
/**
diff --git a/lib/plot-layers.ts b/lib/plot-layers.ts
index 14913bf3..c4bc096d 100644
--- a/lib/plot-layers.ts
+++ b/lib/plot-layers.ts
@@ -4,8 +4,11 @@
/**
* How many cases a plot draws as points before it stops being a drawing: at or under it every case
* is a point; past it the plotted cohort is density, and the point layers share this many points.
+ * Set from a measurement (docs/plot-case-level.md): an SVG of n points paints in about 18 ms per
+ * thousand on the dev server's headless Chrome, so 10,000 keeps a redraw near 200 ms; the points
+ * travel as about 44 bytes each.
*/
-export const POINTS_CAP = 5000;
+export const POINTS_CAP = 10000;
/** layers one plot may draw: each takes a colour of its own, and past four they stop being told apart */
export const POINT_LAYER_CAP = 4;
/** a layer's colour, by its place in the view's list */
diff --git a/lib/plot-oql.ts b/lib/plot-oql.ts
index 51b73584..3eecde3c 100644
--- a/lib/plot-oql.ts
+++ b/lib/plot-oql.ts
@@ -1,4 +1,13 @@
-// The two categories an OQL plot axis takes. Browser-safe, so the method panel names an OQL
-// covariate's reference level the way the engine chose it.
+// The categories an alteration plot axis takes, and its limits. Browser-safe, so the method panel
+// names an OQL covariate's reference level the way the engine chose it, and the axis's gene dialog
+// holds the reader to the limits the engine enforces.
export const OQL_ALTERED = "Altered";
export const OQL_UNALTERED = "Unaltered";
+/** a case not profiled for a statement it did not match: not altered as far as anyone knows, but never "Unaltered" */
+export const OQL_NOT_PROFILED = "Not profiled";
+/** one group per statement: a case matching three or more statements */
+export const OQL_SEVERAL = "3 or more lines";
+/** one group per statement: at most this many statements, so the groups (and their pairs) stay readable */
+export const OQL_STATEMENT_CAP = 6;
+/** a gene-set alteration axis takes at most this many genes */
+export const ALTERATION_GENE_SET_CAP = 20;
diff --git a/lib/plot-profile.ts b/lib/plot-profile.ts
new file mode 100644
index 00000000..cba8d27b
--- /dev/null
+++ b/lib/plot-profile.ts
@@ -0,0 +1,42 @@
+// A profile axis resolved against the studies it is drawn over: the study's own description of the
+// profile (lib/profile-catalog.ts) decides what the axis is, and the engine then runs the form that
+// reading calls for. A profile a study lacks, or one two pooled studies describe differently, is
+// refused by name rather than read as something it is not.
+import type { PlotAxis } from "./plot.ts";
+import { profileUse, plottableProfiles, type ProfileInfo } from "./profile-catalog.ts";
+
+/** each study the axis is drawn over, with its profiles as it describes them */
+export type StudyCatalog = { study: string; catalog: ProfileInfo[] };
+
+export function resolveProfileAxis(a: PlotAxis, which: string, studies: StudyCatalog[]): PlotAxis {
+ if (!a || a.source !== "profile") return a;
+ const found = studies.map((s) => ({ study: s.study, catalog: s.catalog, info: s.catalog.find((p) => p.profile === a.profile) }));
+ const missing = found.filter((f) => !f.info);
+ if (missing.length) {
+ const offer = plottableProfiles(missing[0].catalog).map((p) => p.profile);
+ throw new Error(which + " axis: " + missing.map((m) => m.study).join(", ") + " has no profile '" + a.profile + "'" +
+ (offer.length ? "; it has " + offer.slice(0, 12).join(", ") + (offer.length > 12 ? " and " + (offer.length - 12) + " more" : "") : "; it has none a plot axis can read"));
+ }
+ const uses = [...new Set(found.map((f) => profileUse(f.info!)))];
+ if (uses.length > 1)
+ throw new Error(which + " axis: the profile '" + a.profile + "' is described differently by " + found.map((f) => f.study + " (" + f.info!.type + ", " + f.info!.datatype + ")").join(" and ") +
+ ", so one axis cannot read it across them");
+ const info = found[0].info!, use = uses[0];
+ if (!use) throw new Error(which + " axis: '" + a.profile + "' (" + info.type + ", " + info.datatype + ") is not a profile a plot axis reads");
+ if (a.by != null && use !== "mutation") throw new Error(which + " axis: only a mutation profile groups by mutation type or mutated; '" + a.profile + "' is " + info.name);
+ if (a.log && use !== "number") throw new Error(which + " axis: a log scale applies to numbers; '" + a.profile + "' is " + info.name);
+ const gene = String(a.gene).toUpperCase();
+ const label = gene + ": " + info.name;
+ if (use === "mutation") return { source: "alteration", gene, by: a.by || "mutationType" };
+ if (use === "sv") return { source: "alteration", gene, by: "sv" };
+ if (use === "discrete") return { source: "molecular", profileType: a.profile, gene, discrete: true, label };
+ if (use === "number") return { source: "molecular", profileType: a.profile, gene, ...(a.log ? { log: true } : {}), label };
+ throw new Error(which + " axis: '" + a.profile + "' is a generic assay (" + info.name + "), which a plot axis does not read yet");
+}
+
+/** every axis of a view resolved; `null` axes (an unset colour) pass through */
+export function resolveAxes>(axes: T, studies: StudyCatalog[]): T {
+ const out: Record = {};
+ for (const [k, a] of Object.entries(axes)) out[k] = a ? resolveProfileAxis(a, k, studies) : a;
+ return out as T;
+}
diff --git a/lib/plot.ts b/lib/plot.ts
index bf9e071e..e4f6a04f 100644
--- a/lib/plot.ts
+++ b/lib/plot.ts
@@ -17,12 +17,34 @@ import { DB } from "./ch.ts";
import { STYLE } from "./oncoprint-style.ts";
import { clinicalTable, ATTR_NAME_RE } from "./custom-data.ts";
import { parseOql, oqlCasesSql } from "./oql.ts";
-import { OQL_ALTERED, OQL_UNALTERED } from "./plot-oql.ts";
+import { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP, ALTERATION_GENE_SET_CAP } from "./plot-oql.ts";
+import { oqlStatements } from "./oql-parse.ts";
+import { profiledCasesSql } from "./cohort2.ts";
+
+/** the alteration types a statement must be profiled in before "not matched" can mean "not altered" */
+const OQL_PROFILED_TYPES: Partial> = {
+ MUT: ["MUTATION"], CNA: ["CNA"], SV: ["SV"], ANY: ["MUTATION", "CNA", "SV"]
+};
/** what goes on one axis */
export type PlotAxis =
| { source: "clinical"; attribute: string; log?: boolean }
- | { source: "molecular"; profileType: string; gene: string; log?: boolean }
+ /**
+ * One of the study's own molecular profiles, by its id without the study prefix, and a gene. What
+ * the axis is follows from the profile's own type and datatype (lib/profile-catalog.ts): a
+ * mutation profile groups cases by mutation type (or `by: "mutated"`), a structural-variant
+ * profile by whether the case has one, a discrete copy-number profile by its levels, and a
+ * continuous one (expression, z-scores, log2 copy number, protein) is a number. Resolved against
+ * the study before it runs (lib/plot-profile.ts) into the forms below; a profile the study does
+ * not have is refused by name.
+ */
+ | { source: "profile"; profile: string; gene: string; by?: "mutationType" | "mutated"; log?: boolean }
+ /**
+ * The forms a profile axis resolves to, which the engine runs, and which a document may also give
+ * directly: one gene's value in a profile (`discrete` reads it as copy-number levels), and below,
+ * a gene's alterations from the event table. `label` is what the figure calls the axis.
+ */
+ | { source: "molecular"; profileType: string; gene: string; log?: boolean; discrete?: boolean; label?: string }
/**
* `genes` (2+) asks "is ANY of these altered" as one categorical — a pathway rollup (e.g. a
* DDR gene set) instead of one gene at a time — and only for `by:"mutated"`: combining several
@@ -32,19 +54,22 @@ export type PlotAxis =
| { source: "alteration"; gene: string; by?: AlterationBy }
| { source: "alteration"; genes: string[]; by?: "mutated" }
/**
- * An OQL query as one category: "Altered" when the case matches any entry, the portal's own
- * reading of a multi-gene query, "Unaltered" otherwise. Per patient (any of its samples) or per
- * sample, following the plot's unit.
+ * An OQL query, its statements separated by ";", a new line or ",". `by: "any"` (the default)
+ * is one category: "Altered" when the case matches any statement, the portal's own reading of a
+ * multi-gene query, "Unaltered" when it matches none and is profiled for every statement's gene,
+ * "Not profiled" otherwise. `by: "statement"` gives each statement its own group, a case
+ * matching two a group of the pair, three or more "3 or more lines", none "Unaltered" or "Not
+ * profiled" by the same rule. Per patient (any of its samples) or per sample, following the unit.
*/
- | { source: "alteration"; oql: string }
+ | { source: "alteration"; oql: string; by?: "any" | "statement" }
| { source: "none" };
-export type AlterationBy = "mutated" | "mutationType" | "cna";
-const AXIS_SOURCES: readonly string[] = ["clinical", "molecular", "alteration", "none"] satisfies PlotAxis["source"][];
+export type AlterationBy = "mutated" | "mutationType" | "cna" | "sv";
+const AXIS_SOURCES: readonly string[] = ["clinical", "profile", "molecular", "alteration", "none"] satisfies PlotAxis["source"][];
/** an alteration axis given as an OQL query, rather than as genes */
-function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string } {
+function isOqlAxis(a: PlotAxis): a is { source: "alteration"; oql: string; by?: "any" | "statement" } {
return a.source === "alteration" && "oql" in a && a.oql != null;
}
-export { OQL_ALTERED, OQL_UNALTERED };
+export { OQL_ALTERED, OQL_UNALTERED, OQL_NOT_PROFILED, OQL_SEVERAL, OQL_STATEMENT_CAP, ALTERATION_GENE_SET_CAP };
/** every gene named on an alteration axis, singular or set, uppercased */
function alterationGenes(a: { gene?: string; genes?: string[] }): string[] {
@@ -52,8 +77,6 @@ function alterationGenes(a: { gene?: string; genes?: string[] }): string[] {
}
/** a patient-mode category for a patient whose samples carry different values */
export const MULTIPLE = "Multiple";
-/** a gene-set alteration axis takes at most this many genes */
-export const ALTERATION_GENE_SET_CAP = 20;
export type AxisKind = "num" | "cat" | "none";
export type PlotShape = "scatter" | "box" | "table" | "waterfall";
@@ -117,15 +140,19 @@ export function plotShape(x: AxisKind, y: AxisKind): PlotShape {
/** what an axis is called on the figure */
export function axisLabel(a: PlotAxis): string {
if (a.source === "clinical") return a.log ? "log10(" + a.attribute + " + 1)" : a.attribute;
- if (a.source === "molecular") return a.log ? "log10(" + a.gene + " " + a.profileType + " + 1)" : a.gene + " " + a.profileType;
+ if (a.source === "molecular") {
+ const name = a.label || a.gene + " " + a.profileType;
+ return a.log ? "log10(" + name + " + 1)" : name;
+ }
+ if (a.source === "profile") return a.gene + " " + a.profile;
if (isOqlAxis(a)) {
- const q = a.oql.trim().replace(/\s*[;\n]\s*/g, "; ");
- return (q.length > 48 ? q.slice(0, 47) + "…" : q) + " (OQL)";
+ const q = oqlStatements(a.oql).join("; ");
+ return (q.length > 48 ? q.slice(0, 47) + "…" : q) + (a.by === "statement" ? " (OQL, by line)" : " (OQL)");
}
if (a.source === "alteration") {
const genes = alterationGenes(a);
const name = genes.length > 1 ? (genes.length <= 4 ? genes.join("/") + " (any)" : genes.length + " genes (any)") : genes[0];
- return name + (a.by === "cna" ? " copy number" : a.by === "mutationType" ? " mutation type" : " mutation status");
+ return name + (a.by === "cna" ? " copy number" : a.by === "sv" ? " structural variant" : a.by === "mutationType" ? " mutation type" : " mutation status");
}
return "ordered cases";
}
@@ -137,7 +164,8 @@ export function axisLabel(a: PlotAxis): string {
*/
export function axisKind(a: PlotAxis, clinicalKind?: (attr: string) => "num" | "cat" | undefined): AxisKind {
if (a.source === "none") return "none";
- if (a.source === "molecular") return "num";
+ if (a.source === "profile") throw new Error("a profile axis is read only once resolved against the study's profiles (lib/plot-profile.ts)");
+ if (a.source === "molecular") return a.discrete ? "cat" : "num";
if (a.source === "alteration") return "cat";
const k = clinicalKind ? clinicalKind(a.attribute) : undefined;
if (!k) throw new Error("clinical attribute '" + a.attribute + "' is not declared by this study, so its type is unknown");
@@ -150,6 +178,13 @@ export function validateAxis(a: PlotAxis, which: string): void {
throw new Error(which + " axis: unknown source '" + String(a.source).slice(0, 40) + "' — use one of " + AXIS_SOURCES.join(", "));
if (a.source === "clinical" && !ATTR_RE.test(String(a.attribute || "")))
throw new Error(which + " axis: '" + a.attribute + "' is not a clinical attribute name");
+ if (a.source === "profile") {
+ if (!PROFILE_RE.test(String(a.profile || ""))) throw new Error(which + " axis: '" + a.profile + "' is not a profile");
+ if (!GENE_RE.test(String(a.gene || "").toUpperCase()))
+ throw new Error(which + " axis: a profile axis reads one gene; name it");
+ if (a.by != null && a.by !== "mutationType" && a.by !== "mutated")
+ throw new Error(which + " axis: a mutation profile groups by \"mutationType\" or \"mutated\"; not " + JSON.stringify(a.by));
+ }
if (a.source === "molecular") {
if (!PROFILE_RE.test(String(a.profileType || ""))) throw new Error(which + " axis: '" + a.profileType + "' is not a profile type");
// A molecular axis without a gene would ask the database for every gene of every case —
@@ -158,10 +193,15 @@ export function validateAxis(a: PlotAxis, which: string): void {
throw new Error(which + " axis: a molecular axis needs one gene. Plotting a whole profile would read every gene of every case; name the gene you mean.");
}
if (isOqlAxis(a)) {
- if ("gene" in a || "genes" in a || "by" in a)
- throw new Error(which + " axis: an OQL axis is the query alone; drop gene, genes and by");
+ if ("gene" in a || "genes" in a)
+ throw new Error(which + " axis: an OQL axis is the query alone; drop gene and genes");
+ if (a.by != null && a.by !== "any" && a.by !== "statement")
+ throw new Error(which + " axis: an OQL axis groups by \"any\" (Altered or not) or \"statement\" (one group per line); not " + JSON.stringify(a.by));
try { parseOql(String(a.oql)); }
catch (e) { throw new Error(which + " axis: " + (e as Error).message, { cause: e }); }
+ const n = oqlStatements(String(a.oql)).length;
+ if (a.by === "statement" && n > OQL_STATEMENT_CAP)
+ throw new Error(which + " axis: one group per line takes at most " + OQL_STATEMENT_CAP + " lines; this query has " + n);
return;
}
if (a.source === "alteration") {
@@ -229,6 +269,21 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s
return "SELECT id, if(countIf(pref = 1) > 0, anyIf(v, pref = 1), " + combined + ") AS v FROM (\n" + own + "\n UNION ALL\n" + other + "\n) GROUP BY id";
}
+ if (a.source === "molecular" && a.discrete) {
+ // A discrete copy-number profile's own calls, one level per case, labelled as the portal labels
+ // them; a value outside the five levels is its own category, as written. In patient mode a
+ // patient whose samples disagree is "Multiple", the rule every categorical axis follows.
+ const gene = String(a.gene).toUpperCase();
+ const level = "multiIf(x = -2, 'Deep deletion', x = -1, 'Shallow deletion', x = 0, 'Diploid', x = 1, 'Gain', x = 2, 'Amplification', toString(x))";
+ return "SELECT id, if(uniqExact(lv) > 1, " + sq(MULTIPLE) + ", any(lv)) AS v FROM (\n" +
+ "SELECT " + strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study) + " AS id, " + level.replace(/\bx\b/g, "toFloat64OrNull(ga.alteration_value)") + " AS lv\n" +
+ "FROM " + db + ".genetic_alteration_derived ga\n" +
+ "JOIN " + db + ".sample_derived sd ON sd.sample_unique_id = ga.sample_unique_id\n" +
+ "WHERE ga.cancer_study_identifier = " + sq(study) + " AND ga.profile_type = " + sq(a.profileType) +
+ " AND ga.hugo_gene_symbol = " + sq(gene) + "\n" +
+ " AND toFloat64OrNull(ga.alteration_value) IS NOT NULL AND " + inCohort("sd.patient_unique_id") + ")\n" +
+ "GROUP BY id";
+ }
if (a.source === "molecular") {
const gene = String(a.gene).toUpperCase();
// patient mode averages a patient's samples, the same rule the oncoprint's heatmap uses; a
@@ -245,14 +300,30 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s
}
if (isOqlAxis(a)) {
- // every case in the cohort, so "unaltered" is counted rather than missing
- const matched = oqlCasesSql(a.oql, { oncokbPin: ctx.oncokbPin, profiles: ctx.profiles, study, db, unit: ctx.unit });
+ // Every case in the cohort, so "unaltered" is counted rather than missing. Each statement is
+ // matched on its own, and a case it did not match is "Unaltered" only when it is profiled for
+ // that statement's gene in every alteration type the statement asks about (panel-aware, as a
+ // cohort split decides it); a sample whose panel omits the gene is never counted as wild type.
+ // each statement names its group as typed, its gene in capitals
+ const statements = oqlStatements(a.oql).map((st) => st.replace(/^[A-Za-z0-9.-]+/, (g) => g.toUpperCase()).replace(/\s+/g, " "));
const idCol = strip(sample ? "sd.sample_unique_id" : "sd.patient_unique_id", study);
- return "SELECT " + idCol + " AS id, if(" + idCol + " IN (SELECT pid FROM (" + matched + ")), " +
- sq(OQL_ALTERED) + ", " + sq(OQL_UNALTERED) + ") AS v\n" +
- "FROM " + db + ".sample_derived sd\n" +
- "WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "\n" +
- "GROUP BY id";
+ const o = { oncokbPin: ctx.oncokbPin, profiles: ctx.profiles, study, db, unit: ctx.unit };
+ // the case column is `oql_case`, not `id`: ClickHouse would otherwise resolve `id` to another axis's alias
+ const matched = statements.map((st) => "oql_case IN (SELECT pid FROM (" + oqlCasesSql(st, o) + "))");
+ const profiled = statements.map((st) => {
+ const types = [...new Set(parseOql(st)[0].specs.flatMap((sp) => OQL_PROFILED_TYPES[sp.type] || []))];
+ const gene = parseOql(st)[0].gene;
+ // expression and protein thresholds read a value, not a panel: a case with no value is not matched, and not "not profiled"
+ return types.length ? types.map((t) => "oql_case IN (SELECT profiled_id FROM (" + profiledCasesSql(gene, t, { study, db, unit: ctx.unit }) + "))").join(" AND ") : "1";
+ });
+ const none = "if(" + profiled.map((p) => "(" + p + ")").join(" AND ") + ", " + sq(OQL_UNALTERED) + ", " + sq(OQL_NOT_PROFILED) + ")";
+ const value = a.by === "statement"
+ ? "multiIf(length(hit) = 0, " + none + ", length(hit) = 1, hit[1], length(hit) = 2, concat(hit[1], ' + ', hit[2]), " + sq(OQL_SEVERAL) + ")"
+ : "if(length(hit) > 0, " + sq(OQL_ALTERED) + ", " + none + ")";
+ return "SELECT oql_case AS id, " + value + " AS v FROM (\n" +
+ "SELECT oql_case, arrayFilter((l, m) -> m, [" + statements.map(sq).join(", ") + "], [" + matched.join(", ") + "]) AS hit FROM (\n" +
+ "SELECT DISTINCT " + idCol + " AS oql_case FROM " + db + ".sample_derived sd\n" +
+ "WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "))";
}
if (a.source === "alteration") {
const genes = alterationGenes(a), by = a.by || "mutated";
@@ -261,19 +332,29 @@ export function axisSelect(a: PlotAxis, ctx: AxisCtx, kind: AxisKind = "num"): s
const events = "SELECT sample_unique_id, mutation_type, variant_type, cna_alteration\n" +
" FROM " + db + ".genomic_event_derived\n" +
" WHERE cancer_study_identifier = " + sq(study) + " AND hugo_gene_symbol IN (" + genes.map(sq).join(",") + ")";
+ const hit = by === "cna" ? "abs(g.cna_alteration) >= 2" : by === "sv" ? "g.variant_type = 'structural_variant'" : "g.variant_type = 'mutation'";
const value = by === "cna"
? "if(countIf(g.cna_alteration >= 2) > 0, 'Amplification', if(countIf(g.cna_alteration <= -2) > 0, 'Deep deletion', 'Neutral'))"
+ : by === "sv"
+ ? "if(countIf(" + hit + ") > 0, 'Structural variant', 'No structural variant')"
: by === "mutationType"
// one class per case; a case carrying two of them is its own honest category
? "multiIf(countIf(g.variant_type = 'mutation') = 0, 'Not mutated'," +
" length(groupUniqArrayIf(" + MUT_CLASS + ", g.variant_type = 'mutation')) > 1, 'Multiple'," +
" arrayElement(groupUniqArrayIf(" + MUT_CLASS + ", g.variant_type = 'mutation'), 1))"
: "if(countIf(g.variant_type = 'mutation') > 0, 'Mutated', 'Not mutated')";
- return "SELECT " + idCol + " AS id, " + value + " AS v\n" +
+ // A case with no such alteration is "not mutated" (neutral, no SV) only when it is profiled for
+ // every gene of the axis in that alteration type; otherwise it is "Not profiled", never counted
+ // as wild type (the panel-aware rule). The case column is alt_case, not id, for the reason the
+ // OQL axis's is oql_case.
+ const type = by === "cna" ? "CNA" as const : by === "sv" ? "SV" as const : "MUTATION" as const;
+ const profiled = genes.map((g) => "alt_case IN (SELECT profiled_id FROM (" + profiledCasesSql(g, type, { study, db, unit: ctx.unit }) + "))").join(" AND ");
+ return "SELECT alt_case AS id, if(hits = 0 AND NOT (" + profiled + "), " + sq(OQL_NOT_PROFILED) + ", v0) AS v FROM (\n" +
+ "SELECT " + idCol + " AS alt_case, " + value + " AS v0, countIf(" + hit + ") AS hits\n" +
"FROM " + db + ".sample_derived sd\n" +
"LEFT JOIN (\n " + events + "\n) g ON g.sample_unique_id = sd.sample_unique_id\n" +
"WHERE sd.cancer_study_identifier = " + sq(study) + " AND " + inCohort("sd.patient_unique_id") + "\n" +
- "GROUP BY id";
+ "GROUP BY alt_case)";
}
throw new Error("axis source '" + (a as { source: string }).source + "' cannot be selected");
}
@@ -679,6 +760,10 @@ export interface PlotLayer {
n: number;
/** points drawn, once cases an earlier layer already drew are left to it */
drawn?: number;
+ /** the layer's cases an earlier layer drew: hidden on the figure, shown when the layer that drew them is hidden */
+ shared?: HighlightPoint[];
+ /** for each shared case, the place in the list of the layer that drew it */
+ sharedOwners?: number[];
shown: "points" | "density" | "boxes" | "none";
/** why it is not drawn as points, when it is not */
why?: string;
diff --git a/lib/profile-catalog.ts b/lib/profile-catalog.ts
new file mode 100644
index 00000000..c7a10adc
--- /dev/null
+++ b/lib/profile-catalog.ts
@@ -0,0 +1,62 @@
+// A study's molecular profiles as the study itself describes them (cBioPortal's genetic_profile):
+// each profile's alteration type, datatype and name. What a plot axis can do with a profile follows
+// from these, never from its name. Browser-safe, so the page's Data picker and the engine read the
+// same description.
+
+/** one profile of one study, its id without the study prefix (the derived tables' profile_type) */
+export interface ProfileInfo {
+ profile: string;
+ /** cBioPortal's genetic_alteration_type: MRNA_EXPRESSION, COPY_NUMBER_ALTERATION, GENERIC_ASSAY, … */
+ type: string;
+ /** cBioPortal's datatype: CONTINUOUS, Z-SCORE, LOG2-VALUE, DISCRETE, MAF, SV, CATEGORICAL, LIMIT-VALUE, … */
+ datatype: string;
+ name: string;
+ /** a generic assay's own kind: METHYLATION, PHOSPHOSITE_QUANTIFICATION, ARMLEVEL_CNA, … */
+ assayType?: string;
+ /** the study's own order among its profiles */
+ order: number;
+}
+
+/**
+ * What an axis does with a profile, from its type and datatype: a mutation (grouped by mutation
+ * type, or mutated or not), a structural variant (present or not), a discrete copy-number call
+ * (one category per level), a number per case (continuous, z-scores, log2), or, for a generic assay,
+ * a number or a category per case for one entity. null: not something a plot axis reads.
+ */
+export type ProfileUse = "mutation" | "sv" | "discrete" | "number" | "assayNumber" | "assayCategory";
+export function profileUse(p: Pick): ProfileUse | null {
+ const t = p.type.toUpperCase(), d = p.datatype.toUpperCase();
+ if (t === "MUTATION_EXTENDED") return d === "MAF" ? "mutation" : null;
+ if (t === "STRUCTURAL_VARIANT") return "sv";
+ if (t === "GENERIC_ASSAY") return d === "CATEGORICAL" || d === "BINARY" ? "assayCategory" : "assayNumber";
+ if (t === "COPY_NUMBER_ALTERATION" && d === "DISCRETE") return "discrete";
+ if (t === "COPY_NUMBER_ALTERATION" || t === "MRNA_EXPRESSION" || t === "PROTEIN_LEVEL" || t === "METHYLATION")
+ return d === "CONTINUOUS" || d === "Z-SCORE" || d === "LOG2-VALUE" || d === "LOG-VALUE" ? "number" : null;
+ return null;
+}
+
+/** the portal's order of data types in its Plots tab */
+const TYPE_ORDER = ["MUTATION_EXTENDED", "STRUCTURAL_VARIANT", "COPY_NUMBER_ALTERATION", "MRNA_EXPRESSION", "PROTEIN_LEVEL", "METHYLATION", "GENERIC_ASSAY"];
+/** cBioPortal's display names for its alteration types, as its Plots tab shows them */
+const TYPE_LABEL: Record = {
+ MUTATION_EXTENDED: "Mutation", STRUCTURAL_VARIANT: "Structural variant", COPY_NUMBER_ALTERATION: "Copy number",
+ MRNA_EXPRESSION: "mRNA", PROTEIN_LEVEL: "Protein level", METHYLATION: "DNA methylation",
+};
+/** cBioPortal's display names for the generic-assay kinds it knows; any other is shown as written */
+const ASSAY_LABEL: Record = {
+ METHYLATION: "DNA methylation", PHOSPHOSITE_QUANTIFICATION: "Phosphosite quantification", ARMLEVEL_CNA: "Arm-level copy number",
+ TREATMENT_RESPONSE: "Treatment response", MUTATIONAL_SIGNATURE: "Mutational signature", GENETIC_ANCESTRY: "Genetic ancestry",
+};
+
+/** the Data picker's group a profile sits under */
+export function profileGroup(p: Pick): string {
+ if (p.type === "GENERIC_ASSAY") return ASSAY_LABEL[p.assayType || ""] || (p.assayType || "Generic assay").replace(/_/g, " ").toLowerCase().replace(/^./, (c) => c.toUpperCase());
+ return TYPE_LABEL[p.type] || p.type;
+}
+
+/** the profiles a plot axis can read, in the portal's order: by type, then the study's own order */
+export function plottableProfiles(ps: ProfileInfo[]): ProfileInfo[] {
+ const rank = (p: ProfileInfo) => { const i = TYPE_ORDER.indexOf(p.type); return i < 0 ? TYPE_ORDER.length : i; };
+ return ps.filter((p) => profileUse(p)).sort((a, b) => rank(a) - rank(b) ||
+ profileGroup(a).localeCompare(profileGroup(b)) || a.order - b.order || a.name.localeCompare(b.name));
+}
diff --git a/lib/types.ts b/lib/types.ts
index 4dc954f8..c2cc6c27 100644
--- a/lib/types.ts
+++ b/lib/types.ts
@@ -549,7 +549,7 @@ export interface PlotView {
/** the grid a scatter is binned onto (default 240 x 160) */
bins?: [number, number];
/**
- * Every case is a point when there are at most 5,000 of them and the reader may see cases
+ * Every case is a point when there are at most POINTS_CAP (10,000) of them and the reader may see cases
* (default true); false draws the density alone at any size.
*/
allPoints?: boolean;
@@ -558,7 +558,7 @@ export interface PlotView {
* outliers (past the cap: a box's values beyond its whiskers, a scatter's sparse cells), the
* extremes on an axis, the cases furthest from the fitted line, or a cohort's cases. Default
* [{rule:"outliers"}]; [] draws none. A case in two layers is drawn by the first. Every layer
- * shares one 5,000-point budget; a cohort layer past it is drawn as density cells or boxes.
+ * shares one POINTS_CAP budget; a cohort layer past it is drawn as density cells or boxes.
*/
points?: PointLayerSpec[];
/**
@@ -1523,6 +1523,7 @@ export interface AlterationsResult extends TableRows {
}
import type { PlotAxis, PlotData } from "./plot.ts";
import type { PointLayerSpec } from "./plot-layers.ts";
+import type { ProfileInfo } from "./profile-catalog.ts";
export type { PlotAxis, PlotData };
export type View = CompareView | SurvivalView | OncoprintView | CooccurrenceView | ClinicalView | CoexpressionView | HotspotsView | CourseView | SequenceView | PlotView | CoxView | LogisticView
| AttritionView | GenomeView | PathwaysView | DiffexprView | PairedView | ClonalityView | IncidenceView | LabsView
@@ -2177,5 +2178,7 @@ export interface ExploreResult {
totals?: { patients: number; samples: number };
caseLists: { id: string; name: string; n: number }[];
profiles: string[]; gaProfiles: string[];
+ /** every profile as the studies describe them (genetic_profile), for a plot axis's Data picker */
+ profileCatalog?: ProfileInfo[];
stats: { ms: number; perStudy?: Record };
}
diff --git a/lib/views/cox.ts b/lib/views/cox.ts
index 6dbb83a8..4d4c16d8 100644
--- a/lib/views/cox.ts
+++ b/lib/views/cox.ts
@@ -6,7 +6,8 @@ import { isPatientLevel, type EndpointColumns } from "../federation.ts";
import { axisLabel, validateAxis, OQL_UNALTERED, type AxisKind } from "../plot.ts";
import { coxMethod, type CoxTrace } from "../method.ts";
import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts";
-import { overLimit } from "./shared.ts";
+import { overLimit, studyCatalogs } from "./shared.ts";
+import { resolveProfileAxis } from "../plot-profile.ts";
export interface CoxColumn { axisIndex: number; label: string; reference?: string; level?: string }
/**
@@ -46,6 +47,12 @@ export function buildCoxColumns(covariates: CoxCovariate[], kindsFinal: AxisKind
return columns;
}
+/** what a model's caption says about the patients it left out as not profiled for a covariate's gene */
+export function notProfiledNote(n: number): string {
+ return n > 0 ? " " + n.toLocaleString("en-US") + " patient" + (n === 1 ? " was" : "s were") + " not profiled for the gene of a covariate and " +
+ (n === 1 ? "was" : "were") + " left out: an unknown alteration status is not wild type, and not a level to fit." : "";
+}
+
export async function runCox(view: DocView, c: ViewRun): Promise {
const { doc, cohorts, resolvers, vNames, vStudies, participationFor } = c;
// Multivariate Cox: unlike survival/compare, this is fit from real per-patient rows
@@ -69,10 +76,15 @@ export async function runCox(view: DocView, c: ViewRun): Promise validateAxis(c, "covariate " + (i + 1))); }
+ try {
+ // a profile covariate is read as the studies describe the profile; its reference level stays
+ const catalogs = await studyCatalogs(c, vStudies);
+ covariates = covariates.map((cv, i) => ({ ...resolveProfileAxis(cv, "covariate " + (i + 1), catalogs), ...(cv.reference != null ? { reference: cv.reference } : {}) }));
+ covariates.forEach((cv, i) => validateAxis(cv, "covariate " + (i + 1)));
+ }
catch (e) { return untimed({ kind: "cox", endpoint, error: (e as Error).message }); }
const rawCutoff = cx.cutoffMonths;
let cutoffMonths: number | undefined;
@@ -88,7 +100,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise => {
const cohortTotal = names.reduce((n, cn) => n + cohorts[cn].count, 0);
@@ -98,7 +110,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise ({ label: col.label, beta: fit.beta[i], se: fit.se[i], p: fit.p[i] })),
};
- return { coefficients, n: fit.n, events: fit.events, termCount: columns.length, excludedRemote, patientLevelTotal, trace };
+ return { coefficients, n: fit.n, events: fit.events, termCount: columns.length, excludedRemote, patientLevelTotal, notProfiled, trace };
};
const coxMethodOptions = {
@@ -199,7 +212,7 @@ export async function runCox(view: DocView, c: ViewRun): Promise 0 ? " " + missing + " patient" + (missing === 1 ? "" : "s") + " from patient-level participants lacked the endpoint or a covariate and were excluded." : "") +
+ notProfiledNote(fit0.notProfiled) +
(cutoffMonths != null ? " Administratively censored at " + cutoffMonths + " months." : "") +
(cx.stratifyByStudy
? " Stratified by study: each study fits its own baseline hazard, so a case-mix difference between studies cannot masquerade as a covariate effect."
diff --git a/lib/views/logistic.ts b/lib/views/logistic.ts
index 4b959199..4d0ff87b 100644
--- a/lib/views/logistic.ts
+++ b/lib/views/logistic.ts
@@ -5,9 +5,10 @@ import type { ViewResult, LogisticView, LogisticCoefficient } from "../types.ts"
import { isPatientLevel } from "../federation.ts";
import { axisLabel, validateAxis, type AxisKind } from "../plot.ts";
import { logisticMethod, type CoxTrace } from "../method.ts";
-import { buildCoxColumns } from "./cox.ts";
+import { buildCoxColumns, notProfiledNote } from "./cox.ts";
import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts";
-import { overLimit } from "./shared.ts";
+import { overLimit, studyCatalogs } from "./shared.ts";
+import { resolveProfileAxis } from "../plot-profile.ts";
export async function runLogistic(view: DocView, c: ViewRun): Promise {
const { doc, cohorts, resolvers, vNames, vStudies, participationFor } = c;
@@ -15,8 +16,15 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise ({ ...resolveProfileAxis(cv, "covariate " + (i + 1), catalogs),
+ ...((cv as { reference?: string }).reference != null ? { reference: (cv as { reference?: string }).reference } : {}) })) } : {}) };
+ validateAxis(lv.outcome, "outcome");
+ }
catch (e) { return untimed({ kind: "logistic", error: (e as Error).message }); }
const outcomeLabel = axisLabel(lv.outcome);
// alteration/molecular axes are deterministically categorical/numeric without a DB round
@@ -45,7 +53,7 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise ({
label: col.label, ...(col.reference ? { reference: col.reference } : {}), ...(col.level ? { level: col.level } : {}),
axisIndex: col.axisIndex,
@@ -130,6 +139,7 @@ export async function runLogistic(view: DocView, c: ViewRun): Promise 0 ? " " + lMissing + " patient" + (lMissing === 1 ? "" : "s") + " from patient-level participants lacked the outcome or a covariate and were excluded." : "") +
+ notProfiledNote(lNotProfiled).replace("the gene of a covariate", "the gene of the outcome or a covariate") +
(vStudies.length > 1 ? " Clinical-attribute vocabularies and declared types are NOT harmonized across studies." : "") +
(lExcludedRemote.length
? " " + lExcludedRemote.map((e) => e.study + " (" + e.patients + " patients)").join(", ") +
diff --git a/lib/views/plot.ts b/lib/views/plot.ts
index e74e7561..7e8bc910 100644
--- a/lib/views/plot.ts
+++ b/lib/views/plot.ts
@@ -7,6 +7,7 @@ import { DEFAULT_BINS, CATEGORY_CAP, POINTS_CAP, OUTLIER_CELL_MAX, tukey, covera
type AxisKind, type HighlightPoint, type PlotData, type PlotGroup, type PlotLayer, type PlotPoint, type PlotRequest, type PointRank } from "../plot.ts";
import { DEFAULT_POINT_LAYERS, LAYER_COLORS, POINT_LAYER_CAP, POINT_RULE_N, POINT_RULE_N_MAX, layerLabel, type PointLayerSpec } from "../plot-layers.ts";
import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts";
+import { resolveAxes } from "../plot-profile.ts";
type BoxRow = { g: string; n: number; lo: number; hi: number; q1: number; median: number; q3: number };
@@ -31,6 +32,16 @@ function poolBoxRows(parts: BoxRow[][]): PlotGroup[] {
}
type Def = ViewRun["doc"]["cohorts"][string];
+
+/** copy-number levels in their own order, as the portal draws them; anything else keeps its place after them */
+const CNA_LEVELS = ["Deep deletion", "Shallow deletion", "Diploid", "Gain", "Amplification"];
+/** a box grouped by a discrete copy-number profile runs from deep deletion to amplification, not by size */
+function orderLevels(data: PlotData, spec: PlotRequest): void {
+ const cat = data.x.kind === "cat" ? spec.x : spec.y;
+ if (data.shape !== "box" || !data.groups || cat.source !== "molecular" || !cat.discrete) return;
+ const rank = (g: string) => { const i = CNA_LEVELS.indexOf(g); return i < 0 ? CNA_LEVELS.length : i; };
+ data.groups = [...data.groups].sort((a, b) => rank(a.g) - rank(b.g));
+}
const TABLE_WHY = "a contingency table counts cases and has no points to mark";
const fmt = (n: number) => n.toLocaleString("en-US");
@@ -121,7 +132,8 @@ async function pointLayers(pv: PlotView, data: PlotData, c: ViewRun, cn: string,
const def = c.doc.cohorts[cn];
const xk = data.x.kind, yk = data.y.kind;
const closed = sites.filter((s) => !isPatientLevel(c.resolvers[s]));
- const seen = new Set();
+ // each drawn case, and the layer (its place in the list) that drew it
+ const seen = new Map();
let budget = POINTS_CAP;
const out: PlotLayer[] = [];
for (const [i, l] of specs.entries()) {
@@ -136,8 +148,12 @@ async function pointLayers(pv: PlotView, data: PlotData, c: ViewRun, cn: string,
const { layer, keys } = picked;
if (layer.shown === "points" && layer.points && keys) {
const kept = layer.points.filter((_, j) => !seen.has(keys[j]));
- for (const k of keys) seen.add(k);
- layer.drawn = kept.length; layer.points = kept;
+ const sharedAt = keys.map((k, j) => seen.has(k) ? j : -1).filter((j) => j >= 0);
+ const owners = sharedAt.map((j) => seen.get(keys[j])!);
+ for (const k of keys) if (!seen.has(k)) seen.set(k, i);
+ layer.drawn = kept.length;
+ if (sharedAt.length) { layer.shared = sharedAt.map((j) => layer.points![j]); layer.sharedOwners = owners; }
+ layer.points = kept;
budget -= kept.length;
}
out.push(layer);
@@ -296,8 +312,14 @@ export async function runPlot(view: DocView, c: ViewRun): Promise ({ study: st, catalog: (await c.studyProfiles(st)).catalog || [] })));
+ axes = resolveAxes({ x: pv.x, y: pv.y, colorBy: pv.colorBy, ringBy: pv.ringBy }, catalogs);
+ } catch (e) { return untimed({ kind: "plot", error: String((e as Error).message || e) }); }
+ const spec = { x: axes.x, y: axes.y, unit,
+ ...(axes.colorBy ? { colorBy: axes.colorBy } : {}), ...(axes.ringBy ? { ringBy: axes.ringBy } : {}) };
const bins: [number, number] = Array.isArray(pv.bins) && pv.bins.length === 2
? [Number(pv.bins[0]), Number(pv.bins[1])] : DEFAULT_BINS;
const problem = layersProblem(pv, c, cn);
@@ -306,6 +328,7 @@ export async function runPlot(view: DocView, c: ViewRun): Promise fr[s].plotCoverage(doc.cohorts[cn], spec, xk, yk)));
diff --git a/lib/views/run.ts b/lib/views/run.ts
index c7ed4822..ef3e92b1 100644
--- a/lib/views/run.ts
+++ b/lib/views/run.ts
@@ -1,5 +1,6 @@
// The contract between runAnalysis and the view runners: the context each view gets, and the refusal
// that skips timing.
+import type { ProfileInfo } from "../profile-catalog.ts";
import type { AnalysisDoc, ViewResult, ViewParticipation } from "../types.ts";
import type { Resolver } from "../federation.ts";
import type { RiskFit, RiskFitArgs } from "./riskmodel.ts";
@@ -27,7 +28,7 @@ export interface ViewRun {
participationFor: (studies: string[], requiresPatientLevel: boolean) => ViewParticipation;
pin: string | null;
/** the molecular and generic-assay profiles a study holds (runAnalysis's cached inventory) */
- studyProfiles: (study: string) => Promise<{ profiles: string[]; gaProfiles: string[] }>;
+ studyProfiles: (study: string) => Promise<{ profiles: string[]; gaProfiles: string[]; catalog?: ProfileInfo[] }>;
/** a paged view's page (the mutation and clinical tables): rows offset to offset + limit of its one global order */
page?: { offset: number; limit: number };
/** a table view's column whose values to return instead of rows (what its filter offers) */
diff --git a/lib/views/shared.ts b/lib/views/shared.ts
index f81e7c38..e140ec32 100644
--- a/lib/views/shared.ts
+++ b/lib/views/shared.ts
@@ -4,6 +4,7 @@ import type { ViewResult } from "../types.ts";
import { hasViewAggregates, type Resolver, type ViewAggregateCapable } from "../federation.ts";
import { geneCytobandsSql } from "../federation-views.ts";
import type { ViewRun } from "./run.ts";
+import type { StudyCatalog } from "../plot-profile.ts";
/** 12 genes = 66 pairs; beyond this the multiplicity correction eats the signal */
export const COOCCUR_GENE_LIMIT = 12;
@@ -51,3 +52,8 @@ export async function cytobands(ok: Record, genes: string[
const rows = await first.viewMeta("cytobands", (s, d) => geneCytobandsSql(genes, s, d)).catch(() => []);
return new Map(rows.map((r) => [String(r.gene), String(r.cytoband || "")]));
}
+
+/** each study's profiles as it describes them, for resolving a profile axis (lib/plot-profile.ts) */
+export async function studyCatalogs(c: ViewRun, studies: string[]): Promise {
+ return Promise.all(studies.map(async (st) => ({ study: st, catalog: (await c.studyProfiles(st)).catalog || [] })));
+}
diff --git a/test/fixtures/example-counts.json b/test/fixtures/example-counts.json
index 31d70c4c..6de6cc0d 100644
--- a/test/fixtures/example-counts.json
+++ b/test/fixtures/example-counts.json
@@ -1,7 +1,7 @@
{
"note": "Expected cohort counts for every example in lib/dsl-reference.ts, asserted by npm run docs:check. Regenerate with `npm run docs:counts` ONLY after reviewing why a number moved.",
"oncokbPin": "v7.5",
- "generated": "2026-10-14",
+ "generated": "2026-10-20",
"examples": {
"Lung adenocarcinoma: a four-view workup": {
"EGFR-mutant NSCLC": 1893,
@@ -81,6 +81,9 @@
"OQL on a plot axis: STK11 or KEAP1 mutant lung cancer": {
"NSCLC": 7809
},
+ "Plot axes from the study's own profiles: ERBB2 copy number against expression": {
+ "all samples": 1084
+ },
"Merged gene rows: a pathway as one track": {
"Pancreatic": 3109
},
diff --git a/test/gene-query.test.ts b/test/gene-query.test.ts
index 44285b49..32f7be45 100644
--- a/test/gene-query.test.ts
+++ b/test/gene-query.test.ts
@@ -35,6 +35,9 @@ describe("gene query: OQL", () => {
test("the browser-safe parser is the one the engine uses", () => {
assert.equal(parseOql, parsePure);
});
+ test("the engine reads a comma between statements as the dialog does: a cohort's or an axis's OQL alike", () => {
+ assert.deepEqual(parseOql("KRAS: G12C, STK11: MUT;EGFR: AMP\nTP53").map((e) => e.gene), ["KRAS", "STK11", "EGFR", "TP53"]);
+ });
});
describe("gene query: gene list", () => {
diff --git a/test/plot-layers.test.ts b/test/plot-layers.test.ts
index 68011bdb..5fb0930b 100644
--- a/test/plot-layers.test.ts
+++ b/test/plot-layers.test.ts
@@ -10,6 +10,7 @@ import type { ViewRun, EvaluatedCohort } from "../lib/views/run.ts";
import type { PlotData, ViewResult } from "../lib/types.ts";
type Calls = string[];
+const CAP = POINTS_CAP.toLocaleString("en-US");
const EXTENT = { x0: 0, x1: 100, y0: 0, y1: 50 };
const scatter = (): PlotData => ({ shape: "scatter", n: 1000, x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" },
extent: { ...EXTENT }, grid: [240, 160], bins: [{ bx: 1, by: 1, n: 1000 }],
@@ -83,7 +84,7 @@ describe("a highlight on one study", () => {
assert.match(r.note!, /statistics are over the whole plotted cohort/);
});
- test("past the cap the same cases come back as cells on the plot's own grid and extent, never the first 5,000", async () => {
+ test("past the cap the same cases come back as cells on the plot's own grid and extent, never the first POINTS_CAP", async () => {
const calls: Calls = [];
const r = await run({ s: fake("patient-level", calls, { points: null }) });
const h = r.plot!.layers![0];
@@ -92,7 +93,7 @@ describe("a highlight on one study", () => {
assert.deepEqual(h.bins, [{ bx: 2, by: 3, n: 7 }]);
assert.equal(h.n, 7);
assert.ok(calls.includes("bins [240,160] " + JSON.stringify(EXTENT)), "the highlight is binned on the scatter's grid: " + calls.join("; "));
- assert.match(r.note!, /drawn as shaded cells \(more than 5,000 cases/);
+ assert.match(r.note!, new RegExp("drawn as shaded cells \\(more than " + CAP + " cases"));
});
test("an aggregate-only participant is never asked for points, and its highlight is an aggregate", async () => {
@@ -226,7 +227,7 @@ describe("rule layers", () => {
for (const extra of [{}, { allPoints: false }]) {
const r = await run({ s: fake("patient-level", []) }, { points: [{ rule: "outliers" }], extra });
assert.deepEqual([r.plot!.layers![0].shown, r.plot!.layers![0].why],
- ["none", "at or under 5,000 cases there is no density for them to stand out of"], JSON.stringify(extra));
+ ["none", "at or under " + CAP + " cases there is no density for them to stand out of"], JSON.stringify(extra));
}
});
diff --git a/test/plot-points.test.ts b/test/plot-points.test.ts
index 82c38c88..cfcb2715 100644
--- a/test/plot-points.test.ts
+++ b/test/plot-points.test.ts
@@ -9,6 +9,7 @@ import { POINTS_CAP, OUTLIER_CELL_MAX, sparseCellMax } from "../lib/plot.ts";
import type { ViewRun, EvaluatedCohort } from "../lib/views/run.ts";
import type { PlotData, ViewResult } from "../lib/types.ts";
+const CAP = POINTS_CAP.toLocaleString("en-US");
const EXTENT = { x0: 0, x1: 100, y0: 0, y1: 50 };
const scatter = (n: number, bins: { bx: number; by: number; n: number }[]): PlotData => ({
shape: "scatter", n, x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" },
@@ -83,7 +84,7 @@ describe("a scatter's points", () => {
assert.equal(calls.length, 1, "over the cap the every-case query is not run");
assert.deepEqual(calls[0].extra, { grid: [240, 160], extent: EXTENT, cellMax: OUTLIER_CELL_MAX });
assert.equal(calls[0].cap, POINTS_CAP);
- assert.deepEqual(r.plot!.pointSet, { rule: "none", drawn: 0, why: "more than 5,000 cases" });
+ assert.deepEqual(r.plot!.pointSet, { rule: "none", drawn: 0, why: "more than " + CAP + " cases" });
const l = r.plot!.layers![0];
assert.deepEqual([l.kind, l.shown, l.n, l.drawn, l.cellMax], ["outliers", "points", 1, 1, OUTLIER_CELL_MAX]);
assert.match(r.note!, /outliers: 1 sample in cells holding at most 3, left unshaded under them/);
@@ -98,7 +99,7 @@ describe("a scatter's points", () => {
assert.equal(r.plot!.pointSet!.rule, "none");
assert.equal(r.plot!.xyPoints, undefined);
assert.equal(r.plot!.layers![0].shown, "none");
- assert.match(r.note!, /outliers: not drawn, the cases alone in a cell are more than the 5,000 points left to draw/);
+ assert.match(r.note!, new RegExp("outliers: not drawn, the cases alone in a cell are more than the " + CAP + " points left to draw"));
});
test("an aggregate-only participant is never asked, and the caption names it", async () => {
@@ -139,11 +140,11 @@ describe("a box's points", () => {
assert.match(r.note!, /outliers: 1 value beyond the whiskers/);
});
- test("more outliers than the cap: none, never the first 5,000", async () => {
+ test("more outliers than the cap: none, never the first POINTS_CAP", async () => {
const calls: Call[] = [];
const r = await run(fake("patient-level", calls, () => box(POINTS_CAP * 3), { outliers: null }));
assert.equal(r.plot!.points, undefined);
assert.equal(r.plot!.layers![0].shown, "none");
- assert.match(r.note!, /outliers: not drawn, more than 5,000 values lie beyond the whiskers/);
+ assert.match(r.note!, new RegExp("outliers: not drawn, more than " + CAP + " values lie beyond the whiskers"));
});
});
diff --git a/test/plot-profile.test.ts b/test/plot-profile.test.ts
new file mode 100644
index 00000000..d37f410d
--- /dev/null
+++ b/test/plot-profile.test.ts
@@ -0,0 +1,84 @@
+// A profile axis: what the study's own description of a profile makes of it, how it is refused when
+// the study lacks it or two studies disagree about it, and the SQL each reading runs.
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import { profileUse, profileGroup, plottableProfiles, type ProfileInfo } from "../lib/profile-catalog.ts";
+import { resolveProfileAxis } from "../lib/plot-profile.ts";
+import { axisSelect, axisKind, axisLabel, validateAxis } from "../lib/plot.ts";
+
+const P = (profile: string, type: string, datatype: string, name: string, extra: Partial = {}): ProfileInfo =>
+ ({ profile, type, datatype, name, order: 0, ...extra });
+const BRCA: ProfileInfo[] = [
+ P("mutations", "MUTATION_EXTENDED", "MAF", "Mutations"),
+ P("structural_variants", "STRUCTURAL_VARIANT", "SV", "Structural variants"),
+ P("gistic", "COPY_NUMBER_ALTERATION", "DISCRETE", "Putative copy-number alterations from GISTIC"),
+ P("log2CNA", "COPY_NUMBER_ALTERATION", "LOG2-VALUE", "Log2 copy-number values"),
+ P("rna_seq_v2_mrna", "MRNA_EXPRESSION", "CONTINUOUS", "mRNA Expression, RSEM"),
+ P("rna_seq_v2_mrna_median_Zscores", "MRNA_EXPRESSION", "Z-SCORE", "mRNA expression z-scores relative to diploid samples"),
+ P("methylation_hm450", "GENERIC_ASSAY", "LIMIT-VALUE", "Methylation (HM450)", { assayType: "METHYLATION" }),
+ P("armlevel_cna", "GENERIC_ASSAY", "CATEGORICAL", "Putative arm-level copy-number from GISTIC", { assayType: "ARMLEVEL_CNA" }),
+];
+const one = [{ study: "brca", catalog: BRCA }];
+const ctx = { study: "x", db: "db", unit: "sample" as const, cohortSql: "SELECT pid FROM p" };
+
+describe("what a profile is, from the study's own description", () => {
+ test("type and datatype decide; the name never does", () => {
+ assert.deepEqual(BRCA.map((p) => profileUse(p)), ["mutation", "sv", "discrete", "number", "number", "number", "assayNumber", "assayCategory"]);
+ assert.equal(profileUse(P("rna_seq_v2_mrna", "MRNA_EXPRESSION", "DISCRETE", "x")), null, "a discrete mRNA profile is nothing an axis reads");
+ assert.equal(profileUse(P("cna", "COPY_NUMBER_ALTERATION", "DISCRETE", "named like anything")), "discrete");
+ });
+ test("grouped under cBioPortal's own type names, generic assays under their own kind, in the portal's order", () => {
+ assert.deepEqual(BRCA.map(profileGroup), ["Mutation", "Structural variant", "Copy number", "Copy number", "mRNA", "mRNA", "DNA methylation", "Arm-level copy number"]);
+ assert.equal(profileGroup({ type: "GENERIC_ASSAY", assayType: "SOMETHING_NEW" }), "Something new", "an unknown kind is shown as written");
+ const shuffled = [...BRCA].reverse();
+ assert.deepEqual(plottableProfiles(shuffled).map((p) => p.type).slice(0, 4), ["MUTATION_EXTENDED", "STRUCTURAL_VARIANT", "COPY_NUMBER_ALTERATION", "COPY_NUMBER_ALTERATION"]);
+ });
+});
+
+describe("a profile axis resolved against the studies it is drawn over", () => {
+ test("each reading resolves to the form that runs it", () => {
+ assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "mutations", gene: "tp53" }, "x", one), { source: "alteration", gene: "TP53", by: "mutationType" });
+ assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "mutations", gene: "TP53", by: "mutated" }, "x", one), { source: "alteration", gene: "TP53", by: "mutated" });
+ assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "structural_variants", gene: "ERBB2" }, "x", one), { source: "alteration", gene: "ERBB2", by: "sv" });
+ assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2" }, "x", one),
+ { source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true, label: "ERBB2: Putative copy-number alterations from GISTIC" });
+ assert.deepEqual(resolveProfileAxis({ source: "profile", profile: "rna_seq_v2_mrna", gene: "ESR1", log: true }, "y", one),
+ { source: "molecular", profileType: "rna_seq_v2_mrna", gene: "ESR1", log: true, label: "ESR1: mRNA Expression, RSEM" });
+ const clin = { source: "clinical" as const, attribute: "AGE" };
+ assert.equal(resolveProfileAxis(clin, "x", one), clin, "any other axis passes through untouched");
+ });
+ test("refused by name: a profile the study lacks (saying what it has), one two studies describe differently, and settings it cannot take", () => {
+ assert.throws(() => resolveProfileAxis({ source: "profile", profile: "rppa", gene: "ESR1" }, "x", one), /^Error: x axis: brca has no profile 'rppa'; it has mutations, structural_variants, log2CNA, gistic, rna_seq_v2_mrna_median_Zscores/);
+ const other = [{ study: "brca", catalog: BRCA }, { study: "panel", catalog: [P("gistic", "COPY_NUMBER_ALTERATION", "LOG2-VALUE", "other")] }];
+ assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2" }, "x", other), /described differently by brca \(COPY_NUMBER_ALTERATION, DISCRETE\) and panel/);
+ assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2", by: "mutated" }, "x", one), /only a mutation profile groups by/);
+ assert.throws(() => resolveProfileAxis({ source: "profile", profile: "gistic", gene: "ERBB2", log: true }, "x", one), /a log scale applies to numbers/);
+ assert.throws(() => resolveProfileAxis({ source: "profile", profile: "methylation_hm450", gene: "ERBB2" }, "x", one), /is a generic assay \(Methylation \(HM450\)\), which a plot axis does not read yet/);
+ });
+ test("a profile axis is shaped before it is read, and only once resolved does it have a kind", () => {
+ assert.throws(() => validateAxis({ source: "profile", profile: "", gene: "TP53" }, "x"), /is not a profile/);
+ assert.throws(() => validateAxis({ source: "profile", profile: "mutations", gene: "" }, "x"), /reads one gene/);
+ assert.throws(() => axisKind({ source: "profile", profile: "mutations", gene: "TP53" }), /only once resolved/);
+ assert.equal(axisKind({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true }), "cat");
+ assert.equal(axisLabel({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true, label: "ERBB2: GISTIC" }), "ERBB2: GISTIC");
+ });
+});
+
+describe("the SQL each reading runs", () => {
+ test("a discrete profile's own levels, the portal's names, and Multiple for a patient whose samples disagree", () => {
+ const sql = axisSelect({ source: "molecular", profileType: "gistic", gene: "ERBB2", discrete: true }, ctx, "cat");
+ assert.match(sql, /= -2, 'Deep deletion',[\s\S]*?= -1, 'Shallow deletion',[\s\S]*?= 0, 'Diploid',[\s\S]*?= 1, 'Gain',[\s\S]*?= 2, 'Amplification', toString\(/);
+ assert.match(sql, /ga\.profile_type = 'gistic'/);
+ assert.match(sql, /if\(uniqExact\(lv\) > 1, 'Multiple', any\(lv\)\)/);
+ });
+ test("a gene's mutations, copy number or SVs: no such event is the wild-type answer only when the case is profiled for it", () => {
+ for (const [by, type, label] of [["mutated", "MUTATION_EXTENDED", "Not mutated"], ["cna", "COPY_NUMBER_ALTERATION", "Neutral"], ["sv", "STRUCTURAL_VARIANT", "No structural variant"]] as const) {
+ const sql = axisSelect({ source: "alteration", gene: "ERBB2", by }, ctx, "cat");
+ assert.match(sql, new RegExp("if\\(hits = 0 AND NOT \\(alt_case IN \\(SELECT profiled_id"), by);
+ assert.match(sql, new RegExp("alteration_type = '" + type + "'"), by);
+ assert.match(sql, new RegExp("'" + label + "'"), by);
+ assert.match(sql, /'Not profiled', v0\) AS v/, by);
+ }
+ assert.match(axisSelect({ source: "alteration", gene: "ERBB2", by: "sv" }, ctx, "cat"), /g\.variant_type = 'structural_variant'/);
+ });
+});
diff --git a/test/plot.test.ts b/test/plot.test.ts
index 8bd7d4e8..427b8249 100644
--- a/test/plot.test.ts
+++ b/test/plot.test.ts
@@ -37,7 +37,7 @@ test("an axis that would read a whole profile is refused by name", () => {
/needs one gene.*name the gene you mean/s);
assert.throws(() => validateAxis({ source: "alteration" } as never, "y"), /needs a gene, a gene set, or an OQL query/);
assert.throws(() => validateAxis({ source: "alateration", gene: "APC" } as never, "covariate 1"),
- /unknown source 'alateration' — use one of clinical, molecular, alteration, none/);
+ /unknown source 'alateration' — use one of clinical, profile, molecular, alteration, none/);
assert.throws(() => validateAxis({ source: "clinical", attribute: "not a name!" }, "x"), /not a clinical attribute name/);
assert.throws(() => validateAxis({ source: "alteration", gene: "TP53", by: "driver" } as never, "x"), /not supported yet/);
validateAxis(num, "x"); // the ordinary cases do not throw
@@ -212,7 +212,7 @@ test("log: true reads log10(v + 1), leaves out what has no such log, and says so
assert.equal(axisLabel({ source: "molecular", profileType: "mrna", gene: "ESR1", log: true }), "log10(ESR1 mrna + 1)");
});
-test("an OQL axis: each case is Altered when it matches any entry, Unaltered otherwise", () => {
+test("an OQL axis: Altered when a case matches any statement, Unaltered only when it is profiled for every one", () => {
const oql = { source: "alteration" as const, oql: "KRAS: G12C; STK11: MUT" };
validateAxis(oql, "x");
assert.equal(axisKind(oql), "cat");
@@ -220,9 +220,26 @@ test("an OQL axis: each case is Altered when it matches any entry, Unaltered oth
const sql = axisSelect(oql, ctx, "cat");
// every case of the cohort is a row, so the unaltered are counted, not missing
assert.match(sql, /FROM db\.sample_derived sd/);
- assert.match(sql, /'Altered', 'Unaltered'\) AS v/);
+ assert.match(sql, /if\(length\(hit\) > 0, 'Altered', if\(/);
+ assert.match(sql, /, 'Unaltered', 'Not profiled'\)\) AS v/);
assert.match(sql, /hugo_gene_symbol = 'KRAS'/);
assert.match(sql, /hugo_gene_symbol = 'STK11'/);
+ // panel-aware: a sample whose panel omits the gene is not profiled for it, never wild type
+ assert.match(sql, /sample_to_gene_panel_derived sp[\s\S]*alteration_type = 'MUTATION_EXTENDED'[\s\S]*WHERE gene = 'KRAS'/);
+ assert.match(sql, /WHERE gene = 'STK11'/);
+});
+
+test("an OQL axis by statement: a group per line, a pair for two, '3 or more lines' past that; commas separate lines", () => {
+ const oql = { source: "alteration" as const, oql: "kras: G12C, STK11: MUT\nKEAP1", by: "statement" as const };
+ validateAxis(oql, "x");
+ assert.equal(axisLabel(oql), "kras: G12C; STK11: MUT; KEAP1 (OQL, by line)");
+ const sql = axisSelect(oql, ctx, "cat");
+ assert.match(sql, /\['KRAS: G12C', 'STK11: MUT', 'KEAP1'\]/, "each group is named by its statement, the gene in capitals");
+ assert.match(sql, /length\(hit\) = 2, concat\(hit\[1\], ' \+ ', hit\[2\]\), '3 or more lines'\)/);
+ // a bare gene is any alteration, so it must be profiled for all three kinds before "Unaltered"
+ assert.match(sql, /'COPY_NUMBER_ALTERATION'[\s\S]*WHERE gene = 'KEAP1'/);
+ assert.throws(() => validateAxis({ source: "alteration", oql: "A;B;C;D;E;F;G", by: "statement" }, "x"), /at most 6 lines; this query has 7/);
+ assert.throws(() => validateAxis({ source: "alteration", oql: "KRAS", by: "cna" } as never, "x"), /groups by "any" .* or "statement"/);
});
test("an OQL axis matches samples in sample mode and patients in patient mode", () => {
diff --git a/test/view-svg.test.ts b/test/view-svg.test.ts
index c960905d..d9248b71 100644
--- a/test/view-svg.test.ts
+++ b/test/view-svg.test.ts
@@ -404,6 +404,18 @@ describe("plot point layers: subsets drawn over the plotted cohort", () => {
assert.match(svg, /1 value of a point layer lies more than a whisker span past the/);
});
+ test("each layer is a group tagged with its place in the list, its shared cases hidden groups by the layer that drew them", () => {
+ const svg = viewToSvg(scatter([
+ hl({ points: [{ id: "a", x: 10, y: 5 }] }),
+ hl({ name: "S", color: LAYER_COLORS[1], points: [{ id: "b", x: 20, y: 8 }], shared: [{ id: "a", x: 10, y: 5 }], sharedOwners: [0] }),
+ ]))!;
+ assert.match(svg, //);
+ assert.match(svg, /]*fill="#0072B2"/);
+ assert.match(svg, / {
const v: ViewResult = { kind: "plot", plot: {
shape: "box", n: 3, x: { label: "G", kind: "cat" }, y: { label: "V", kind: "num" },