From b877e49a7c1100b710673a4aa3bd621a5a04cd92 Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Tue, 13 Oct 2026 18:24:34 +0000 Subject: [PATCH 1/5] Plot points: every case up to the cap, the outliers past it A plot now draws its cases as points the way cBioPortal's Plots tab does, without shipping a quarter of a million of them. At or under POINTS_CAP (5,000) every case is a point, on a scatter as well as a box. Past the cap the bulk stays density and the outliers are points: - box: the values beyond each group's Tukey whiskers, drawn over a violin that now stops at the whiskers (plotPointsSql's `outside`, whiskers passed as literals) - scatter: the cases in sparse grid cells, i.e. every cell holding at most k cases, where k is the largest value up to OUTLIER_CELL_MAX (3) whose points fit under the cap (sparseCellMax). They are binned with the same expression and extent as the density (binIndex), so those cells are left unshaded and no case is drawn twice. If a subset is itself over the cap, none of it is drawn, never the first 5,000. PlotData.pointSet records the rule, how many points were drawn, and why there are none when there are none; the caption says the same in words. Only PatientLevelResolver returns points (plotPoints, plus the new plotScatterPoints). An aggregate-only participant gets density, and the caption names it. Box axis: points beyond the whiskers used to spill outside the frame. The axis now covers the points, but extends at most one whisker span past the whiskers, so one TMB of 570 can't flatten every box. Points further out are drawn as triangles at the frame edge and counted in the caption. Explore's 48x32 density passes points:false. docs/plot-case-level.md is rewritten to this model (density for the bulk, a chosen subset as points) with the later phases; the DSL reference and docs/dsl.md are regenerated. Verified on msk_chord_2024: - TP53-mut scatter (13,087 samples): 1,224 sparse-cell points, equal to the bins' own count of cases in cells <=3, and none lands outside a sparse cell when re-binned client-side. - TP53-mut box: 77 outliers, matching an independent count over all 13,087 values. - KRAS G12C (1,116): every case drawn. - /lab renders all four with no console errors. Gate: build, unit, db, portal counts, cohort logic, method scripts, Python, and e2e all pass except "the upload example matches the explored study in full", which fails the same way on master's own CI run (job 115375037223) and touches no plot code. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/explore-charts.tsx | 2 +- docs/dsl.md | 40 +++++--- docs/plot-case-level.md | 188 ++++++++++++++++-------------------- lib/dsl-reference.ts | 44 +++++---- lib/federation.ts | 46 ++++++--- lib/figures/plot.ts | 57 ++++++++--- lib/plot.ts | 110 ++++++++++++++++++--- lib/views/plot.ts | 109 ++++++++++++++++----- test/boundary.test.ts | 9 +- test/plot-highlight.test.ts | 4 + test/plot-points.test.ts | 143 +++++++++++++++++++++++++++ test/plot.test.ts | 32 +++++- test/view-svg.test.ts | 40 ++++++++ 13 files changed, 619 insertions(+), 205 deletions(-) create mode 100644 test/plot-points.test.ts diff --git a/app/lab/explore-charts.tsx b/app/lab/explore-charts.tsx index 921f0094..1dfc6e49 100644 --- a/app/lab/explore-charts.tsx +++ b/app/lab/explore-charts.tsx @@ -47,7 +47,7 @@ export function ExploreCharts({ studies, draft, attrs, endpoints, addClauses, km const cohort: CohortDef = { ...(studies.length > 1 ? { studies } : {}), all: (draft ? draft.all : []) as CohortDef["all"] }; const views: View[] = []; if (ep) views.push({ kind: "survival", endpoint: ep }); - if (hasScatter) views.push({ kind: "plot", unit: "sample", x: { source: "clinical", attribute: X_ATTR, log: true }, y: { source: "clinical", attribute: Y_ATTR }, bins: [48, 32] }); + if (hasScatter) views.push({ kind: "plot", unit: "sample", x: { source: "clinical", attribute: X_ATTR, log: true }, y: { source: "clinical", attribute: Y_ATTR }, bins: [48, 32], points: false }); // a superseded request is cancelled, so the server stops its queries instead of finishing them const ac = new AbortController(); api.analysis({ study: studies[0], cohorts: { Selected: cohort }, views }, { signal: ac.signal }) diff --git a/docs/dsl.md b/docs/dsl.md index 1637396c..6403c9e2 100644 --- a/docs/dsl.md +++ b/docs/dsl.md @@ -275,7 +275,7 @@ patient — so the arm counts summing to more than the base cohort is expected, | `logistic` | `outcome:axis, outcomeEvent?:"", covariates:[axis, ...(max 8)], cohorts:[names]` | multivariate logistic regression: a BINARY outcome (not time-to-event — no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as `cox`'s. `outcome` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs `outcomeEvent` naming which observed value counts as the event (1) — every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given `outcomeEvent`. Covariates use the exact same dummy-coding and reference-level rules as `cox` (see above): a mutated-vs-not axis defaults to `"Not mutated"`, others default to the most frequent level, `reference` overrides either. Same patient-level-only boundary as `cox` | | `course` | `eventTypes:[≤6], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin — first recorded event to last contact (the later of their last event and their OS follow-up) — never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) `anchor` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED — read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (`Treatment`/`TREATMENT`) and the fold is named in the caveat; nothing else is merged | | `sequence` | `labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES — therapies less than `lineGap` days apart are one line, labelled by their whole combination (sorted, so `A + B` and `B + A` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one `lineGap` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days — so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, `lineClasses` `"all"` for every treatment record or a list of class names for your own; `labelKey` only names a line’s therapies (e.g. AGENT), it never decides which records form lines. `exclude` drops records by label value, matched ignoring case. More than two cohorts is refused | -| `plot` | `x:axis, y:axis, unit?:"patient"\|"sample", bins?:[240,160], points?:true, colorBy?:axis, ringBy?:axis, highlight?:[names(max 4)], cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size | +| `plot` | `x:axis, y:axis, unit?:"patient"\|"sample", bins?:[240,160], points?:true, colorBy?:axis, ringBy?:axis, highlight?:[names(max 4)], cohorts:[names]` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. `points` (default true): every case is a point up to 5,000, and past that the outliers are (a box's values beyond its whiskers, a scatter's cases in sparse cells), from a participant that may show cases | | `attrition` | `cohorts:[names] (≤6)` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its `all`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated | | `genome` | `binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ `threshold`) or lost (≤ −`threshold`) in every `binMb`-megabase bin (1–50) along chromosomes 1–22 (and X with `includeX` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out | | `pathways` | `pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"|"any", topGenes?:6, cohorts:[names] (≤4)` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own `sets`, each as the share of the cohort with at least one member altered. `measure` defaults to `"driver"` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or `"any"`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as `compare`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The `topGenes` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named | @@ -371,19 +371,27 @@ overlay a single-study box can fall back to (that needs the same shared-scale bi uses, not yet built for a box); its quartiles are a size-weighted approximation across sites, the same one Explore's federated numeric histograms already disclose. -**What sits behind the boxes.** cBioPortal draws every value as a jittered point over its box. -So does this, up to 5,000 values, and only when the study is one this participant may see cases -in — the points carry case ids, so they come from the same entitlement that lets a -patient-level participant list cohort members, and an aggregate-only participant has no method -that could return them. The jitter is fixed per case, so the figure does not move between the -server SVG, the browser and the Python package. - -Above the cap, or with `points: false`, or from a participant that may not disclose a case, the -same values are drawn as a **violin** from per-group bins: a quarter of a million jittered dots -is a solid block, while the density is 19KB and is what the reader was looking for. Each violin -is scaled to its own densest bin, because one group holding 178,489 of the values would -otherwise flatten every other group to nothing; how many cases a group has is printed under its -name. The caption always says which of the two it drew. +**Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every +case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the +study is one this participant may see cases in — the points carry case ids, so they come from the +same entitlement that lets a patient-level participant list cohort members, and an aggregate-only +participant has no method that could return them. The jitter across a box is fixed per case, so +the figure does not move between the server SVG, the browser and the Python package. + +Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a +hover on it would land on an arbitrary case — and the **outliers** stay points: on a box, the +values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops +at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at +most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of +as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure +draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With +`points: false`, or from a participant that may not disclose a case, the figure is density alone. +The caption always says which it drew, how many points, and why there are none when there are +none. Each violin is scaled to its own densest bin, because one group holding 178,489 of the +values would otherwise flatten every other group to nothing; how many cases a group has is +printed under its name. A box's axis spans the points it draws but reaches at most one whisker +span past the whiskers, so one extreme value cannot flatten every box; a point beyond that is a +triangle at the frame's edge, counted in the caption. **Colour.** `colorBy` fills each value with the colour of a category and `ringBy` rings it, the portal's "Color samples by" with its two checkboxes — so a case that is amplified *and* mutated @@ -416,7 +424,7 @@ aggregate on the plot's own scale — cells on the scatter's grid, or a narrow b caption says which and why. A highlight is never cut to its first 5,000 cases. Highlighting changes no statistic: the coefficients and quartiles stay those of the whole plotted cohort. A contingency table has no points to mark, so its highlights are counted but not drawn. On a box plot a -highlighted value beyond the whiskers is pinned to the frame's edge as a triangle and counted. +highlighted value beyond the axis is pinned to the frame's edge as a triangle and counted. A case with no value on an axis is simply absent, so a pair is an inner join: only cases measured on both axes are counted, and the figure reports how many that was. @@ -1865,7 +1873,7 @@ ERBB2-amplified TCGA breast tumors with an mRNA z-score heatmap under the gene t ### Example 16: A plot of 13,000 samples, drawn as density -Tumour mutational burden against fraction of the genome altered, over every sample of a TP53-mutant cohort. The figure is a density field: the samples are counted into a 240x160 grid, and only the occupied cells travel. Pearson, Spearman and the fitted line are computed over EVERY sample inside ClickHouse, so they describe the whole cohort rather than the points that happened to be painted last. The second view groups the same measure by TP53 mutation class, as quartiles with Tukey whiskers. +Tumour mutational burden against fraction of the genome altered, over every sample of a TP53-mutant cohort. The figure is a density field: the samples are counted into a 240x160 grid, and only the occupied cells travel, plus the outliers — the cases in the sparse cells at the fringe, drawn as points. Pearson, Spearman and the fitted line are computed over EVERY sample inside ClickHouse, so they describe the whole cohort rather than the points that happened to be painted last. The second view groups the same measure by TP53 mutation class, as quartiles with Tukey whiskers over a violin, with the values beyond the whiskers drawn as points. ```json { diff --git a/docs/plot-case-level.md b/docs/plot-case-level.md index d06f15a4..d5b667c0 100644 --- a/docs/plot-case-level.md +++ b/docs/plot-case-level.md @@ -1,125 +1,99 @@ -# Plot view: case-level colouring (gene alterations, clinical values) +# Plot view: case-level points, colouring and interaction -Status: **proposed, not started.** Written so a later session can pick it up cold. Read the -whole file, then `docs/DATA-BOUNDARY.md` (this plan leans on it throughout), then start at -"Phases" step 1. +Status: **phase 1 in progress** (`session/plots`). Written so a later session can pick it up +cold. Read the whole file, then `docs/DATA-BOUNDARY.md` (this plan leans on it throughout). ## The gap -cBioPortal's results-view **Plots tab** colours each case. Its rules are in -`src/shared/components/plots/PlotsTabUtils.tsx`, `makeScatterPlotPointAppearance`, in the -reference clone at `~/cbioportal-frontend`: +cBioPortal's results-view **Plots tab** draws every case as a point and makes each one +interactive. Its rules are in `src/shared/components/plots/PlotsTabUtils.tsx` +(`makeScatterPlotPointAppearance`) in the reference clone at `~/cbioportal-frontend`: - **Fill = mutation type** of the case's mutation in the chosen gene, using the oncoprint's - palette. With driver annotation on, drivers and VUS get separate shades - (`oncoprintMutationTypeToAppearanceDrivers`). -- **Border = copy number or structural variant**: amplification red, deep deletion blue - (`CNA_COLOR_AMP` / `CNA_COLOR_HOMDEL`), SV its own colour (`getCnaAndSvAppearance`). -- **"Not profiled"** has its own look (`notProfiledMutationsAppearance`), distinct from "not - mutated". This is the panel-aware distinction this repo already insists on elsewhere. -- **Clinical colouring** (`ColoringType.ClinicalData`) replaces all of the above. -- Every point has a hover tooltip naming the sample, and cases can be searched and highlighted. -- The portal fetches every case to the browser and, for big scatters, merges nearby points - with a quadtree (`downsampleByGrouping.ts`). + palette. With driver annotation on, drivers and VUS get separate shades. +- **Border = copy number or structural variant**: amplification red, deep deletion blue, SV its + own colour. SV takes precedence over CNA. +- **"Not profiled"** has its own look, distinct from "not mutated". +- **Clinical colouring** replaces all of the above (categorical, or a numeric gradient). +- Every point has a hover tooltip (sample link, both values, mutations, CNA, SV), cases can be + searched by id or protein change and highlighted, a legend entry click highlights its class, + and a brush on the scatter selects cases for the study view. +- Everything is Victory SVG, one DOM node per point, with no cap and no downsampling. Our plot view (`lib/plot.ts`, `lib/federation.ts` `plot()`/`plotPoints()`, `plotSvg` in -`lib/view-svg.ts`) is aggregate-first: - -- A scatter is a density grid, a box is quartiles plus a violin, a table is counts. -- Colour and ring exist **only on box plots**. -- Case-level points exist only as the box plot's overlay: `PatientLevelResolver.plotPoints`, - capped at `POINTS_CAP` = 5,000. - -So a scatter can't be coloured by gene alteration at all. - -## Constraints that shape the design - -- **The patient-data boundary** (`docs/DATA-BOUNDARY.md`, invariant 10). Only a patient-level - participant can return case rows. `plotPoints` lives on `PatientLevelResolver` alone, and - `isPatientLevel` is the type guard; this is the "missing method, not a flag" rule. An - aggregate-only or remote participant must still get a useful, honest answer. -- **Nothing is dropped without saying so** (`CLAUDE.md`, UI conventions). If points are merged, - or the plot falls back to density, the caption says so and says why. A legend lists every - category it colours, or pools the rest into a disclosed "other" (`legendFor`, `COLOR_CAP`). -- **Panel-aware profiling.** "Not profiled" is a category of its own, never folded into "not - mutated". Reuse the oncoprint's coverage logic; don't re-derive it. -- **One renderer per figure.** Chat and MCP need a static image, so the server SVG stays. An - interactive client renderer is added *next to* it, the way the oncoprint has both. The - interactive version is not a replacement. -- **The document holds what changes the picture** (invariant 9). A colour choice goes in the - view (`colorBy`); hover and highlighted cases stay in browser state. - -## Design: two tiers - -### Tier 1: case-level, for entitled participants - -When every site in the plot is patient-level (the common single-study case): - -1. **One row per case**, from a generalised `plotPoints` that covers scatter and table as well - as box. Each row carries: - - `id` (sample or patient id; allowed here because the participant is entitled) - - `x` and `y` (numbers, or categories for a categorical axis) - - `fill`: the mutation class - - `stroke`: the CNA/SV class - - optionally a clinical colour value -2. **Colour by gene**, as a new colour source mirroring the portal's `ColoringType`: - `colorBy: { source: "gene", gene, show?: ["mutation", "cna", "sv"] }`. Computed per case in - SQL, in one pass beside the axis selects: - - `fill`: the case's highest-priority mutation in the gene, using the oncoprint's mutation - priority (`lib/oncoprint-style.ts`, so nothing forks), split driver/VUS by the pinned - OncoKB, as the oncoprint and the lollipop do. - - `stroke`: amplification / deep deletion / SV / none. - - Not profiled for the gene (panel coverage) is its own class. - - Clinical `colorBy` stays as it is and replaces gene colouring, as on the portal. -3. **Payload and the cap.** About 30 bytes a case gives roughly 0.75 MB for MSK-CHORD's 25k - samples, which is fine. GENIE's 290k would be about 9 MB, which is not. Raise the case cap - (start at ~50k and measure). Above the cap, or on any aggregate-only site, the plot uses - tier 2 and says so. Don't copy the portal's silent quadtree merge. If merging is ever - added, each merged glyph must state how many cases it stands for. -4. **Rendering.** Add a client `PlotFigure` on a ``, like `OncoprintFigure`: points by - fill and stroke, hover showing the sample id, the axis values and the alterations, - search-to-highlight, and a legend of the classes actually present. `plotSvg` keeps drawing - the same points for chat and MCP (SVG up to a few thousand, density above that, with a - caption). - -### Tier 2: aggregates, for everyone else - -For federated, aggregate-only or over-cap plots, colour still works, as **counts per (bin, -category)**. That's an aggregate, so any participant can answer it, and sites pool by summing -against the shared extent the federated scatter already negotiates (`plotExtent` / -`plotBinCounts`): - -- **Scatter:** each density cell is coloured by its commonest class. Its mix shows on hover, - and the caption states the share of cases in mixed cells. -- **Box:** unchanged. The violin already splits by colour class. -- **Table:** each cell shows its class split as a small stacked bar. -- The site applies the suppression floor before returning (`minCount`, as `lollipopCte` does) - wherever a (bin, category) count could single someone out. +`lib/view-svg.ts`) is aggregate-first: a scatter is a density grid, a box is quartiles plus a +violin, a table is counts. Case-level points existed only as the box plot's overlay below +`POINTS_CAP` (5,000) and as highlight cohorts. + +## The model: every point when it is a drawing, the cases that matter when it is not + +Past a few thousand cases a scatter is a density picture whatever renders it: points overlap and +a hover lands on an arbitrary one. Shipping and drawing every case (the "WebGL, a million +points" design) buys interaction with cases nobody can tell apart. So: + +- **At or under `POINTS_CAP`, every case is a point**, on a scatter as on a box — the portal's + picture, and what most single studies get. +- **Over the cap, the crowd is density and a chosen subset is points.** The subset is decided + in the database and is always small enough to draw (and to send with ids inline, so no + binary columns, no lazy tooltip fetch, no client spatial index): + 1. **Outliers.** On a box, the values beyond each group's Tukey whiskers. On a scatter, the + cases in *sparse* cells of the density grid: every cell holding at most `k` cases is + replaced by its points, `k` the largest value up to `OUTLIER_CELL_MAX` whose points fit + under the cap (`sparseCellMax`). Dense regions stay density; the fringe becomes cases. + This falls out of the bin query the scatter already runs. + 2. **Matches** (phase 3): cases matching a search (case id, protein change) or a highlight — + the reader asked for them, so they are always points. + 3. **Rare colour classes** (phase 2): with colour by gene, the mutated / amplified minority + drawn as points over a grey density of the rest — the reading the portal's colouring + exists for ("where do the mutated cases sit?"). + 4. **Drill-down** (phase 4): a brushed region or a clicked cell returns its cases as points; + "open in study view" resolves the region to the FULL case list on the server. +- **Nothing is dropped silently.** `PlotData.pointSet` says which rule drew the points, how + many, and why there are none when there are none; the caption says it in words ("312 cases + in sparse cells drawn as points; 12,775 as density"). +- **The boundary.** Only `PatientLevelResolver` returns points (`plotPoints`, + `plotScatterPoints`, `plotHighlightPoints`). An aggregate-only or federated participant gets + the density alone and the caption says why. +- **The document holds the rule** (`points: false` turns points off — invariant 9: it changes + the server SVG). Hover, search text, a legend-click highlight and a brush stay in browser + state. ## Phases -Do these in order; each one ships on its own. Run `npm test` and a focused Playwright check -per phase, not the full gate each time. - -1. **Per-case gene classes, in SQL.** A builder next to `plotPointsSql` that produces `fill` - and `stroke` per case for one gene: mutation priority, driver split, CNA, SV, not profiled. - Unit tests for the class rules. A boundary test that only `PatientLevelResolver` can call it. -2. **Scatter points, tier 1.** Generalise `plotPoints` beyond box. Add `colorBy.source = - "gene"` to the document, `validateAxis` and `lib/dsl-reference.ts` (then `npm run - docs:gen`). Have `plotSvg` draw coloured points for a small cohort. Check against the - portal: for one study and gene, the per-class case counts must match its Plots tab. -3. **The interactive `PlotFigure`** (canvas): hover with sample id, search/highlight, legend. - The app uses it; chat and MCP keep the SVG. -4. **Borders and clinical colouring on scatter**: CNA/SV strokes, the not-profiled look, - clinical colour, `ringBy` carried over. -5. **Tier 2**: `(bin, category)` count builders (boundary-swept), pooled across sites, - rendered as dominant-class cells with the mix on hover. Captions for every fallback. +Each ships on its own. `npm test` and a focused check per phase; the gate before each push. + +1. **Point subsets on the server (done on `session/plots`).** Scatter: every point at or + under the cap, sparse-cell outliers over it. Box: every point at or under, whisker outliers + over it (over a violin clipped to the whiskers). `pointSet` and captions. The box axis now + spans the points it draws (points beyond the whiskers used to spill outside the frame). +2. **Colour by gene per case.** `colorBy: { source: "gene", gene }`: fill from the case's + highest-priority mutation (oncoprint priority, driver/VUS by the pinned OncoKB), stroke from + CNA/SV, "not profiled" its own class (panel coverage, reuse the oncoprint's). Colour on + scatters too, not only boxes; the rare classes are added to the point subset over the cap. + Check per-class counts against the portal's Plots tab for one study and gene. +3. **Interactive `PlotFigure`** (canvas, beside the server SVG, which stays for chat/MCP): + hover tooltip (sample id linking to the patient view, values, alterations), legend-click + highlight, case-id / protein-change search whose matches come back from the server as + points. +4. **Brush and drill-down**: a region or a cell returns its cases as points; "N cases → study + view" built from the full list on the server. +5. **Sidebar parity**: log toggle, "same gene", Driver vs VUS and VAF axes, box category filter + and sort-by-median, regression line toggle, R² and the equation. + +Backlog: waterfall ("ordered samples"), generic assays and SV axes, bar charts for two +categoricals, the examples bar, SVG/PNG/data export, a method record for the plot's +correlations (it is on the method-check exempt list as "no statistic", which it is not). ## Open questions -- **The cap**: 50k to start? Measure the SQL time and payload on GENIE before deciding. +- **The cap**: 5,000 for points is the box plot's existing value. Measure the canvas figure + before raising it. +- **`OUTLIER_CELL_MAX`**: 3 on the default 240×160 grid. A coarser grid makes "sparse" mean + more; revisit if `bins` becomes a UI control. +- **Federated points**: a federated box concatenates each entitled site's points; a federated + scatter draws none yet (its sparse cells are only known after pooling, so it needs a second + pass per site with the pooled cell list). - **Driver definition**: OncoKB-only here; the portal also counts hotspots and custom annotations. Say so in the legend, as the lollipop's note already does. -- **Multiple genes**: the portal colours by one gene at a time. Keep it to one unless asked. - **Sample vs patient unit**: in patient mode, which of a patient's samples sets the colour? Probably the highest-priority alteration across them. Decide and caption it. diff --git a/lib/dsl-reference.ts b/lib/dsl-reference.ts index 7955609f..f342533d 100644 --- a/lib/dsl-reference.ts +++ b/lib/dsl-reference.ts @@ -369,7 +369,7 @@ export const EXAMPLES: DslExample[] = [ title: "A plot of 13,000 samples, drawn as density", demonstrates: "The portal's Plots tab fetches every point to the browser. This one counts them in the database and sends the counts, so the same picture draws for a hundred cases or four hundred thousand.", tags: ["scale"], - note: "Tumour mutational burden against fraction of the genome altered, over every sample of a TP53-mutant cohort. The figure is a density field: the samples are counted into a 240x160 grid, and only the occupied cells travel. Pearson, Spearman and the fitted line are computed over EVERY sample inside ClickHouse, so they describe the whole cohort rather than the points that happened to be painted last. The second view groups the same measure by TP53 mutation class, as quartiles with Tukey whiskers.", + note: "Tumour mutational burden against fraction of the genome altered, over every sample of a TP53-mutant cohort. The figure is a density field: the samples are counted into a 240x160 grid, and only the occupied cells travel, plus the outliers — the cases in the sparse cells at the fringe, drawn as points. Pearson, Spearman and the fitted line are computed over EVERY sample inside ClickHouse, so they describe the whole cohort rather than the points that happened to be painted last. The second view groups the same measure by TP53 mutation class, as quartiles with Tukey whiskers over a violin, with the values beyond the whiskers drawn as points.", doc: { study: "msk_chord_2024", cohorts: { @@ -387,7 +387,7 @@ export const EXAMPLES: DslExample[] = [ }, { title: "Highlight cohorts on a plot: KRAS G12C and STK11 against 25,000 patients", - demonstrates: "A `highlight` on a plot view: other cohorts drawn over the plotted one, each in its own colour, so the cases you care about are visible against a density that no longer has individual points.", + demonstrates: "A `highlight` on a plot view: other cohorts drawn over the plotted one, each in its own colour, so the cases you care about are visible against a density that has only its outliers as individual points.", tags: ["highlight", "scale"], note: "Age against tumour mutational burden for every patient in MSK-CHORD (24,947 with both), drawn as a grey density. KRAS G12C (1,113 patients) and STK11-mutant (1,206) are drawn over it as points, each in its own colour. The coefficients and the fitted line are still those of the whole cohort. The second view groups the same measure by cancer type: KRAS G12C is again points, while TP53-mutant patients (13,105) are too many for points, so they are drawn as a narrow box per group beside the cohort's own box, and the caption says why.", doc: { @@ -1522,7 +1522,7 @@ patient — so the arm counts summing to more than the base cohort is expected, | \`logistic\` | \`outcome:axis, outcomeEvent?:"", covariates:[axis, ...(max 8)], cohorts:[names]\` | multivariate logistic regression: a BINARY outcome (not time-to-event \u2014 no endpoint/months convention here) regressed on several covariates at once, reported as odds ratios with Wald confidence intervals, drawn as a forest plot the same shape as \`cox\`'s. \`outcome\` is a plot axis, exactly like a covariate (clinical, molecular, or alteration status). A categorical outcome (a clinical attribute, or an alteration axis) needs \`outcomeEvent\` naming which observed value counts as the event (1) \u2014 every other observed value collapses into "not event" (0), and which values got collapsed is disclosed in the caveat; omitting it, or naming a value never observed, is refused with the actual observed values listed rather than guessed at. A numeric outcome must already be coded exactly 0/1 (refused otherwise) and must NOT be given \`outcomeEvent\`. Covariates use the exact same dummy-coding and reference-level rules as \`cox\` (see above): a mutated-vs-not axis defaults to \`\"Not mutated\"\`, others default to the most frequent level, \`reference\` overrides either. Same patient-level-only boundary as \`cox\` | | \`course\` | \`eventTypes:[\u22646], anchor?:{eventType,key?,value?,occurrence?}, binMonths?:3, fromMonths?:-12, toMonths?:60, cohorts:[names]\` | the **clinical course**: per time bin, the share of the cohort in each event state. Two things it refuses to fake. (1) The denominator is the patients still under observation in that bin \u2014 first recorded event to last contact (the later of their last event and their OS follow-up) \u2014 never the whole cohort, which would make a curve fall merely because follow-up ran out. (2) \`anchor\` is optional for one study but **required across studies**: day 0 is birth in some studies and first sequencing in others, and nothing in the schema records which, so raw day 0 compares incomparable clocks. Event types that carry real durations are drawn as a state the patient is IN; the rest as an event that HAPPENED \u2014 read from the data, not from the name. Bins are half-open [t, t+w). Event-name spellings differing only in case are folded (\`Treatment\`/\`TREATMENT\`) and the fold is named in the caveat; nothing else is merged | | \`sequence\` | \`labelKey?:"SUBTYPE", eventType?:"treatment", lineClasses?:"systemic"|"all"|[classes], exclude?:[values], lineGap?:90, maxSteps?:3, topN?:8, cohorts:[names]\` | **what follows what**: therapy transitions as lines, drawn as a Sankey. Records are first grouped into LINES \u2014 therapies less than \`lineGap\` days apart are one line, labelled by their whole combination (sorted, so \`A + B\` and \`B + A\` are one regimen). That grouping is the point: a patient has a median of 6 treatment records and 4 distinct agents, and combination therapy is several records describing ONE decision. Paths that end flow into TWO explicit terminals, never one: **no further line** (the study watched for at least one \`lineGap\` and saw no further therapy) and **record ends here** (it did not, so nothing can be concluded). Measured on MSK-CHORD, 68% of patients have under three months of observation after their last therapy and the median is 15 days \u2014 so collapsing the two would turn record-keeping into an apparent clinical decision. Flows out of a node sum to it, so nobody vanishes. Up to two cohorts are drawn as panels sharing one scale and one colour per regimen. Lines are the shared definition (see "Treatment line (derived)" above): systemic-therapy records by default, \`lineClasses\` \`"all"\` for every treatment record or a list of class names for your own; \`labelKey\` only names a line\u2019s therapies (e.g. AGENT), it never decides which records form lines. \`exclude\` drops records by label value, matched ignoring case. More than two cohorts is refused | -| \`plot\` | \`x:axis, y:axis, unit?:"patient"\\|"sample", bins?:[240,160], points?:true, colorBy?:axis, ringBy?:axis, highlight?:[names(max 4)], cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size | +| \`plot\` | \`x:axis, y:axis, unit?:"patient"\\|"sample", bins?:[240,160], points?:true, colorBy?:axis, ringBy?:axis, highlight?:[names(max 4)], cohorts:[names]\` | two axes over the same cases, aggregated **inside the database**: a scatter comes back as counts per grid cell, a box as quartiles per group, two categoricals as a contingency table. The payload is a few kilobytes whatever the cohort's size. \`points\` (default true): every case is a point up to 5,000, and past that the outliers are (a box's values beyond its whiskers, a scatter's cases in sparse cells), from a participant that may show cases | | \`attrition\` | \`cohorts:[names] (≤6)\` | **cohort attrition**, CONSORT-style: for each cohort, the study base and then the patients left after each clause of its \`all\`, in the order written, with how many that clause alone removed. Each step is evaluated as its own cohort (the first k clauses ANDed), with the same compiler and the same per-study counts, so the last step is the cohort's own count by construction — a mismatch is reported, never smoothed over. The order of clauses changes the path, never the final count. Counts only, so every participant (aggregate-only and remote included) can answer; across studies the counts are summed and overlap is not deduplicated | | \`genome\` | \`binMb?:2, threshold?:0.2, includeX?:false, cohorts:[names] (≤4)\` | **genome-wide copy-number frequency** from the copy-number SEGMENTS: the share of each cohort's segmented samples gained (log2 segment mean ≥ \`threshold\`) or lost (≤ −\`threshold\`) in every \`binMb\`-megabase bin (1–50) along chromosomes 1–22 (and X with \`includeX\` — its log ratio mostly records sex, so it is off by default), drawn as gains up and losses down, one panel per cohort. Each bin's denominator is the samples whose segmentation covers it — a bin nothing covers is drawn as unmeasured, never as copy-neutral. A bin takes the value of the segment over its midpoint, so an event narrower than a bin can be missed; the caption says so. Unit is the SAMPLE, not the patient. Exactly two cohorts add a per-bin two-sided Fisher test (gains and losses separately, Benjamini–Hochberg across bins), drawn as a strip coloured by which cohort is higher. Studies on different reference genomes (hg19/hg38) are refused rather than drawn on one axis. Counts only; a remote peer cannot answer yet and is named as left out | | \`pathways\` | \`pathways?:[names], sets?:[{name, genes:[≤40]} ×≤4], measure?:"driver"\|"any", topGenes?:6, cohorts:[names] (≤4)\` | **oncogenic signalling pathways**: the ten canonical pathways of Sanchez-Vega et al. (Cell 2018, Table S3: RTK-RAS, PI3K, TP53, Cell cycle, MYC, WNT, NOTCH, Hippo, TGF-β, NRF2 — 334 genes), and/or the document's own \`sets\`, each as the share of the cohort with at least one member altered. \`measure\` defaults to \`"driver"\` (OncoKB-annotated drivers, as the paper counted curated oncogenic events; refused without an annotation pin) or \`"any"\`. A pathway's denominator is the patients profiled for at least ONE member (the union of the members' panels, the oncoprint's merged-row rule, computed in the database with the same event table, CNA rule and driver join as \`compare\`, so a one-gene set equals that gene's own count); each member's % is over the patients profiled for THAT gene. The \`topGenes\` most-altered members are drawn per pathway; the rest are pooled as "+N more altered", and members on no participating panel are listed as never profiled. Two cohorts add Fisher tests with BH q across pathways, and separately across member genes. Counts only; a remote peer cannot answer yet and is named | @@ -1618,19 +1618,27 @@ overlay a single-study box can fall back to (that needs the same shared-scale bi uses, not yet built for a box); its quartiles are a size-weighted approximation across sites, the same one Explore's federated numeric histograms already disclose. -**What sits behind the boxes.** cBioPortal draws every value as a jittered point over its box. -So does this, up to 5,000 values, and only when the study is one this participant may see cases -in — the points carry case ids, so they come from the same entitlement that lets a -patient-level participant list cohort members, and an aggregate-only participant has no method -that could return them. The jitter is fixed per case, so the figure does not move between the -server SVG, the browser and the Python package. - -Above the cap, or with \`points: false\`, or from a participant that may not disclose a case, the -same values are drawn as a **violin** from per-group bins: a quarter of a million jittered dots -is a solid block, while the density is 19KB and is what the reader was looking for. Each violin -is scaled to its own densest bin, because one group holding 178,489 of the values would -otherwise flatten every other group to nothing; how many cases a group has is printed under its -name. The caption always says which of the two it drew. +**Points: every case when it is a drawing, the outliers when it is not.** cBioPortal draws every +case as a point. So does this, up to 5,000 cases, on a scatter as on a box, and only when the +study is one this participant may see cases in — the points carry case ids, so they come from the +same entitlement that lets a patient-level participant list cohort members, and an aggregate-only +participant has no method that could return them. The jitter across a box is fixed per case, so +the figure does not move between the server SVG, the browser and the Python package. + +Past 5,000 the crowd is drawn as density — a quarter of a million dots is a solid block, and a +hover on it would land on an arbitrary case — and the **outliers** stay points: on a box, the +values beyond each group's Tukey whiskers, drawn over a **violin** (from per-group bins) that stops +at the whiskers; on a scatter, the cases in **sparse cells** of the grid — every cell holding at +most 3 cases (fewer, if that is what keeps the points under 5,000) is drawn as its cases instead of +as a shade. Both subsets are chosen in the database, against the same grid and whiskers the figure +draws, and if a subset is itself past 5,000 none of it is drawn, never the first 5,000. With +\`points: false\`, or from a participant that may not disclose a case, the figure is density alone. +The caption always says which it drew, how many points, and why there are none when there are +none. Each violin is scaled to its own densest bin, because one group holding 178,489 of the +values would otherwise flatten every other group to nothing; how many cases a group has is +printed under its name. A box's axis spans the points it draws but reaches at most one whisker +span past the whiskers, so one extreme value cannot flatten every box; a point beyond that is a +triangle at the frame's edge, counted in the caption. **Colour.** \`colorBy\` fills each value with the colour of a category and \`ringBy\` rings it, the portal's "Color samples by" with its two checkboxes — so a case that is amplified *and* mutated @@ -1663,7 +1671,7 @@ aggregate on the plot's own scale — cells on the scatter's grid, or a narrow b caption says which and why. A highlight is never cut to its first 5,000 cases. Highlighting changes no statistic: the coefficients and quartiles stay those of the whole plotted cohort. A contingency table has no points to mark, so its highlights are counted but not drawn. On a box plot a -highlighted value beyond the whiskers is pinned to the frame's edge as a triangle and counted. +highlighted value beyond the axis is pinned to the frame's edge as a triangle and counted. A case with no value on an axis is simply absent, so a pair is an inner join: only cases measured on both axes are counted, and the figure reports how many that was. @@ -1880,7 +1888,7 @@ export const AGENT_GRAMMAR = `Document shape: // | {"source":"alteration","gene":G,"by":"mutated|mutationType|cna|driver"?} // | {"source":"alteration","genes":[G,...(2-20)],"by":"mutated"?} // any-of-N as one category; only "mutated" // | {"source":"alteration","oql":"KRAS: G12C; STK11: MUT"} // Altered (matches any entry) vs Unaltered - // two numeric axes -> scatter (binned, never points); one categorical -> box; + // two numeric axes -> scatter (binned; every case a point up to 5,000, else sparse-cell outliers); one categorical -> box; // two categorical -> a contingency table {"kind":"attrition","cohorts":[names?(max 6)]}, {"kind":"genome","binMb":2,"threshold":0.2,"includeX":false,"cohorts":[names?(max 4)]}, diff --git a/lib/federation.ts b/lib/federation.ts index 9dae5189..6fadbfd6 100644 --- a/lib/federation.ts +++ b/lib/federation.ts @@ -28,11 +28,12 @@ import { axisKind, axisLabel, axisSelect, corrPValue, plotShape, tukey, validate categoryColor, CATEGORY_CAP, COLOR_CAP, DEFAULT_BINS, POINTS_CAP, VIOLIN_BINS, plotBinsSql, plotBoxSql, plotCountSql, plotCoverageSql, plotMomentsSql, plotPointsSql, plotStatsSql, plotTableSql, plotViolinSql, highlightCohortSql, plotHighlightCountSql, plotScatterPointsSql, - type AxisKind, type HighlightPoint, type PlotAxis, type PlotData, type PlotPoint, type PlotRequest, type PlotShape } from "./plot.ts"; + type AxisKind, type HighlightPoint, type PlotAxis, type PlotData, type PlotPoint, type PlotRequest, type PlotShape, + type PlotXYPoint, type SparseCells, type Whiskers } from "./plot.ts"; // re-exported so test/boundary.test.ts sweeps them with the other aggregate SQL builders export * from "./federation-views.ts"; export { plotBinsSql, plotBoxSql, plotCountSql, plotCoverageSql, plotHighlightCountSql, plotMomentsSql, plotStatsSql, plotTableSql, plotViolinSql }; -export type { PlotData, PlotRequest, PlotGroup, PlotPoint, PlotHighlight, HighlightPoint } from "./plot.ts"; +export type { PlotData, PlotRequest, PlotGroup, PlotPoint, PlotXYPoint, PlotHighlight, HighlightPoint } from "./plot.ts"; import { impactSql } from "./lollipop.ts"; import type { Moments } from "./stats.ts"; // a table view's row queries return ids: imported for PatientLevelResolver.rowsPage, never re-exported @@ -1272,8 +1273,10 @@ export interface QueryLogEntry { label: string; sql: string; ms?: number; remote /** Additionally available when the instance permits patient-level access. */ export interface PatientLevelCapable extends AggregateCapable { - /** the values behind a box plot, with their case ids — the entitlement, in one method */ - plotPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap?: number): Promise; + /** the values behind a box plot (every one, or those beyond `outside`'s whiskers), with their case ids; null past the cap */ + plotPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap?: number, outside?: Whiskers[]): Promise; + /** a scatter's cases (every one, or those in `sparse` cells), with their case ids; null past the cap */ + plotScatterPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap?: number, sparse?: SparseCells): Promise; /** a highlight cohort's points over the plot, with their case ids; null past the cap */ plotHighlightPoints(def: CohortDef | null | undefined, highlight: CohortDef, spec: PlotRequest, xk: AxisKind, yk: AxisKind, cap?: number): Promise; @@ -1938,15 +1941,16 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa this.instance = { ...instance, mode: "patient-level" }; } /** - * The points behind a box plot, with their case ids, so a reader can find one on the figure. + * The points behind a box plot, with their case ids, so a reader can find one on the figure: + * every value, or with `outside` only the values beyond their group's whiskers (the outliers + * drawn over a violin once the cohort is past the cap). * * It lives here and nowhere else. `AggregateResolver` has no such method, and neither does a * remote peer, so a participant that may not disclose a case cannot be asked for one — the * same guarantee as `members()`, and for the same reason. The cap is what stops a quarter of - * a million dots being called a drawing; above it the caller falls back to the density every - * participant can compute. + * a million dots being called a drawing: past it this returns null, never the first `cap`. */ - async plotPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap = POINTS_CAP): Promise { + async plotPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap = POINTS_CAP, outside?: Whiskers[]): Promise { const db = this.instance.conn.db, conn = this.instance.conn; const unit = spec.unit === "sample" ? "sample" as const : "patient" as const; const ctx = { study: this.study, db, unit, cohortSql: this.cohortSql(def), ...this.oqlCtx() }; @@ -1958,14 +1962,32 @@ export class PatientLevelResolver extends BaseResolver implements PatientLevelCa const catSel = axisSelect(numFirst ? spec.y : spec.x, ctx, "cat"); const colorSel = spec.colorBy ? axisSelect(spec.colorBy, ctx, "cat") : undefined; const ringSel = spec.ringBy ? axisSelect(spec.ringBy, ctx, "cat") : undefined; - const r = await this.runLogged("plot:points", - plotPointsSql(numSel, catSel, ctx.cohortSql, cap, colorSel, ringSel), conn); - // one over the cap means there were more than a drawing can hold: the caller draws density - if (r.data.length > cap) return []; + const r = await this.runLogged(outside ? "plot:outliers" : "plot:points", + plotPointsSql(numSel, catSel, ctx.cohortSql, cap, colorSel, ringSel, outside), conn); + if (r.data.length > cap) return null; return r.data.map((x) => ({ id: String(x.id), g: String(x.g), v: Number(x.v), ...(colorSel ? { c: String(x.c ?? "") } : {}), ...(ringSel ? { r: String(x.r ?? "") } : {}) })); } + /** + * A scatter's cases with their ids: every one, or with `sparse` only those in the density's + * sparse cells (binned against the plot's own grid and extent, so they are exactly the cells + * the figure leaves undrawn). Case-level, so here and nowhere else, for the reason plotPoints + * is; null past the cap. + */ + async plotScatterPoints(def: CohortDef | null | undefined, spec: PlotRequest, cap = POINTS_CAP, sparse?: SparseCells): Promise { + const db = this.instance.conn.db, conn = this.instance.conn; + const unit = spec.unit === "sample" ? "sample" as const : "patient" as const; + const ctx = { study: this.study, db, unit, cohortSql: this.cohortSql(def), ...this.oqlCtx() }; + const kinds = await this.clinicalKinds([spec.x, spec.y]); + const xk = axisKind(spec.x, (a) => kinds.get(a)), yk = axisKind(spec.y, (a) => kinds.get(a)); + if (plotShape(xk, yk) !== "scatter") return []; + const r = await this.runLogged(sparse ? "plot:outliers" : "plot:points", + plotScatterPointsSql(axisSelect(spec.x, ctx, xk), axisSelect(spec.y, ctx, yk), ctx.cohortSql, cap, sparse), conn); + if (r.data.length > cap) return null; + return r.data.map((p) => ({ id: String(p.id), x: Number(p.x), y: Number(p.y) })); + } + /** * A highlight's points: the plotted cohort's cases that are also in `highlight`, each with its * case id — (x, y) on a scatter, (group, value) on a box. Case-level, so here and nowhere else, diff --git a/lib/figures/plot.ts b/lib/figures/plot.ts index b33bcd22..998d148b 100644 --- a/lib/figures/plot.ts +++ b/lib/figures/plot.ts @@ -1,8 +1,9 @@ // The plot view, from the aggregated contract: a density field for a scatter, boxes for a numeric // axis grouped by a categorical one, a shaded contingency for two categoricals. // -// Nothing here has ever seen a point. The cells, the quartiles and the coefficients arrive already -// computed, which is why the same picture draws for a hundred cases or four hundred thousand. +// The cells, the quartiles and the coefficients arrive already computed, which is why the same +// picture draws for a hundred cases or four hundred thousand. The only cases drawn one by one are +// the points the engine chose (every case under the cap, the outliers over it), at most POINTS_CAP. import type { ViewResult } from "../types.ts"; import type { PlotData } from "../plot.ts"; import { isAbsent, jitterFor } from "../plot-draw.ts"; @@ -85,8 +86,17 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { const cell = (b: { bx: number; by: number }, fill: string, opacity: number) => ``; + // Cells whose cases are drawn as points are not also shaded: every case, or the sparse + // cells (at most cellMax cases) the outlier points stand in for. + const ps = d.pointSet; + const shaded = ps?.rule === "all" && d.xyPoints?.length ? [] + : ps?.rule === "outliers" && ps.cellMax ? d.bins!.filter((b) => b.n > ps.cellMax!) : d.bins!; // under highlights the plotted cohort is a floor-raised grey, so its faintest cells still read - for (const b of d.bins!) out.push(cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))); + for (const b of shaded) out.push(cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))); + const own = d.xyPoints || []; + const rOwn = Math.max(1.4, Math.min(3.4, 80 / Math.sqrt(Math.max(1, own.length)))); + for (const pt of own) + out.push(``); // a highlight too large to draw as points is cells on the same grid, shaded by its own densest cell for (const h of hs) if (h.shown === "density" && h.bins) { const hMax = Math.max(1, ...h.bins.map((b) => b.n)); @@ -108,7 +118,9 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { let y = T + PLOT + bottomAxisHeight(xTitle) + 8; const hk = highlightItems(d); if (hk.length) { const lg = legend(hk, 8, y, W - 16, p, { title: "highlighted" }); out.push(lg.svg); y += lg.height + 4; } - const cap = caption((v.note ? v.note + " " : "") + "Darker is more " + (d.n > 1 ? "cases" : "case") + "; the shade is logarithmic." + + const cap = caption((v.note ? v.note + " " : "") + (shaded.length + ? "Darker is more " + (d.n > 1 ? "cases" : "case") + "; the shade is logarithmic." + (own.length ? " Each dot is one case." : "") + : "Each dot is one case.") + (st && st.slope != null ? " The dashed line is the least-squares fit over every case." : ""), W, y, p.muted); return frame(W, y + cap.height + 8, out.join("") + cap.svg, p.bg, false, "Plot — " + yTitle + " against " + xTitle); } @@ -123,7 +135,13 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const numAxis = d.x.kind === "num" ? d.x : d.y, catAxis = d.x.kind === "num" ? d.y : d.x; const numTitle = axisTitle(numAxis), catTitle = axisTitle(catAxis); const T = 12, R = 16, PLOT = plotHeight(W0); - const lo = Math.min(...gs.map((g) => g.loWhisker)), hi = Math.max(...gs.map((g) => g.hiWhisker)); + // The axis spans the whiskers and the points beyond them, but reaches at most one whisker + // span further each way: one TMB of 570 would otherwise flatten every box to a line. A point + // past that is pinned to the frame's edge as a triangle and counted in the caption. + const ptVals = (d.points || []).map((pt) => pt.v); + const wLo = Math.min(...gs.map((g) => g.loWhisker)), wHi = Math.max(...gs.map((g) => g.hiWhisker)); + const wSpan = Math.max(1e-9, wHi - wLo); + const lo = Math.max(Math.min(wLo, ...ptVals), wLo - wSpan), hi = Math.min(Math.max(wHi, ...ptVals), wHi + wSpan); // the whiskers' own range, padded a little, with round ticks inside it: rounding the ends out // would stretch a −1 whisker into an empty band down to −20 const pad = (hi - lo) * 0.03 || 1; @@ -174,7 +192,13 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const otherLabel = (n: number, noValue: boolean) => "other (" + [n ? n + " more categor" + (n === 1 ? "y" : "ies") : "", noValue ? "no value" : ""].filter(Boolean).join(", ") + ")"; const vBins = d.violinBins || 40; - let offScale = 0; + let offScale = 0, ptOffScale = 0; + /** a value off the axis: a triangle at the frame's edge pointing the way it lies */ + const edgeMark = (px: number, up: boolean, r: number, fill: string) => { + const ty = up ? T + 1 : T + PLOT - 1, k = r * 1.3; + return ``; + }; const inAxis = (val: number) => Math.max(vlo, Math.min(vhi, val)); gs.forEach((g, i) => { @@ -184,13 +208,15 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { // a bin stands for a range of VALUES, mapped through the extent the bins were cut over const [vx0, vx1] = d.violinExtent || [lo, hi]; const binW = (vx1 - vx0) / vBins; + // with the outliers drawn as points, the density stops at this group's whiskers + const [cLo, cHi] = d.pointSet?.rule === "outliers" ? [g.loWhisker, g.hiWhisker] : [vlo, vhi]; for (const [b, byClass] of cells) { const vLo = vx0 + b * binW, vHi = vLo + binW; - if (vHi < vlo || vLo > vhi) continue; + if (vHi < cLo || vLo > cHi) continue; const total = [...byClass.values()].reduce((a, n) => a + n, 0); const w = half * 1.6 * Math.sqrt(total / (vioMax.get(g.g) || 1)); - const yTop = ys(Math.min(vhi, vHi)); - const step = Math.max(1.2, ys(Math.max(vlo, vLo)) - yTop); + const yTop = ys(Math.min(cHi, vHi)); + const step = Math.max(1.2, ys(Math.max(cLo, vLo)) - yTop); if (!colorKey.length) { out.push(``); continue; @@ -206,12 +232,17 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { } } } - if (d.overlay === "points" && byGroup.has(g.g)) { + if (byGroup.has(g.g)) { // two encodings on one dot need a dot big enough to carry them; a crowded plot gets smaller ones const rBase = Math.max(1.4, Math.min(4.5, 90 / Math.sqrt(Math.max(1, d.points!.length)))); for (const pt of byGroup.get(g.g)!) { const fill = pt.c != null && colorKey.length ? fillOf(pt.c) : base; const ring = pt.r != null && !isAbsent(pt.r) ? ringOf(pt.r) : null; + if (pt.v > vhi || pt.v < vlo) { + ptOffScale++; + out.push(edgeMark(cx + jitterFor(pt.id) * half * 0.9, pt.v > vhi, rBase, fill)); + continue; + } out.push(``); @@ -241,9 +272,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { const px = cx + jitterFor(pt.id) * half * 0.9; if (pt.v > vhi || pt.v < vlo) { offScale++; - const up = pt.v > vhi, ty = up ? T + 1 : T + PLOT - 1, k = rH * 1.3; - out.push(``); + out.push(edgeMark(px, pt.v > vhi, rH, h.color)); continue; } out.push(``); @@ -281,6 +310,8 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { (d.overlay === "points" ? " Every value is drawn over its box; the jitter is fixed per case." : d.overlay === "violin" ? " The shaded width is the share of that group's values sitting at that height, scaled per group." : "") + + (ptOffScale ? " " + fmtInt(ptOffScale) + " value" + (ptOffScale === 1 ? " lies" : "s lie") + " more than a whisker span" + + " past the whiskers and " + (ptOffScale === 1 ? "is" : "are") + " drawn as a triangle at the frame's edge." : "") + (offScale ? " " + fmtInt(offScale) + " highlighted value" + (offScale === 1 ? " lies" : "s lie") + " beyond the axis and " + (offScale === 1 ? "is" : "are") + " drawn as a triangle at the frame's edge." : ""), W, y, p.muted); return frame(W, y + cap.height + 8, out.join("") + cap.svg, p.bg, W > W0, "Plot — " + numTitle + " by " + catTitle); diff --git a/lib/plot.ts b/lib/plot.ts index 105e1633..aafb8e0f 100644 --- a/lib/plot.ts +++ b/lib/plot.ts @@ -68,6 +68,13 @@ export const POINTS_CAP = 5000; export { HIGHLIGHT_CAP, HIGHLIGHT_COLORS, HIGHLIGHT_BASE } from "./plot-highlight.ts"; export { isAbsent, jitterFor } from "./plot-draw.ts"; import { isAbsent } from "./plot-draw.ts"; +/** + * Over the points cap, a scatter cell holding at most this many cases is drawn as its cases + * rather than as a shade: the sparse fringe of the density, which is where the outliers are. + * The threshold actually used is the largest value up to this whose points fit under the cap + * (`sparseCellMax`). + */ +export const OUTLIER_CELL_MAX = 3; /** bins per group in a violin */ export const VIOLIN_BINS = 40; /** distinct colours a legend can carry before it stops being a legend */ @@ -358,21 +365,51 @@ export function plotMomentsSql(xSel: string, ySel: string, cohortSql: string): s */ export function plotBinsSql(xSel: string, ySel: string, cohortSql: string, bins: [number, number] = DEFAULT_BINS, extent?: { x0: number; x1: number; y0: number; y1: number }): string { - const [bx, by] = [Math.max(2, Math.min(600, Math.round(bins[0]))), Math.max(2, Math.min(600, Math.round(bins[1])))]; + const [bx, by] = clampGrid(bins); const eCte = extent ? ", e AS (SELECT " + num(extent.x0) + " AS x0, " + num(extent.x1) + " AS x1, " + num(extent.y0) + " AS y0, " + num(extent.y1) + " AS y1)\n" : ", e AS (SELECT min(x) x0, max(x) x1, min(y) y0, max(y) y1 FROM pair)\n"; return pairCte(xSel, ySel, cohortSql) + eCte + - "SELECT least(" + (bx - 1) + ", toUInt32(floor((x - x0) / nullIf(x1 - x0, 0) * " + bx + "))) AS bx,\n" + - " least(" + (by - 1) + ", toUInt32(floor((y - y0) / nullIf(y1 - y0, 0) * " + by + "))) AS by,\n" + + "SELECT " + binIndex("x", bx) + " AS bx,\n" + + " " + binIndex("y", by) + " AS by,\n" + " count() AS n\n" + "FROM pair, e GROUP BY bx, by"; } -/** a finite number as a SQL literal — extents cross this module's boundary as numbers, never strings */ +/** the grid a request asks for, held to what a figure can draw */ +function clampGrid(bins: [number, number]): [number, number] { + return [Math.max(2, Math.min(600, Math.round(bins[0]))), Math.max(2, Math.min(600, Math.round(bins[1])))]; +} + +/** + * Which of `n` cells a value falls in, against the extent CTE `e`. One expression for the + * density and for the sparse-cell points, so a point is never assigned a cell the density did + * not count it in. + */ +function binIndex(col: "x" | "y", n: number): string { + return "least(" + (n - 1) + ", toUInt32(floor((" + col + " - " + col + "0) / nullIf(" + col + "1 - " + col + "0, 0) * " + n + ")))"; +} + +/** + * The cell threshold for a scatter's outlier points: the largest k up to `max` such that every + * case in a cell of at most k cases fits under `cap` points. 0 when even the cells of one case + * hold more than `cap` — then no subset is drawn, and the caller says so rather than drawing + * some of them. + */ +export function sparseCellMax(bins: { n: number }[], cap: number, max = OUTLIER_CELL_MAX): number { + let k = 0; + for (let t = 1; t <= max; t++) { + const cases = bins.reduce((a, b) => b.n <= t ? a + b.n : a, 0); + if (cases > cap) break; + k = t; + } + return k; +} + +/** a finite number as a SQL literal — extents and whiskers cross this module's boundary as numbers, never strings */ function num(x: number): string { - if (!Number.isFinite(x)) throw new Error("plotBinsSql: extent must be finite numbers"); + if (!Number.isFinite(x)) throw new Error("a plot's extent and whiskers must be finite numbers"); return String(x); } @@ -420,7 +457,7 @@ export function plotViolinSql(numSel: string, catSel: string, cohortSql: string, * site has no method that could call it — see docs/DATA-BOUNDARY.md. */ export function plotPointsSql(numSel: string, catSel: string, cohortSql: string, cap = POINTS_CAP, - colorSel?: string, ringSel?: string): string { + colorSel?: string, ringSel?: string, outside?: Whiskers[]): string { return "WITH cohort AS (" + cohortSql + "),\n" + " vs AS (" + numSel + "),\n" + " gs AS (" + catSel + ")" + @@ -430,9 +467,26 @@ export function plotPointsSql(numSel: string, catSel: string, cohortSql: string, "FROM vs INNER JOIN gs ON gs.id = vs.id" + (colorSel ? "\n LEFT JOIN cs ON cs.id = vs.id" : "") + (ringSel ? "\n LEFT JOIN rs ON rs.id = vs.id" : "") + "\n" + + (outside ? outsideWhiskers(outside) + "\n" : "") + "LIMIT " + (Math.max(1, Math.min(50000, Math.round(cap))) + 1); } +/** one group's whiskers, which is all a box's outlier query needs to know about it */ +export interface Whiskers { g: string; loWhisker: number; hiWhisker: number } + +/** + * WHERE a value lies beyond its own group's Tukey whiskers. The whiskers were computed by the + * box query and cross back in as literals, a group label quoted and a bound a finite number. A + * group not listed has no whiskers to be outside of, so none of its values match. + */ +function outsideWhiskers(ws: Whiskers[]): string { + if (!ws.length) return "WHERE 0"; + const at = "indexOf([" + ws.map((w) => sq(w.g)).join(", ") + "], gs.v)"; + const arr = (f: (w: Whiskers) => number) => "[" + ws.map((w) => num(f(w))).join(", ") + "]"; + return "WHERE " + at + " > 0 AND (vs.v < arrayElement(" + arr((w) => w.loWhisker) + ", " + at + ")" + + " OR vs.v > arrayElement(" + arr((w) => w.hiWhisker) + ", " + at + "))"; +} + /** * The cases of a plotted cohort that are also in a highlight cohort: the plotted cohort's SQL * narrowed by the highlight's. A cohort expression like any other (it projects `pid`), so the @@ -453,15 +507,30 @@ export function plotHighlightCountSql(cohortSql: string, highlightSql: string): * Like plotPointsSql it returns identifiers, so it stays out of lib/federation.ts's aggregate * builders and is reachable only from `PatientLevelResolver`. */ -export function plotScatterPointsSql(xSel: string, ySel: string, cohortSql: string, cap = POINTS_CAP): string { - return "WITH cohort AS (" + cohortSql + "),\n" + +export function plotScatterPointsSql(xSel: string, ySel: string, cohortSql: string, cap = POINTS_CAP, + sparse?: SparseCells): string { + const limit = "LIMIT " + (Math.max(1, Math.min(50000, Math.round(cap))) + 1); + if (!sparse) return "WITH cohort AS (" + cohortSql + "),\n" + " xs AS (" + xSel + "),\n" + " ys AS (" + ySel + ")\n" + "SELECT xs.id AS id, xs.v AS x, ys.v AS y\n" + - "FROM xs INNER JOIN ys ON ys.id = xs.id\n" + - "LIMIT " + (Math.max(1, Math.min(50000, Math.round(cap))) + 1); + "FROM xs INNER JOIN ys ON ys.id = xs.id\n" + limit; + // the cases of every cell holding at most `cellMax` of them, binned exactly as plotBinsSql + // bins the density over the same extent and grid + const [bx, by] = clampGrid(sparse.grid), e = sparse.extent; + if (!(sparse.cellMax >= 1)) throw new Error("plotScatterPointsSql: cellMax must be at least 1"); + return "WITH cohort AS (" + cohortSql + "),\n" + + " xs AS (" + xSel + "),\n" + + " ys AS (" + ySel + "),\n" + + " e AS (SELECT " + num(e.x0) + " AS x0, " + num(e.x1) + " AS x1, " + num(e.y0) + " AS y0, " + num(e.y1) + " AS y1),\n" + + " c AS (SELECT xs.id AS id, xs.v AS x, ys.v AS y, " + binIndex("x", bx) + " AS bx, " + binIndex("y", by) + " AS by\n" + + " FROM xs INNER JOIN ys ON ys.id = xs.id, e),\n" + + " sparse AS (SELECT bx, by FROM c GROUP BY bx, by HAVING count() <= " + Math.round(sparse.cellMax) + ")\n" + + "SELECT id, x, y FROM c WHERE (bx, by) IN (SELECT bx, by FROM sparse)\n" + limit; } +/** which cells of a scatter's density are drawn as their cases: the plot's own grid and extent, and the threshold */ +export interface SparseCells { grid: [number, number]; extent: { x0: number; x1: number; y0: number; y1: number }; cellMax: number } /** the contingency table, one row per occupied cell */ export function plotTableSql(xSel: string, ySel: string, cohortSql: string): string { @@ -531,6 +600,21 @@ function lgamma(z: number): number { /** what a plot asks for; the shape it draws as is decided from the axes, not requested */ /** one drawn value, with its case id and whatever it is coloured by */ export interface PlotPoint { id: string; g: string; v: number; c?: string; r?: string } +/** one drawn scatter case, with its id */ +export interface PlotXYPoint { id: string; x: number; y: number } +/** + * Which cases a plot draws as points, and why the rest are not. `all`: every case. `outliers`: + * over the points cap, the values beyond a box's whiskers or the cases in a scatter's sparse + * cells, over the density of the rest. `none`: density alone, and `why` says why. + */ +export interface PointSet { + rule: "all" | "outliers" | "none"; + /** points drawn */ + drawn: number; + /** a scatter's outliers: every cell holding at most this many cases is drawn as its cases */ + cellMax?: number; + why?: string; +} export interface PlotRequest { x: PlotAxis; y: PlotAxis; unit?: "patient" | "sample"; bins?: [number, number]; @@ -572,8 +656,12 @@ export interface PlotData { stats?: { pearson: number | null; spearman: number | null; slope: number | null; intercept: number | null; pPearson: number | null; pSpearman: number | null }; groups?: PlotGroup[]; - /** the values behind the boxes: points when few enough and the reader may see cases, else a density */ + /** the values behind the boxes: every one when few enough and the reader may see cases, else the outliers */ points?: PlotPoint[]; + /** a scatter's cases drawn as points: every one when few enough, else those in sparse cells */ + xyPoints?: PlotXYPoint[]; + /** which rule chose the points, how many, and why there are none when there are none */ + pointSet?: PointSet; violin?: { g: string; b: number; n: number; c?: string }[]; /** what the colours mean: the fill category, the ring category, and the key to draw */ colorLabel?: string; diff --git a/lib/views/plot.ts b/lib/views/plot.ts index 3f11720a..4fe05f10 100644 --- a/lib/views/plot.ts +++ b/lib/views/plot.ts @@ -3,7 +3,7 @@ import type { PlotView, ViewResult } from "../types.ts"; import { pearsonFromMoments, regressionFromMoments, type Moments } from "../stats.ts"; import { isPatientLevel, hasFederatedPlot, legendFor, type Resolver, type FederatedPlotCapable } from "../federation.ts"; -import { DEFAULT_BINS, CATEGORY_CAP, POINTS_CAP, HIGHLIGHT_CAP, HIGHLIGHT_COLORS, tukey, coverageNote, +import { DEFAULT_BINS, CATEGORY_CAP, POINTS_CAP, HIGHLIGHT_CAP, HIGHLIGHT_COLORS, OUTLIER_CELL_MAX, tukey, coverageNote, sparseCellMax, type PlotData, type PlotGroup, type PlotHighlight, type PlotRequest } from "../plot.ts"; import { untimed, type Untimed, type ViewRun, type DocView } from "./run.ts"; @@ -114,6 +114,87 @@ function highlightNote(hs: PlotHighlight[], unit: "patient" | "sample"): string (hs.some((h) => h.shown !== "none") ? " The statistics are over the whole plotted cohort." : ""); } +/** + * The cases a single-study plot draws as points, set on `data`, and the sentence its caption says + * about them. At or under POINTS_CAP every case is a point. Over it the crowd stays density and + * the outliers are points: on a box the values beyond each group's whiskers (over a violin), on + * a scatter the cases in cells of at most `cellMax` (the cells the figure then leaves unshaded). + * Points come only from a participant entitled to see cases — isPatientLevel is a type guard, + * so asking an aggregate-only one is a compile error rather than a review catch. + */ +async function pointLayer(pv: PlotView, r0: Resolver, def: ViewRun["doc"]["cohorts"][string], spec: PlotRequest, + data: PlotData, unit: "patient" | "sample", site: string): Promise { + const f = (n: number) => n.toLocaleString("en-US"); + const cases = (n: number) => f(n) + " " + unit + (n === 1 ? "" : "s"); + if (data.shape === "table" || !data.n) return undefined; + const none = (why: string) => { data.pointSet = { rule: "none", drawn: 0, why }; }; + const densityNote = data.shape === "scatter" + ? "Drawn as density: " + cases(data.n) + " counted into " + f((data.bins || []).length) + " cells. The coefficients are over every " + + unit + ", not over what you can see." + : "Drawn as a density per group: " + f(data.n) + " values is past what a cloud of dots can show, and the quartiles are over every one of them."; + if (pv.points === false) { none("points: false"); return data.n > POINTS_CAP ? densityNote : undefined; } + if (!isPatientLevel(r0)) { + none(site + " may not show individual cases"); + return densityNote + " No case is drawn as a point: " + site + " may not show individual cases."; + } + const colourKey = (pts: { c?: string; r?: string }[]) => { + if (pv.colorBy) data.legend = legendFor(pts.map((x) => x.c || "")); + if (pv.ringBy) data.legend = [...(data.legend || []), ...legendFor(pts.map((x) => x.r || ""), true)]; + }; + + if (data.shape === "box") { + const all = data.n <= POINTS_CAP ? await r0.plotPoints(def, spec) : null; + if (all && all.length) { + data.points = all; data.overlay = "points"; + data.pointSet = { rule: "all", drawn: all.length }; + colourKey(all); + return undefined; + } + const groups = data.groups || []; + const outliers = await r0.plotPoints(def, spec, POINTS_CAP, groups); + if (!outliers) { + none("more than " + f(POINTS_CAP) + " values lie beyond the whiskers"); + return densityNote + " More than " + f(POINTS_CAP) + " values lie beyond the whiskers, too many to draw as points."; + } + data.points = outliers; + data.pointSet = { rule: "outliers", drawn: outliers.length }; + // the key describes what is drawn: the violin's classes, and any the outliers add + if (pv.colorBy || pv.ringBy) { + const fromViolin = data.legend || []; + colourKey(outliers); + const fromPoints = data.legend || []; + data.legend = [...fromViolin, ...fromPoints.filter((k) => !fromViolin.some((v) => v.label === k.label && !!v.ring === !!k.ring))]; + } + return densityNote + " " + (outliers.length + ? f(outliers.length) + " value" + (outliers.length === 1 ? "" : "s") + " beyond the whiskers " + + (outliers.length === 1 ? "is" : "are") + " drawn as points." + : "No value lies beyond the whiskers."); + } + + if (data.n <= POINTS_CAP) { + const all = await r0.plotScatterPoints(def, spec); + if (all && all.length) { + data.xyPoints = all; + data.pointSet = { rule: "all", drawn: all.length }; + return undefined; + } + } + if (!data.bins || !data.grid || !data.extent) return densityNote; + const cellMax = sparseCellMax(data.bins, POINTS_CAP, OUTLIER_CELL_MAX); + if (!cellMax) { + const lone = data.bins.reduce((a, b) => b.n === 1 ? a + b.n : a, 0); + none(f(lone) + " " + unit + "s sit alone in a cell, more than the " + f(POINTS_CAP) + " a figure draws as points"); + return densityNote + " " + f(lone) + " " + unit + "s sit alone in a cell, more than the " + f(POINTS_CAP) + + " drawn as points, so none is."; + } + const pts = await r0.plotScatterPoints(def, spec, POINTS_CAP, { grid: data.grid, extent: data.extent, cellMax }); + if (!pts) { none("the sparse cells hold more than " + f(POINTS_CAP) + " cases"); return densityNote; } + data.xyPoints = pts; + data.pointSet = { rule: "outliers", drawn: pts.length, cellMax }; + return densityNote + " " + cases(pts.length) + " in cells holding at most " + cellMax + " " + + (pts.length === 1 ? "is" : "are") + " drawn as points; the other " + cases(data.n - pts.length) + " as shaded cells."; +} + export async function runPlot(view: DocView, c: ViewRun): Promise { const { doc, cohorts, resolvers, vNames, participationFor } = c; const pv = view as PlotView; @@ -130,29 +211,7 @@ export async function runPlot(view: DocView, c: ViewRun): Promise x.c || "")); - if (pv.ringBy) data.legend = [...(data.legend || []), ...legendFor(pts.map((x) => x.r || ""), true)]; - } - } - const shapeNote = data.shape === "box" && data.overlay === "violin" - ? "Drawn as a density per group: " + data.n.toLocaleString("en-US") + " values is past what a cloud of dots can show, " + - "and the quartiles are over every one of them." - : data.shape === "box" && data.overlay === "points" - ? "Every value is drawn over its box; the jitter is fixed per case, so the figure does not move between renderings." - : data.shape === "scatter" && data.n > 5000 - ? "Drawn as density: " + data.n.toLocaleString("en-US") + " " + (unit === "sample" ? "samples" : "patients") + - " counted into " + (data.bins || []).length.toLocaleString("en-US") + " cells. The coefficients are over every " + - (unit === "sample" ? "sample" : "patient") + ", not over what you can see." - : undefined; + const shapeNote = await pointLayer(pv, r0, doc.cohorts[cn], spec, data, unit, sites[0]); const hs = await highlightLayers(pv, data, c, cn, sites, spec); if (hs.length) data.highlights = hs; // what the pair left out leads: cases missing either axis are not on the figure at all @@ -251,7 +310,7 @@ export async function runPlot(view: DocView, c: ViewRun): Promise isPatientLevel(resolvers[s]) ? resolvers[s].plotPoints(doc.cohorts[cn], spec) : Promise.resolve([]))); - const pts = perSite.flat(); + const pts = perSite.every((p) => p) ? perSite.flatMap((p) => p!) : []; if (pts.length && pts.length <= POINTS_CAP) { data.points = pts; data.overlay = "points"; if (pv.colorBy) data.legend = legendFor(pts.map((x) => x.c || "")); diff --git a/test/boundary.test.ts b/test/boundary.test.ts index 760ce5bf..c29cbeea 100644 --- a/test/boundary.test.ts +++ b/test/boundary.test.ts @@ -221,6 +221,13 @@ describe("remote peers: nothing identifying leaves before the network is touched "AggregateResolver must not be able to return the points behind a box"); assert.equal(typeof (PatientLevelResolver.prototype as unknown as Record).plotPoints, "function", "the entitled participant is the one that answers with points"); + // a scatter's own points (every case, or its sparse-cell outliers) are the same entitlement + for (const R of [AggregateResolver, RemoteAggregateResolver]) + assert.equal(typeof (R.prototype as unknown as Record).plotScatterPoints, "undefined", + R.name + " must not be able to return a scatter's points"); + assert.equal(typeof (PatientLevelResolver.prototype as unknown as Record).plotScatterPoints, "function"); + assert.match(plotScatterPointsSql("SELECT 1 AS id, 1 AS v", "SELECT 1 AS id, 2 AS v", "SELECT '' AS pid", 10, + { grid: [4, 4], extent: { x0: 0, x1: 1, y0: 0, y1: 1 }, cellMax: 2 }), /xs\.id AS id/); }); test("a highlight's points are the entitlement too; its aggregates are anyone's", () => { @@ -562,7 +569,7 @@ describe("patient-level federation is opt-in, on both sides, per study and kind" try { await fn(); } finally { for (const [k, v] of Object.entries(saved)) { if (v == null) delete process.env[k]; else process.env[k] = v; } } }; const PEERS = "s1=http://peer.invalid/api/mcp,s2=http://peer.invalid/api/mcp"; - const PATIENT_METHODS = ["members", "rowsPage", "rowsCounts", "rowsValues", "rowsRank", "plotPoints", "plotHighlightPoints", "coxRows", "logisticRows", "viewRows", "viewAggregate"]; + const PATIENT_METHODS = ["members", "rowsPage", "rowsCounts", "rowsValues", "rowsRank", "plotPoints", "plotScatterPoints", "plotHighlightPoints", "coxRows", "logisticRows", "viewRows", "viewAggregate"]; test("the rules: a study (or *) and the kinds allowed for it; a bare study is the timeline alone; an unknown kind is refused", () => { const r = parseRules("s1:timeline+alterations, s2 *:alterations", "X"); diff --git a/test/plot-highlight.test.ts b/test/plot-highlight.test.ts index ed888587..f178a03b 100644 --- a/test/plot-highlight.test.ts +++ b/test/plot-highlight.test.ts @@ -32,6 +32,10 @@ function fake(mode: "patient-level" | "aggregate-only", calls: Calls, o: { plotHighlightBoxRows: async () => { calls.push("boxRows"); return o.boxRows || [{ g: "A", n: 7, lo: 1, hi: 9, q1: 3, median: 5, q3: 7 }]; }, }; // the case-level method exists only where the entitlement does, exactly as on the real resolvers + if (mode === "patient-level") { + r.plotScatterPoints = async () => [{ id: "P1", x: 10, y: 5 }]; + r.plotPoints = async () => [{ id: "P1", g: "A", v: 5 }]; + } if (mode === "patient-level") r.plotHighlightPoints = async (_d: unknown, _h: unknown, _s: unknown, _xk: unknown, _yk: unknown, cap: number) => { calls.push("points cap=" + cap); diff --git a/test/plot-points.test.ts b/test/plot-points.test.ts new file mode 100644 index 00000000..b2724c32 --- /dev/null +++ b/test/plot-points.test.ts @@ -0,0 +1,143 @@ +// Which cases a plot draws as points: every case at or under the cap, the outliers over it (a +// box's values beyond its whiskers, a scatter's sparse cells), none from a participant that may +// not show cases. The resolvers are fakes that record their calls, so the test can say what was +// asked of whom, and what was NOT. +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { runPlot } from "../lib/views/plot.ts"; +import { POINTS_CAP, OUTLIER_CELL_MAX, sparseCellMax } from "../lib/plot.ts"; +import type { ViewRun, EvaluatedCohort } from "../lib/views/run.ts"; +import type { PlotData, ViewResult } from "../lib/types.ts"; + +const EXTENT = { x0: 0, x1: 100, y0: 0, y1: 50 }; +const scatter = (n: number, bins: { bx: number; by: number; n: number }[]): PlotData => ({ + shape: "scatter", n, x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" }, + extent: { ...EXTENT }, grid: [240, 160], bins, + stats: { pearson: 0.1, spearman: 0.1, slope: 1, intercept: 0, pPearson: 0.5, pSpearman: 0.5 } }); +const box = (n: number): PlotData => ({ shape: "box", n, x: { label: "CANCER_TYPE", kind: "cat" }, y: { label: "TMB", kind: "num" }, + groups: [{ g: "A", n, lo: 0, hi: 100, q1: 2, median: 4, q3: 6, loWhisker: 0, hiWhisker: 12 }], overlay: "violin", violin: [] }); + +type Call = { m: string; cap?: number; extra?: unknown }; +function fake(mode: "patient-level" | "aggregate-only", calls: Call[], data: () => PlotData, answers: { + all?: unknown[] | null; outliers?: unknown[] | null; +} = {}) { + const r: Record = { + instance: { mode, conn: { db: "db" }, label: mode }, + plot: async () => data(), + }; + if (mode === "patient-level") { + r.plotScatterPoints = async (_d: unknown, _s: unknown, cap?: number, sparse?: unknown) => { + calls.push({ m: "scatterPoints", cap, extra: sparse }); + return sparse ? (answers.outliers === undefined ? [{ id: "o1", x: 99, y: 49 }] : answers.outliers) + : (answers.all === undefined ? [{ id: "a1", x: 1, y: 1 }, { id: "a2", x: 2, y: 2 }] : answers.all); + }; + r.plotPoints = async (_d: unknown, _s: unknown, cap?: number, outside?: unknown) => { + calls.push({ m: "boxPoints", cap, extra: outside }); + return outside ? (answers.outliers === undefined ? [{ id: "o1", g: "A", v: 90 }] : answers.outliers) + : (answers.all === undefined ? [{ id: "a1", g: "A", v: 3 }] : answers.all); + }; + } + return r; +} + +const evaluated = (studies: string[], count: number): EvaluatedCohort => + ({ studies, perStudy: Object.fromEntries(studies.map((s) => [s, { sql: "", count, mode: "patient-level" }])), count, text: "" }); + +async function run(resolver: Record, view: Record = {}): Promise { + const c = { + doc: { cohorts: { All: { all: [] } } }, + cohorts: { All: evaluated(["s"], 1000) }, resolvers: { s: resolver }, vNames: ["All"], vStudies: ["s"], + participationFor: (studies: string[]) => ({ studies: studies.map((s) => ({ study: s, mode: "patient-level", instance: "" })), requiresPatientLevel: false, aggregateOnly: [] }), + pin: null, studyProfiles: async () => ({ profiles: [], gaProfiles: [] }), + } as unknown as ViewRun; + const v = { kind: "plot", cohorts: ["All"], x: { source: "clinical", attribute: "AGE" }, y: { source: "clinical", attribute: "TMB" }, ...view }; + const r = await runPlot(v as never, c); + return "result" in (r as object) ? (r as { result: ViewResult }).result : r as ViewResult; +} + +describe("sparseCellMax: the cell threshold for a scatter's outlier points", () => { + test("the largest threshold whose cases fit under the cap, never above the maximum", () => { + const bins = [{ n: 1 }, { n: 1 }, { n: 2 }, { n: 3 }, { n: 50 }]; + assert.equal(sparseCellMax(bins, 100, 3), 3, "1+1+2+3 = 7 cases fit"); + assert.equal(sparseCellMax(bins, 4, 3), 2, "1+1+2 = 4 fit; adding the 3 does not"); + assert.equal(sparseCellMax(bins, 100, 2), 2, "held to the maximum asked for"); + assert.equal(sparseCellMax(bins, 1, 3), 0, "the lone cases alone are 2, more than 1: none is drawn"); + assert.equal(sparseCellMax([{ n: 50 }], 100, 3), 3, "no sparse cell at all draws nothing, at any threshold"); + }); +}); + +describe("a scatter's points", () => { + test("at or under the cap every case is a point and no subset is asked for", async () => { + const calls: Call[] = []; + const r = await run(fake("patient-level", calls, () => scatter(2, [{ bx: 0, by: 0, n: 2 }]))); + assert.deepEqual(r.plot!.pointSet, { rule: "all", drawn: 2 }); + assert.equal(r.plot!.xyPoints!.length, 2); + assert.deepEqual(calls.map((c) => [c.m, c.extra]), [["scatterPoints", undefined]]); + }); + + test("over the cap the cases in sparse cells are points, against the plot's own grid and extent", async () => { + const calls: Call[] = []; + const n = POINTS_CAP + 1000; + const bins = [{ bx: 0, by: 0, n: n - 4 }, { bx: 5, by: 5, n: 1 }, { bx: 9, by: 9, n: 3 }]; + const r = await run(fake("patient-level", calls, () => scatter(n, bins))); + assert.equal(calls.length, 1, "over the cap the every-case query is not run"); + assert.deepEqual(calls[0].extra, { grid: [240, 160], extent: EXTENT, cellMax: OUTLIER_CELL_MAX }); + assert.equal(calls[0].cap, POINTS_CAP); + assert.deepEqual(r.plot!.pointSet, { rule: "outliers", drawn: 1, cellMax: OUTLIER_CELL_MAX }); + assert.match(r.note!, /1 patient in cells holding at most 3 is drawn as points; the other 5,999 patients as shaded cells/); + assert.match(r.note!, /coefficients are over every patient/); + }); + + test("when even the lone cases are more than the cap, none is drawn and the caption says why", async () => { + const calls: Call[] = []; + const lone = Array.from({ length: POINTS_CAP + 1 }, (_, i) => ({ bx: i % 240, by: Math.floor(i / 240), n: 1 })); + const r = await run(fake("patient-level", calls, () => scatter(POINTS_CAP + 1, lone))); + assert.equal(calls.length, 0, "nothing is fetched to be thrown away"); + assert.equal(r.plot!.pointSet!.rule, "none"); + assert.equal(r.plot!.xyPoints, undefined); + assert.match(r.note!, /5,001 patients sit alone in a cell, more than the 5,000 drawn as points, so none is/); + }); + + test("an aggregate-only participant is never asked, and the caption names it", async () => { + const calls: Call[] = []; + const r = await run(fake("aggregate-only", calls, () => scatter(10, [{ bx: 0, by: 0, n: 10 }]))); + assert.equal(calls.length, 0); + assert.deepEqual(r.plot!.pointSet, { rule: "none", drawn: 0, why: "s may not show individual cases" }); + assert.match(r.note!, /No case is drawn as a point: s may not show individual cases/); + }); + + test("points: false asks for none", async () => { + const calls: Call[] = []; + const r = await run(fake("patient-level", calls, () => scatter(10, [{ bx: 0, by: 0, n: 10 }])), { points: false }); + assert.equal(calls.length, 0); + assert.equal(r.plot!.pointSet!.rule, "none"); + }); +}); + +describe("a box's points", () => { + test("at or under the cap every value is a point", async () => { + const calls: Call[] = []; + const r = await run(fake("patient-level", calls, () => box(10))); + assert.deepEqual(r.plot!.pointSet, { rule: "all", drawn: 1 }); + assert.equal(r.plot!.overlay, "points"); + assert.deepEqual(calls.map((c) => c.m), ["boxPoints"]); + }); + + test("over the cap the values beyond each group's whiskers are points over the violin", async () => { + const calls: Call[] = []; + const r = await run(fake("patient-level", calls, () => box(POINTS_CAP + 1))); + assert.equal(calls.length, 1, "over the cap the every-value query is not run"); + assert.deepEqual(calls[0].extra, [{ g: "A", n: POINTS_CAP + 1, lo: 0, hi: 100, q1: 2, median: 4, q3: 6, loWhisker: 0, hiWhisker: 12 }]); + assert.equal(r.plot!.overlay, "violin", "the violin stays: the outliers are drawn over it"); + assert.deepEqual(r.plot!.pointSet, { rule: "outliers", drawn: 1 }); + assert.match(r.note!, /1 value beyond the whiskers is drawn as points/); + }); + + test("more outliers than the cap: none, never the first 5,000", async () => { + const calls: Call[] = []; + const r = await run(fake("patient-level", calls, () => box(POINTS_CAP * 3), { outliers: null })); + assert.equal(r.plot!.points, undefined); + assert.equal(r.plot!.pointSet!.rule, "none"); + assert.match(r.note!, /More than 5,000 values lie beyond the whiskers, too many to draw as points/); + }); +}); diff --git a/test/plot.test.ts b/test/plot.test.ts index 0a07fd48..6a7bf829 100644 --- a/test/plot.test.ts +++ b/test/plot.test.ts @@ -4,7 +4,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { axisKind, axisLabel, axisSelect, corrPValue, plotBinsSql, plotBoxSql, plotCountSql, plotShape, plotStatsSql, plotTableSql, plotPointsSql, plotViolinSql, categoryColor, isAbsent, - tukey, validateAxis, CATEGORY_CAP, DEFAULT_BINS , coverageNote } from "../lib/plot.ts"; + tukey, validateAxis, CATEGORY_CAP, DEFAULT_BINS , coverageNote, plotScatterPointsSql } from "../lib/plot.ts"; import { STYLE } from "../lib/oncoprint-style.ts"; const ctx = { study: "x", db: "db", unit: "sample" as const, cohortSql: "SELECT pid FROM p" }; @@ -225,3 +225,33 @@ test("unaltered is absence: neutral, and never a ring", () => { assert.equal(isAbsent("Unaltered"), true); assert.equal(isAbsent("Altered"), false); }); + +test("a box's outlier query keeps the values beyond their own group's whiskers, labels quoted", () => { + const numSel = "SELECT 1 AS id, 1 AS v", catSel = "SELECT 1 AS id, 'g' AS v"; + const sql = plotPointsSql(numSel, catSel, "SELECT '' AS pid", 100, undefined, undefined, + [{ g: "Missense", loWhisker: 0.5, hiWhisker: 12 }, { g: "it's", loWhisker: -1, hiWhisker: 3 }]); + assert.match(sql, /indexOf\(\['Missense', 'it\\'s'\], gs\.v\)/, "a label is a quoted literal, never spliced"); + assert.match(sql, /vs\.v < arrayElement\(\[0\.5, -1\]/); + assert.match(sql, /vs\.v > arrayElement\(\[12, 3\]/); + assert.match(sql, /WHERE indexOf\([^)]*\) > 0 AND/, "a group with no whiskers matches nothing"); + assert.match(plotPointsSql(numSel, catSel, "SELECT '' AS pid", 100, undefined, undefined, []), /WHERE 0/); + assert.throws(() => plotPointsSql(numSel, catSel, "SELECT '' AS pid", 100, undefined, undefined, + [{ g: "A", loWhisker: NaN, hiWhisker: 1 }]), /finite/); +}); + +test("a scatter's sparse cells are binned exactly as its density is", () => { + const x = "SELECT 1 AS id, 1 AS v", y = "SELECT 1 AS id, 2 AS v"; + const extent = { x0: 0, x1: 696, y0: 0, y1: 1 }; + const bins = plotBinsSql(x, y, "SELECT '' AS pid", [240, 160], extent); + const pts = plotScatterPointsSql(x, y, "SELECT '' AS pid", 5000, { grid: [240, 160], extent, cellMax: 3 }); + // the same cell expression on both sides, so a point is in a cell the density did not shade + for (const axis of ["x", "y"]) { + const expr = bins.match(new RegExp("least\\(\\d+, toUInt32\\(floor\\(\\(" + axis + " - [^\\n]*?\\)\\)\\)"))![0]; + assert.ok(pts.includes(expr), axis + ": " + expr); + } + assert.match(pts, /HAVING count\(\) <= 3/); + assert.match(pts, /0 AS x0, 696 AS x1, 0 AS y0, 1 AS y1/); + assert.match(pts, /LIMIT 5001$/, "one past the cap, so an overflow is seen and refused rather than cut"); + assert.throws(() => plotScatterPointsSql(x, y, "SELECT '' AS pid", 5000, { grid: [240, 160], extent, cellMax: 0 }), /cellMax/); + assert.throws(() => plotScatterPointsSql(x, y, "SELECT '' AS pid", 5000, { grid: [240, 160], extent: { ...extent, x1: Infinity }, cellMax: 2 }), /finite/); +}); diff --git a/test/view-svg.test.ts b/test/view-svg.test.ts index f18d3416..48f06881 100644 --- a/test/view-svg.test.ts +++ b/test/view-svg.test.ts @@ -402,6 +402,46 @@ describe("plot highlights: other cohorts drawn over the plotted one", () => { }); }); +describe("plot points: every case, or the outliers over the density", () => { + const base = { x: { label: "AGE", kind: "num" }, y: { label: "TMB", kind: "num" }, + extent: { x0: 0, x1: 10, y0: 0, y1: 10 }, grid: [10, 10], + stats: { pearson: 0.1, spearman: 0.1, slope: 0, intercept: 0, pPearson: 0.5, pSpearman: 0.5 } }; + const cells = (svg: string) => (svg.match(/ (svg.match(/ { + const svg = viewToSvg({ kind: "plot", plot: { ...base, shape: "scatter", n: 3, bins: [{ bx: 1, by: 1, n: 2 }, { bx: 5, by: 5, n: 1 }], + xyPoints: [{ id: "a", x: 1, y: 1 }, { id: "b", x: 1.2, y: 1.1 }, { id: "c", x: 5, y: 5 }], pointSet: { rule: "all", drawn: 3 } } as any })!; + assert.equal(cells(svg), 0); + assert.equal(dots(svg), 3); + assert.match(svg, /Each dot is one case\./); + assert.doesNotMatch(svg, /Darker is more/); + }); + + test("outliers: the sparse cells are left to their points, the dense ones stay shaded", () => { + const svg = viewToSvg({ kind: "plot", plot: { ...base, shape: "scatter", n: 105, + bins: [{ bx: 1, by: 1, n: 100 }, { bx: 2, by: 2, n: 3 }, { bx: 9, by: 9, n: 2 }], + xyPoints: [{ id: "a", x: 2.1, y: 2.1 }, { id: "b", x: 2.2, y: 2.3 }, { id: "c", x: 2.4, y: 2.2 }, { id: "d", x: 9.5, y: 9.5 }, { id: "e", x: 9.6, y: 9.1 }], + pointSet: { rule: "outliers", drawn: 5, cellMax: 3 } } as any })!; + assert.equal(cells(svg), 1, "only the cell of 100 is shaded"); + assert.equal(dots(svg), 5); + assert.match(svg, /Darker is more cases; the shade is logarithmic\. Each dot is one case\./); + }); + + test("a box's axis reaches its outliers, at most a whisker span past the whiskers; beyond that a counted triangle", () => { + const svg = viewToSvg({ kind: "plot", plot: { shape: "box", n: 1000, x: { label: "CANCER_TYPE", kind: "cat" }, y: { label: "TMB", kind: "num" }, + groups: [{ g: "A", n: 1000, lo: 0, hi: 570, q1: 2, median: 4, q3: 6, loWhisker: 0, hiWhisker: 10 }], + overlay: "violin", violin: [{ g: "A", b: 0, n: 500 }, { g: "A", b: 15, n: 3 }], violinBins: 40, violinExtent: [0, 40], + points: [{ id: "a", g: "A", v: 15 }, { id: "b", g: "A", v: 570 }], pointSet: { rule: "outliers", drawn: 2 } } as any })!; + assert.match(svg, />20<\/text>/, "the axis tops out at whisker + one whisker span (10 + 10)"); + assert.equal((svg.match(/ { const box = (nGroups: number, extra: Partial = {}): ViewResult => ({ kind: "plot", plot: { From 593299ce9c5fde03ec3f92affe899b4c765c4771 Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Tue, 13 Oct 2026 18:46:01 +0000 Subject: [PATCH 2/5] Plot points: one radius on every figure Each layer used to size its points from its own count: box points from 1.3 to 4.5 px, scatter points from 1.4 to 3.4, highlights from 1.6 to 4.2. So the 77 outliers over a violin were drawn at 4.5 px, nearly twice the size of the 1,224 scatter points beside them. PLOT_POINT_R (2.5) is now the radius of every plotted case: scatter or box, plotted cohort or highlight, whatever the count. A ring no longer enlarges its dot; it is a stroke on the same circle. Legend swatches are not cases and keep their size. A test asserts that a scatter with 2 points, one with 4,000, and a box plot with a ringed point all use the same radius. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- lib/figures/plot.ts | 24 +++++++++++++----------- test/view-svg.test.ts | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/lib/figures/plot.ts b/lib/figures/plot.ts index 998d148b..278b5b12 100644 --- a/lib/figures/plot.ts +++ b/lib/figures/plot.ts @@ -16,6 +16,12 @@ import { type Pal = ReturnType; const r1 = (n: number) => Math.round(n * 10) / 10; +/** + * The radius of every case drawn as a point on a plot — scatter or box, the plotted cohort or a + * highlight, five points or five thousand — so a dot means the same thing on every figure. + */ +export const PLOT_POINT_R = 2.5; + /** the plot's height for a figure `w` wide */ const plotHeight = (w: number) => Math.round(Math.max(240, Math.min(420, w * 0.36))); @@ -94,18 +100,15 @@ function scatter(v: ViewResult, d: PlotData, W: number, p: Pal): string { // under highlights the plotted cohort is a floor-raised grey, so its faintest cells still read for (const b of shaded) out.push(cell(b, base, hs.length ? 0.3 + 0.7 * shade(b.n) : shade(b.n))); const own = d.xyPoints || []; - const rOwn = Math.max(1.4, Math.min(3.4, 80 / Math.sqrt(Math.max(1, own.length)))); for (const pt of own) - out.push(``); + out.push(``); // a highlight too large to draw as points is cells on the same grid, shaded by its own densest cell for (const h of hs) if (h.shown === "density" && h.bins) { const hMax = Math.max(1, ...h.bins.map((b) => b.n)); for (const b of h.bins) out.push(cell(b, h.color, 0.3 + 0.7 * Math.log1p(b.n) / Math.log1p(hMax))); } - const nPts = hs.reduce((a, h) => a + (h.shown === "points" ? h.points!.length : 0), 0); - const rPt = Math.max(1.8, Math.min(3.6, 70 / Math.sqrt(Math.max(1, nPts)))); for (const h of hs) if (h.shown === "points") for (const pt of h.points!) if ("x" in pt) - out.push(``); + out.push(``); // the fitted line, over every case rather than over what is visible, clipped to the frame if (st && st.slope != null && st.intercept != null && x1 > x0) { const yAt = (xv: number) => st.intercept! + st.slope! * xv; @@ -233,8 +236,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { } } if (byGroup.has(g.g)) { - // two encodings on one dot need a dot big enough to carry them; a crowded plot gets smaller ones - const rBase = Math.max(1.4, Math.min(4.5, 90 / Math.sqrt(Math.max(1, d.points!.length)))); + const rBase = PLOT_POINT_R; for (const pt of byGroup.get(g.g)!) { const fill = pt.c != null && colorKey.length ? fillOf(pt.c) : base; const ring = pt.r != null && !isAbsent(pt.r) ? ringOf(pt.r) : null; @@ -244,8 +246,8 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { continue; } out.push(``); + `r="${rBase}" fill="${fill}" fill-opacity="0.85"` + + (ring ? ` stroke="${ring}" stroke-width="${r1(rBase * 0.36)}"` : "") + `/>`); } } out.push(``); @@ -267,7 +269,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { // a highlighted value past the axis is pinned to the frame's edge as a triangle pointing off // it, and counted in the caption, rather than drawn outside the figure for (const h of hs) if (h.shown === "points") { - const rH = Math.max(1.8, Math.min(4.2, 80 / Math.sqrt(Math.max(1, h.points!.length)))); + const rH = PLOT_POINT_R; for (const pt of h.points!) if ("g" in pt && pt.g === g.g) { const px = cx + jitterFor(pt.id) * half * 0.9; if (pt.v > vhi || pt.v < vlo) { @@ -275,7 +277,7 @@ function box(v: ViewResult, d: PlotData, W0: number, p: Pal): string { out.push(edgeMark(px, pt.v > vhi, rH, h.color)); continue; } - out.push(``); + out.push(``); } } }); diff --git a/test/view-svg.test.ts b/test/view-svg.test.ts index 48f06881..463dde5c 100644 --- a/test/view-svg.test.ts +++ b/test/view-svg.test.ts @@ -4,6 +4,7 @@ import { test, describe } from "node:test"; import assert from "node:assert/strict"; import { viewToSvg, viewTitle, esc } from "../lib/view-svg.ts"; +import { PLOT_POINT_R } from "../lib/figures/plot.ts"; import { buildLollipopGene } from "../lib/lollipop.ts"; import type { ViewResult } from "../lib/types.ts"; import { HIGHLIGHT_BASE, HIGHLIGHT_COLORS } from "../lib/plot-highlight.ts"; @@ -428,6 +429,21 @@ describe("plot points: every case, or the outliers over the density", () => { assert.match(svg, /Darker is more cases; the shade is logarithmic\. Each dot is one case\./); }); + test("a point is the same size on every plot, whatever the count or shape", () => { + // a plotted case carries a fill-opacity; a legend swatch does not, and is not a case + const radii = (svg: string) => new Set([...svg.matchAll(/]*\br="([^"]+)"[^>]*fill-opacity/g)].map((m) => m[1])); + const few = viewToSvg({ kind: "plot", plot: { ...base, shape: "scatter", n: 2, bins: [{ bx: 1, by: 1, n: 2 }], + xyPoints: [{ id: "a", x: 1, y: 1 }, { id: "b", x: 2, y: 2 }], pointSet: { rule: "all", drawn: 2 } } as any })!; + const many = viewToSvg({ kind: "plot", plot: { ...base, shape: "scatter", n: 4000, bins: [{ bx: 1, by: 1, n: 4000 }], + xyPoints: Array.from({ length: 4000 }, (_, i) => ({ id: "p" + i, x: (i % 100) / 10, y: Math.floor(i / 100) / 4 })), + pointSet: { rule: "all", drawn: 4000 } } as any })!; + const boxed = viewToSvg({ kind: "plot", plot: { shape: "box", n: 3, x: { label: "G", kind: "cat" }, y: { label: "V", kind: "num" }, + groups: [{ g: "A", n: 3, lo: 0, hi: 10, q1: 2, median: 4, q3: 6, loWhisker: 0, hiWhisker: 10 }], overlay: "points", + points: [{ id: "a", g: "A", v: 1 }, { id: "b", g: "A", v: 5, r: "Amplification" }, { id: "c", g: "A", v: 9 }], + legend: [{ label: "Amplification", color: "#f00", ring: true }], pointSet: { rule: "all", drawn: 3 } } as any })!; + for (const svg of [few, many, boxed]) assert.deepEqual([...radii(svg)], [String(PLOT_POINT_R)]); + }); + test("a box's axis reaches its outliers, at most a whisker span past the whiskers; beyond that a counted triangle", () => { const svg = viewToSvg({ kind: "plot", plot: { shape: "box", n: 1000, x: { label: "CANCER_TYPE", kind: "cat" }, y: { label: "TMB", kind: "num" }, groups: [{ g: "A", n: 1000, lo: 0, hi: 570, q1: 2, median: 4, q3: 6, loWhisker: 0, hiWhisker: 10 }], From ff9396c7d9aa12a8196379bc9904038b2cfafcb9 Mon Sep 17 00:00:00 2001 From: Aaron Lisman Date: Fri, 16 Oct 2026 04:53:44 +0000 Subject: [PATCH 3/5] Plot: axes defined in a sidebar, as on cBioPortal's Plots tab The plot card's controls move from one toolbar row into the Plots tab's layout. A left column holds a "Horizontal axis" block and a "Vertical axis" block, each titled down its side, with "Swap axes" between them. Each block's fields are named and stacked: Data type, then what that source needs (Clinical attribute; Molecular profile and Gene; Group cases by and Gene(s) or OQL query), then Log scale. Below the blocks are "Each point is a" (patient or sample) and "Highlight cohorts". On the right, a line says how many cases have data on both axes, then the box plot's "colour samples by" fill/ring row, then the figure. Under a narrow card (container query, 48rem) the sidebar sits above the figure. The layout is PlotControls (app/lab/plot-controls.tsx), which now takes the figure as its child, so the lab's plot card and the Results page's Plots tab share it. AxisPicker and HighlightPicker take `stacked` for the sidebar (field names, full width); the inline form is unchanged for the Cox and logistic cards. A box is always drawn with its categories across, so a box whose categories are in the document's y says that under the horizontal block, and Swap makes the two agree. For a box already drawn as the document says, Swap is disabled and says why. Verified against cbioportal.org's Plots tab (BRCA PanCancer, TP53 mutation type vs FGA) by screenshot, at 1500 and 760 px, with no console errors. A new e2e check asserts both blocks, the field names, the figure beside the sidebar, one control height, and that Swap rewrites x and y. Gate: all stages pass. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BhP3GAdWXr2EtfBDHvRS1L --- app/lab/plot-controls.tsx | 133 ++++++++++++++++++++++++-------------- app/lab/view-cards.tsx | 5 +- app/lab/view-chrome.tsx | 85 +++++++++++++----------- app/portal/PortalTabs.tsx | 22 ++++--- e2e.cjs | 28 ++++++++ 5 files changed, 179 insertions(+), 94 deletions(-) diff --git a/app/lab/plot-controls.tsx b/app/lab/plot-controls.tsx index 6843214f..9d478dbd 100644 --- a/app/lab/plot-controls.tsx +++ b/app/lab/plot-controls.tsx @@ -1,13 +1,14 @@ "use client"; -// The plot view's settings bar, shared by the lab's plot card and the portal pages' Plots tab: -// both axes (with a log scale for a numeric one), a box plot's colour and ring, the highlighted -// cohorts, the unit and the axis swap. Every control is an edit to the view, handed to `update`. -import { ArrowLeftRight } from "lucide-react"; +// The plot view laid out as cBioPortal's Plots tab is, shared by the lab's plot card and the portal +// pages' Plots tab: each axis defined in its own block in a sidebar (horizontal above vertical, a swap +// between them, the cases and highlights below), and beside it a line saying how many cases are shown, +// a box plot's colour and ring, and the figure. Under a narrow card the sidebar sits above the figure. +// Every control is an edit to the view, handed to `update`. +import { ArrowDown, ArrowUp } from "lucide-react"; import { AxisPicker, HighlightPicker } from "./view-chrome.tsx"; import { Button } from "../_ui/button"; import { Checkbox } from "../_ui/checkbox"; import { Choice } from "../_ui/choice.tsx"; -import { TB } from "../_ui/toolbar.ts"; type Update = (fn: (view: any) => any) => void; @@ -15,68 +16,104 @@ function LogToggle({ spec, side, numeric, busy, update }: { spec: any; side: "x" const axis = spec[side]; if (!numeric || !axis || !("source" in axis) || (axis.source !== "clinical" && axis.source !== "molecular")) return null; return ( -