diff --git a/.claude/commands/mxcli-dev/review.md b/.claude/commands/mxcli-dev/review.md index 76fa4079d5..d87abdf933 100644 --- a/.claude/commands/mxcli-dev/review.md +++ b/.claude/commands/mxcli-dev/review.md @@ -119,6 +119,15 @@ All executor code must go through the backend abstraction layer. **`sdk/mpr` no - [ ] **Map iteration is deterministic** — any map iterated for serialization output must sort keys first (`sort.Strings(keys)` pattern); non-deterministic output causes flaky diffs and BSON instability - [ ] **Pluggable widgets via WidgetEngine** — new pluggable widget support uses `.def.json` + `WidgetRegistry`; no hardcoded BSON widget builders in the executor +### Catalog builder passes +A refresh rebuilds only the units a save changed and keeps every other row of the cached catalog (ako/mxcli#1804). A pass that breaks the rules below is still correct, just slower on every refresh, which no correctness test notices. See [ADR-0014](docs/13-decisions/0014-incremental-catalog-builds.md). +- [ ] **Rows have an owner**: each per-document loop starts with `if !b.own(unitID) { continue }` (an entity's unit is its domain model) and ends with `b.disown()`. A table with rows from no document belongs in `projectWideTables` with a reason (`TestCatalog_RowsBelongToADocument`). +- [ ] **Listed through the scope**: documents come from `b.lister()` or a `cached*` accessor, not `b.reader.List*`, which decodes the whole project (`TestBuilderListsDocumentsThroughTheScope`, `TestCatalog_IncrementalRefreshDecodesOnlyWhatItRebuilds`). +- [ ] **Other documents through the tables**: a pass that needs another document's name or shape reads the catalog tables an earlier pass wrote (see `associationEnds`, `entityNamesByID`), not a decoded list, which holds only the rebuilt units. +- [ ] **Dependencies are refs**: if a document's rows depend on another document, the document has a `refs` edge to it, or the dependency is invisible to the plan. Add a step to `incrementalSteps` where the other document changes and this one does not; it must fail with the dependency removed. +- [ ] **New lookups are traced**: a new lookup of another document's name calls `b.dependsOn(site, name)`, or goes through the member-ref index, which probes for you. `TestCatalog_RefsCoverEveryCrossDocumentRead` must stay at 100%; `MXCLI_REF_COVERAGE_MPR= go test ./mdl/executor -run TestCatalog_RefCoverageOfProject -v` measures a real project. +- [ ] **New document type**: an incremental step for it in `incrementalSteps`. A type no per-document pass owns makes every change to it a full build. + ### Full-stack consistency for MDL features New MDL commands or language features must be wired through the full pipeline: - [ ] **Grammar** — rule added to `MDLParser.g4` (and `MDLLexer.g4` if new tokens) diff --git a/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-30-bootstrap-script-asset-name-from-uname-on-windows.json b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-30-bootstrap-script-asset-name-from-uname-on-windows.json new file mode 100644 index 0000000000..19139a7540 --- /dev/null +++ b/.claude/skills/fix-issue/findings/cmd-mxcli/2026-10-30-bootstrap-script-asset-name-from-uname-on-windows.json @@ -0,0 +1 @@ +{"area": "cmd/mxcli", "date": "2026-10-30", "symptom": "`.claude/bootstrap-mxcli.sh` on Windows (Git Bash) with no mxcli on PATH: `curl: (22) The requested URL returned error: 404` for `.../nightly/mxcli-mingw64_nt-10.0-26200-amd64`, and the session starts without mxcli because the hook runs `sh .claude/bootstrap-mxcli.sh || true`", "cause": "The download asset name was `mxcli-$(uname -s | tr A-Z a-z)-`. Under Git Bash, MSYS2 and Cygwin `uname -s` is `MINGW64_NT-10.0-26200` / `MSYS_NT-…` / `CYGWIN_NT-…`, never `windows`, and the Windows release assets carry `.exe`. The script also hard-coded `./mxcli` as its binary path everywhere", "file": "`cmd/mxcli/init_hook.go` (`bootstrapScriptTemplate`); tests `cmd/mxcli/init_hook_platform_test.go`", "insight": "**`uname -s` is a kernel name, not a GOOS.** Mapping it to the release asset needs the same table the Makefile `release` target uses for its output names (linux/darwin/windows plus `.exe`), and that table is easy to check against. The script's tests only ran on Linux, so no test ever hit the Windows branch, and `requireSh` skips Windows CI anyway. A fake `uname` on PATH reaches every platform branch from a Linux runner, and the curl stand-in must log the URL: before this fix it logged only the output file, which is why a wrong asset name could not fail any test. Detect the platform once at the top and keep the binary path in one variable (`$MXCLI` = `./mxcli` or `./mxcli.exe`); otherwise every `[ -x ./mxcli ]` guard has to rely on MSYS's implicit `.exe` lookup. On Windows, remove a stray non-executable `./mxcli` (the Linux build `mxcli new` leaves) before installing `./mxcli.exe`, because Git Bash resolves the exact name first. Remove it before the install and not after, since Cygwin's `rm ./mxcli` can also take `mxcli.exe`", "refs": ["mendixlabs/mxcli#1680"]} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-every-refresh-rebuilds-every-table.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-every-refresh-rebuilds-every-table.json new file mode 100644 index 0000000000..fdfef3ba2b --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-every-refresh-rebuilds-every-table.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-10-30", "symptom": "Every catalog refresh rebuilt every table from every unit after any save: about 4.5 s of decoding and inserting on a 4,753-unit app, most of it for rows identical to the cached ones (5.6-8 s end to end per refresh)", "cause": "The builder had no record of which unit a row came from, so it could neither keep an unchanged unit's rows nor drop a changed unit's; every pass listed and decoded every document of its type", "file": "`mdl/catalog/builder_tx.go` (`builderTx` / `Stmt` record rowid ranges per owner), `mdl/catalog/builder_incremental.go` (`loadPrevious`, `planIncremental`, `dependentUnits`, `dropRebuiltRows`, `verifyIncremental`, `persistOwners`, `clearCatalog`), `mdl/catalog/builder.go` (`buildOn`, `buildAttempt`, phase list), `own()` at the head of every per-document loop, `mdl/backend/modelsdk/units.go` (`WithUnitScope`), `modelsdk/mpr/reader_units.go` (`ListUnitsByTypeWhere`)", "insight": "**Record ownership where rows are written, not per table.** A wrapper on the build's transaction notes the rowid range of every INSERT under the pass's current owner, so 130 insert sites needed no change. The per-document loops each call `own(unit)`, and a row written while the pass is skipping a document fails the build back to a full one, which caught a bug on its own: the backend lists System enumerations that are no units. Start from the cached catalog (sqlite3_deserialize) and delete the rebuilt units' rows rather than copying the kept ones: copying re-tokenized every carried full-text row and cost more than the rest of the build. Deserializing the same connection twice crashed SQLite (modernc) once its full-text tables had been used, so plan on a scratch copy of the same bytes and empty the tables with DELETE to fall back. The equivalence test passed its control at first because mxcli edits rewrite the documents that name what they change; only a step where a document's rows change while its unit does not (a delete over a repointed association) shows the dependents are needed", "fix": "Rebuild only changed units and the documents whose refs name them (members by prefix, specializations transitively), list and decode only those through a unit-scoped backend view, verify after the build that no document names a changed unit's new names (else widen once), fall back to a full build for units no per-unit pass reads (folders, modules, settings, security, navigation). Tests: `TestCatalog_IncrementalBuildEqualsFullBuild` and `TestCatalog_IncrementalFastBuildEqualsFullBuild` (every table vs a forced full build over 19 edit steps, chained incrementally), the source test, and unit tests for ranges, the skipped-document error and dependents"} diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-string-function-compared-with-empty-always-true.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-string-function-compared-with-empty-always-true.json new file mode 100644 index 0000000000..1cbde599f2 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-30-string-function-compared-with-empty-always-true.json @@ -0,0 +1 @@ +{"area": "mdl/exprcheck", "date": "2026-10-30", "symptom": "`if trim($Value) != empty` passed `check --references`, `exec` and `mx check`, and was always true: a blank or empty value took the 'has text' branch and crashed later in the flow.", "cause": "Not an mxcli write defect — a platform semantics gap. Mendix String functions (trim, toLowerCase, toUpperCase, substring, replaceAll, toString) return '' for an empty input, never empty, so the comparison is constant. exprcheck had no rule for it; the expression is well-typed, so no kind check could see it.", "file": "`mdl/exprcheck/parser.go` (checkStringFuncComparedWithEmpty, called from parseCmp)", "insight": "Settled by measurement, not by reading docs: probe microflows run as `mxcli test --local --db-type hsqldb` on a fresh `mxcli new` 10.18.5 app (no Postgres needed), each classifying f(x) as EMPTY / '' / text for x in empty, '', blanks — with the bare variable as the control that DOES compare equal to empty. The same probe found `empty + ''` is the text 'null'; that one is a pitfalls entry, not a warning, because whether an operand is empty is not knowable at check time. Warn only on measured functions.", "refs": ["mendixlabs/mxcli#580"]} diff --git a/.claude/skills/mendix/check-syntax/SKILL.md b/.claude/skills/mendix/check-syntax/SKILL.md index 0a0d2a741b..a86679204f 100644 --- a/.claude/skills/mendix/check-syntax/SKILL.md +++ b/.claude/skills/mendix/check-syntax/SKILL.md @@ -11,15 +11,15 @@ description: "Validate MDL with `mxcli check` before presenting or executing it, | lines | section | |---|---| | 29-35 | When to Use This Skill | -| 36-286 | `exec` refuses what `check` rejects | -| 287-422 | Pre-Flight Validation Checklist | -| 423-458 | Two rules that only real validation used to catch | -| 459-505 | Validation Workflow | -| 506-527 | Script Execution Behavior | -| 528-588 | Script Organization | -| 589-641 | Troubleshooting Parse Errors | -| 642-699 | Studio Pro MCP — Verification Only | -| 700-705 | Related Skills | +| 36-295 | `exec` refuses what `check` rejects | +| 296-431 | Pre-Flight Validation Checklist | +| 432-467 | Two rules that only real validation used to catch | +| 468-514 | Validation Workflow | +| 515-536 | Script Execution Behavior | +| 537-597 | Script Organization | +| 598-650 | Troubleshooting Parse Errors | +| 651-708 | Studio Pro MCP — Verification Only | +| 709-714 | Related Skills | # MDL Syntax Validation Skill @@ -272,6 +272,15 @@ reported by `check`: - **`empty($List)`** — E014. `empty` is a Mendix **keyword**, not a function, so the parser stops at the `(`. Write `$List = empty` or `length($List) = 0`. +A **warning** rather than a refusal, because the model is valid and builds: + +- **`trim($v) != empty`** — E015. A String function (`trim`, `toLowerCase`, + `toUpperCase`, `substring`, `replaceAll`, `toString`) returns `''` for an + empty input, never `empty`, so the comparison is constant. Write + `trim($v) != ''`, which also covers an empty `$v`. + Not reported when the same `and`/`or` chain also compares that call with + `''` (`trim($v) != empty and trim($v) != ''`): redundant, but correct. + **One thing to know about hint ordering**: the reference check runs before expression checking and **exits on its first error**, so an unrelated mistake anywhere in a file hides every expression hint in it. If you expect an E0xx and diff --git a/.claude/skills/mendix/write-lint-rules/SKILL.md b/.claude/skills/mendix/write-lint-rules/SKILL.md index e695d17ed6..0841c61aef 100644 --- a/.claude/skills/mendix/write-lint-rules/SKILL.md +++ b/.claude/skills/mendix/write-lint-rules/SKILL.md @@ -567,7 +567,7 @@ Returned by `permissions()` (all types) or `permissions_for()` (entity-specific) | `target_type` | string | What it points AT, upper-case: `"ENTITY"`, `"ASSOCIATION"`, `"MICROFLOW"`, `"NANOFLOW"`, `"RULE"`, `"PAGE"`, `"LAYOUT"`, `"WORKFLOW"`, `"WIDGET"`, `"JAVA_ACTION"`, `"JAVASCRIPT_ACTION"`, `"REST_OPERATION"`, `"REGULAR_EXPRESSION"`, `"ATTRIBUTE"`, `"ENUMERATION"`, `"ENUMERATION_VALUE"`. `LAYOUT`, `WIDGET`, `ATTRIBUTE`, `ENUMERATION` and `ENUMERATION_VALUE` are only ever targets; `SCHEDULED_EVENT`, `PROJECT_SETTINGS` and the service types only ever sources | | `target_id` | string | Target UUID | | `target_name` | string | `"Sales.Customer"`; three-part for an attribute or an enumeration value: `"Sales.Order.Total"`, `"Sales.OrderStatus.Open"` | -| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"datasource"`, `"action"`, `"layout"`, `"parameter"`, `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"` (a published REST operation, OData resource or action, or a service's authentication microflow), `"event"` (an entity event or a business event handler), `"member"` (binds/reads/writes an attribute or navigates an association), `"xpath"` (an XPath constraint names it), `"type"` (typed as an enumeration), `"value"` (an expression names an enumeration value), `"mapping"` (an import/export mapping maps the entity) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | +| `ref_kind` | string | How it references: `"call"`, `"create"`, `"retrieve"`, `"change"`, `"delete"`, `"commit"` (a commit action, or a create/change that commits — beside its `"create"`/`"change"` edge; a commit of a variable whose entity the flow cannot tell has no edge), `"show_page"`, `"datasource"`, `"action"`, `"layout"`, `"parameter"`, `"return"`, `"generalize"`, `"associate"`, `"home_page"`, `"login_page"`, `"menu_item"`, `"calculate"`, `"schedule"`, `"validate"`, `"settings"`, `"widget"`, `"sync"`, `"publish"` (a published REST operation, OData resource or action, or a service's authentication microflow), `"event"` (an entity event or a business event handler), `"member"` (binds/reads/writes an attribute or navigates an association), `"xpath"` (an XPath constraint names it), `"type"` (typed as an enumeration), `"value"` (an expression names an enumeration value), `"mapping"` (an import/export mapping maps the entity), `"split"` (a decision branches on an object being the entity) — lower-case, unlike the types above. Attribute names used only through a variable in a free-text expression (`$Order/Total`) have no edge | | `module_name` | string | Source module | ### project_security diff --git a/.claude/skills/mendix/write-microflows/reference/pitfalls.md b/.claude/skills/mendix/write-microflows/reference/pitfalls.md index 8f1cd83428..885e86f287 100644 --- a/.claude/skills/mendix/write-microflows/reference/pitfalls.md +++ b/.claude/skills/mendix/write-microflows/reference/pitfalls.md @@ -494,6 +494,36 @@ log info node 'Orders' 'Order ' + $Order/Number + ' shipped'; A template kept in a variable cannot be filled by `with (…)` at run time: write the template in the statement, or build the whole text in the expression. +### 17. A String Function Never Returns `empty` + +`trim`, `toLowerCase`, `toUpperCase`, `substring`, `replaceAll` and `toString` return +`''` for an `empty` input, never `empty` (measured on the Mendix 10.18.5 runtime). So +comparing their result with `empty` is constant: `!= empty` is always true, and a +blank value goes down the "has text" branch. `check` warns (E015). + +```mdl +-- ❌ WRONG: always true — trim(empty) and trim(' ') are both '' +declare $HasText boolean = trim($Value) != empty; + +-- ✅ CORRECT: trim(empty) is '', so this covers empty, '' and blanks +declare $HasText boolean = trim($Value) != ''; +``` + +### 18. Concatenating `empty` Gives the Text `null` + +`+` turns an `empty` String operand into the four characters `null`, not `''`: +`'[' + $Value + ']'` with an empty `$Value` is `[null]` (measured on the Mendix 10.18.5 +runtime). A label built from an attribute that may be empty shows "null" to the user. +`check` cannot know whether a value will be empty, so it does not warn — guard it: + +```mdl +-- ❌ WRONG: shows 'Customer: null' when Name is empty +declare $Label string = 'Customer: ' + $Customer/Name; + +-- ✅ CORRECT +declare $Label string = if $Customer/Name != empty then 'Customer: ' + $Customer/Name else 'Customer: '; +``` + ## Implicit Variable Creation (CE0111 Duplicate Variable) These statements **implicitly create a new variable** with the name on the left side: diff --git a/CHANGELOG.md b/CHANGELOG.md index ae82a97a99..3def47fc56 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,8 @@ Headline: **Write what the project's own Mendix version stores, and commit it th ### Fixed +- **`show references` finds flows that branch on an entity** (ako/mxcli#1804) — an inheritance split (`split type $x when Module.Entity`) names the entity only on its outgoing flows, and the catalog recorded no edge for it, so `show references to` and `impact` on the entity missed the flow. It is now a `split` edge. Found by the new ref coverage trace, which checks that every document a catalog pass reads is reachable through refs. +- **A catalog refresh after a save rebuilds only what the save changed** (ako/mxcli#1804) — every refresh rebuilt every catalog table from every document, about 4.5 s of decoding and inserting on a 4,753-document project, and more on Windows, after any edit. The cache now records which document each row came from. The next refresh starts from the cached catalog and rebuilds only the changed documents and the documents that reference them, reading and decoding only those. A refresh after one edit on that project went from 5.6–8 s to 0.7 s end to end, with or without the source index. It falls back to a full build when a folder, module, settings, security or navigation document changes. Tests compare every table with a forced full build after each of 19 kinds of edit. - **`playwright check` no longer fails a page checked by the app's own host name behind a proxy** (ako/mxcli#1800) — with `https_proxy` set and an app checked as `http://frontend.local:8180/` (an `/etc/hosts` name for `127.0.0.1`), a page that rendered failed on two console errors, `WebSocket connection to 'ws://frontend.local:8180/mxdevtools/' failed: … Unexpected response code: 502`, while the same check on `127.0.0.1` passed. The runtime serves its dev channel on that name; the 502 came from the proxy, because Chromium sends `ws://` to the HTTPS proxy and `no_proxy` listed only the loopback names. The browser that `playwright check`, `run --page-check` and `run --screenshot` start now reaches a host that resolves to this machine directly. No console error is ignored: the channel connects, and a genuine console error still fails the page. - **Published OData entity sets, OData actions and older domain models have the property names Studio Pro writes** (ako/mxcli#1805) — on a Mendix 11.14.0 project `mxcli fix property-sets` reported seven `MIXED` names and could fill none of them. An entity set published without stating its modes was written without `ReadMode`, `InsertMode`, `UpdateMode` and `DeleteMode`, and an action whose microflow takes no parameters without `Parameters`; they are now written, as the values Mendix already read the missing ones as (read from the database, changes not supported, no parameters), so the service behaves as before and `describe` now prints the four modes. The domain-model names (`LocalizeDate` on a DateTime attribute, the four system-attribute flags on an entity without a generalization) are not left out by any current statement: they belong to entities an older mxcli wrote, which every later write of the domain model kept as they were. Both are now filled from values measured with `mx convert` on 10.24.24, 11.13.0 and 11.15.0, on the next write of the document or by `mxcli fix property-sets`. Run `mxcli fix property-sets` once to complete what an earlier mxcli wrote. - **A retrieve over a self-reference is the one object Mendix makes it, and a list use of it is refused by `check -p`** (ako/mxcli#1793) — for a Reference from an entity to itself (`M.Order_Parent`), `create microflow` typed `retrieve $P from $Order/M.Order_Parent` as a list of `M.Order` unless `$P` was later used as an object, so `$n = count $P;` passed `check -p` and `exec` and failed `mx check` with CE0013 "Input variable 'P' must be of type 'List'.". Mendix follows a self-reference from its FROM end: a Reference gives the one object, a Reference set the list. The full build, `alter`'s splice and `check -p` now take that one rule, from the entity and from a specialization of it alike, and `check -p` and `exec`'s pre-flight report MDL-RETRIEVE01 for the one object a Reference gives from its FROM entity — a self-reference or not — taken by an aggregate (CE0013), a list operation other than `range` (CE0097), a `loop` (CE0100) or a list parameter (CE0117). `set` on a self-reference's object is refused like `set` on any object (CE7247); the Change list Replace it used to be written as passed `mx check` only because mx check does not type a Change list's variable. Measured on 11.13.0: before, `count $P` passed `check -p` and `exec` and was CE0013; now `check -p` refuses it, and the object uses and the Reference set's list uses build at 0 errors. diff --git a/cmd/mxcli/init_hook.go b/cmd/mxcli/init_hook.go index ad9ac03a17..81abfdf470 100644 --- a/cmd/mxcli/init_hook.go +++ b/cmd/mxcli/init_hook.go @@ -61,6 +61,22 @@ MPR='%s' TAG="${MXCLI_TAG:-%s}" STAMP=.ai-context/mxcli-tooling.json +# The platform as the release assets name it. Under Git Bash, MSYS2 and Cygwin +# 'uname -s' is MINGW64_NT-10.0-26200 and the like, and the Windows assets end +# in .exe (mendixlabs/mxcli#1680) - so there the binary is ./mxcli.exe, which +# cmd and PowerShell can run as well as Git Bash. +os=$(uname -s | tr 'A-Z' 'a-z') +exe= +case "$os" in + mingw*|msys*|cygwin*) os=windows; exe=.exe ;; +esac +case "$(uname -m)" in + x86_64|amd64) arch=amd64 ;; + arm64|aarch64) arch=arm64 ;; + *) arch=$(uname -m) ;; +esac +MXCLI=./mxcli$exe + # stamp_field KEY: a string field of the tooling stamp, or nothing. stamp_field() { [ -f "$STAMP" ] || return 0 @@ -125,12 +141,12 @@ older_than_stamp() { } stale= -if [ -x ./mxcli ] && older_than_stamp ./mxcli; then - echo "./mxcli is $(bin_version ./mxcli), older than the mxcli that wrote this project's tooling ($(stamp_field version)) — replacing it." >&2 +if [ -x "$MXCLI" ] && older_than_stamp "$MXCLI"; then + echo "$MXCLI is $(bin_version "$MXCLI"), older than the mxcli that wrote this project's tooling ($(stamp_field version)) — replacing it." >&2 stale=1 fi -if [ ! -x ./mxcli ] || [ -n "$stale" ]; then +if [ ! -x "$MXCLI" ] || [ -n "$stale" ]; then # Prefer a copy that is already on this machine. Some environments ship mxcli # pre-installed on PATH, and the bootstrap instructions have you delete the # hardlink 'mxcli new' left in the project — after which this guard could @@ -146,40 +162,38 @@ if [ ! -x ./mxcli ] || [ -n "$stale" ]; then # leaves ./mxcli working, which is what the rest of this script and the # project's own CLAUDE.md assume. onpath=$(command -v mxcli 2>/dev/null || true) - if [ -n "$onpath" ] && [ -x "$onpath" ] && [ "$onpath" != "./mxcli" ] && [ "$onpath" != "$PWD/mxcli" ]; then + if [ -n "$onpath" ] && [ -x "$onpath" ] && [ "$onpath" != "./mxcli" ] && [ "$onpath" != "$PWD/mxcli" ] && [ "$onpath" != "$PWD/mxcli$exe" ]; then if older_than_stamp "$onpath"; then echo "mxcli on PATH (${onpath}) is $(bin_version "$onpath"), older than this project's tooling ($(stamp_field version)) — not linking it; downloading ${TAG} instead." >&2 else echo "mxcli found on PATH (${onpath}) — linking it in rather than downloading." - rm -f ./mxcli - ln -f "$onpath" ./mxcli 2>/dev/null || - ln -sf "$onpath" ./mxcli 2>/dev/null || - cp "$onpath" ./mxcli - chmod +x ./mxcli 2>/dev/null || true + # On Windows also the ./mxcli that 'mxcli new' leaves there: it is the + # Linux build, and beside ./mxcli.exe it shadows it for './mxcli'. + rm -f ./mxcli "$MXCLI" + ln -f "$onpath" "$MXCLI" 2>/dev/null || + ln -sf "$onpath" "$MXCLI" 2>/dev/null || + cp "$onpath" "$MXCLI" + chmod +x "$MXCLI" 2>/dev/null || true stale= fi fi fi -if [ ! -x ./mxcli ] || [ -n "$stale" ]; then - os=$(uname -s | tr 'A-Z' 'a-z') - case "$(uname -m)" in - x86_64|amd64) arch=amd64 ;; - arm64|aarch64) arch=arm64 ;; - *) arch=$(uname -m) ;; - esac - url="https://github.com/mendixlabs/mxcli/releases/download/${TAG}/mxcli-${os}-${arch}" +if [ ! -x "$MXCLI" ] || [ -n "$stale" ]; then + url="https://github.com/mendixlabs/mxcli/releases/download/${TAG}/mxcli-${os}-${arch}${exe}" echo "Downloading mxcli ${TAG} for ${os}/${arch}..." - # Into a temporary file, then renamed over ./mxcli: writing straight to - # ./mxcli would write THROUGH a symlink into the PATH binary it points at, - # and a failed download would leave no binary at all. + # Into a temporary file, then renamed over the binary: writing straight to + # it would write THROUGH a symlink into the PATH binary it points at, and a + # failed download would leave no binary at all. if curl -fsSL -o ./mxcli.download "$url"; then chmod +x ./mxcli.download - mv -f ./mxcli.download ./mxcli + # The Linux ./mxcli 'mxcli new' leaves on Windows would shadow ./mxcli.exe. + [ -z "$exe" ] || rm -f ./mxcli + mv -f ./mxcli.download "$MXCLI" else rm -f ./mxcli.download echo "Could not download mxcli from ${url}." >&2 - if [ ! -x ./mxcli ]; then + if [ ! -x "$MXCLI" ]; then echo "Fetch it manually, or set MXCLI_TAG to a released version." >&2 exit 1 fi @@ -187,8 +201,8 @@ if [ ! -x ./mxcli ] || [ -n "$stale" ]; then fi fi -if older_than_stamp ./mxcli; then - echo "WARNING: ./mxcli ($(bin_version ./mxcli)) is still older than this project's tooling ($(stamp_field version))." >&2 +if older_than_stamp "$MXCLI"; then + echo "WARNING: $MXCLI ($(bin_version "$MXCLI")) is still older than this project's tooling ($(stamp_field version))." >&2 echo " Set MXCLI_TAG to $(stamp_field version) or newer (or 'nightly') and re-run this script." >&2 fi @@ -199,7 +213,7 @@ fi # the same confidence as current guidance. Quiet when already current; refuses # (rather than downgrades) when the binary is older than the tooling; never # fatal, since a refresh must not block the session. -./mxcli init --sync-skills . || true +"$MXCLI" init --sync-skills . || true # Cache MxBuild + the runtime and provision the database. Not exec'd: the # hook runs this script under '|| true' so a failure never blocks the session, @@ -209,7 +223,7 @@ fi log=$(mktemp 2>/dev/null || echo "${TMPDIR:-/tmp}/mxcli-bootstrap.$$.log") st="$log.status" rm -f "$st" -{ ./mxcli run --local --setup --ensure-db -p "$MPR" 2>&1 || echo "$?" >"$st"; } | tee "$log" +{ "$MXCLI" run --local --setup --ensure-db -p "$MPR" 2>&1 || echo "$?" >"$st"; } | tee "$log" status=0 [ -f "$st" ] && status=$(cat "$st") if [ "$status" -ne 0 ]; then diff --git a/cmd/mxcli/init_hook_platform_test.go b/cmd/mxcli/init_hook_platform_test.go new file mode 100644 index 0000000000..53428af609 --- /dev/null +++ b/cmd/mxcli/init_hook_platform_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The bootstrap script names the release asset to download from uname. Under +// Git Bash, MSYS2 and Cygwin `uname -s` is MINGW64_NT-10.0-26200 and friends, +// not "windows", and the Windows assets carry an .exe suffix +// (mendixlabs/mxcli#1680). These tests stand in a fake uname and a curl that +// records the URL it was asked for. + +// fakeUname writes a uname reporting kernel name sys and machine arch. +func fakeUname(t *testing.T, bin, sys, arch string) { + t.Helper() + body := "#!/bin/sh\ncase \"$1\" in\n -s) echo '" + sys + "' ;;\n -m) echo '" + arch + "' ;;\n *) echo '" + sys + "' ;;\nesac\n" + if err := os.WriteFile(filepath.Join(bin, "uname"), []byte(body), 0o755); err != nil { + t.Fatal(err) + } +} + +// urlRecordingCurl replaces the env's curl with one that also logs its URL. +func urlRecordingCurl(t *testing.T, e bootEnv) { + t.Helper() + curl := "#!/bin/sh\nout=\nurl=\nwhile [ $# -gt 0 ]; do case \"$1\" in -o) out=$2; shift ;; -*) ;; *) url=$1 ;; esac; shift; done\n" + + "echo \"curl $out $url\" >> \"$BOOT_LOG\"\n" + + "printf '#!/bin/sh\\nif [ \"$1\" = \"--version\" ]; then echo \"mxcli version nightly-20991231-abcdef0\"; exit 0; fi\\necho \"nightly $*\" >> \"$BOOT_LOG\"\\n' > \"$out\"\n" + if err := os.WriteFile(filepath.Join(e.bin, "curl"), []byte(curl), 0o755); err != nil { + t.Fatal(err) + } +} + +// downloadedURL is the URL the fake curl was asked for, or "". +func (e bootEnv) downloadedURL(t *testing.T) string { + t.Helper() + data, _ := os.ReadFile(e.log) + for _, line := range strings.Split(string(data), "\n") { + if f := strings.Fields(line); len(f) == 3 && f[0] == "curl" { + return f[2] + } + } + return "" +} + +func TestBootstrapScript_DownloadsTheAssetForThePlatform(t *testing.T) { + requireSh(t) + const base = "https://github.com/mendixlabs/mxcli/releases/download/nightly/" + cases := []struct { + name, sys, arch, asset, binary string + }{ + {"git bash", "MINGW64_NT-10.0-26200", "x86_64", "mxcli-windows-amd64.exe", "mxcli.exe"}, + {"git bash 32-bit shell", "MINGW32_NT-10.0-26200", "x86_64", "mxcli-windows-amd64.exe", "mxcli.exe"}, + {"msys2", "MSYS_NT-10.0-26200", "x86_64", "mxcli-windows-amd64.exe", "mxcli.exe"}, + {"cygwin", "CYGWIN_NT-10.0-26200", "x86_64", "mxcli-windows-amd64.exe", "mxcli.exe"}, + {"windows arm64", "MINGW64_NT-10.0-26200", "aarch64", "mxcli-windows-arm64.exe", "mxcli.exe"}, + // Controls: the platforms that already worked keep their asset and ./mxcli. + {"linux", "Linux", "x86_64", "mxcli-linux-amd64", "mxcli"}, + {"macos", "Darwin", "arm64", "mxcli-darwin-arm64", "mxcli"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + e := newBootEnv(t, "") + urlRecordingCurl(t, e) + fakeUname(t, e.bin, c.sys, c.arch) + + e.run(t) + if got := e.downloadedURL(t); got != base+c.asset { + t.Errorf("downloaded %q, want %q", got, base+c.asset) + } + if _, err := os.Stat(filepath.Join(e.project, c.binary)); err != nil { + t.Errorf("./%s not installed: %v", c.binary, err) + } + // The rest of the script must run the binary it installed. + data, _ := os.ReadFile(e.log) + if !strings.Contains(string(data), "nightly run --local --setup --ensure-db") { + t.Errorf("downloaded binary did not provision the app; log:\n%s", data) + } + }) + } +} + +// `mxcli new` on Windows leaves the Linux build as ./mxcli, which Git Bash does +// not treat as executable. It must not survive beside ./mxcli.exe, where it +// would shadow it for `./mxcli` in Git Bash. +func TestBootstrapScript_WindowsReplacesNonExecutableProjectBinary(t *testing.T) { + requireSh(t) + e := newBootEnv(t, "") + urlRecordingCurl(t, e) + fakeUname(t, e.bin, "MINGW64_NT-10.0-26200", "x86_64") + linuxBuild := filepath.Join(e.project, "mxcli") + if err := os.WriteFile(linuxBuild, []byte("\x7fELF"), 0o644); err != nil { + t.Fatal(err) + } + + e.run(t) + if _, err := os.Stat(filepath.Join(e.project, "mxcli.exe")); err != nil { + t.Errorf("./mxcli.exe not installed: %v", err) + } + if _, err := os.Stat(linuxBuild); !os.IsNotExist(err) { + t.Errorf("the non-executable ./mxcli was left beside ./mxcli.exe (stat err %v)", err) + } +} + +// mxcli on PATH under Windows is linked in as ./mxcli.exe, so cmd and +// PowerShell can run it too. +func TestBootstrapScript_WindowsLinksPathBinaryAsExe(t *testing.T) { + requireSh(t) + e := newBootEnv(t, "") + fakeUname(t, e.bin, "MINGW64_NT-10.0-26200", "x86_64") + fakeMxcli(t, filepath.Join(e.bin, "mxcli"), "v0.26.0") + + e.run(t) + if e.downloaded(t) { + t.Error("downloaded although mxcli is on PATH") + } + if _, err := os.Stat(filepath.Join(e.project, "mxcli.exe")); err != nil { + t.Errorf("./mxcli.exe not linked in: %v", err) + } + data, _ := os.ReadFile(e.log) + if !strings.Contains(string(data), "v0.26.0 run --local --setup --ensure-db") { + t.Errorf("the linked binary did not provision the app; log:\n%s", data) + } +} diff --git a/docs-site/src/ide/syncing.md b/docs-site/src/ide/syncing.md index 017355ca5d..4e08cfbee1 100644 --- a/docs-site/src/ide/syncing.md +++ b/docs-site/src/ide/syncing.md @@ -101,6 +101,12 @@ file, so a `./mxcli` that is a symlink to the `PATH` binary never has the download written through it. If the download fails, the older `./mxcli` is kept with a warning rather than leaving the project without a binary. +On Windows (Git Bash, MSYS2 or Cygwin) the script fetches the +`mxcli-windows-.exe` asset and installs it, or links the `PATH` binary, +as `./mxcli.exe`. Git Bash, PowerShell and cmd can all run it as `./mxcli`. A +non-executable `./mxcli` beside it, such as the Linux build that `mxcli new` +leaves on Windows, is removed. + ## Re-running mxcli init `mxcli init` regenerates everything, including the devcontainer and tool diff --git a/docs-site/src/reference/catalog/list-references-impact.md b/docs-site/src/reference/catalog/list-references-impact.md index 4e256dde9b..89e9d0345a 100644 --- a/docs-site/src/reference/catalog/list-references-impact.md +++ b/docs-site/src/reference/catalog/list-references-impact.md @@ -29,6 +29,7 @@ The target may also be an attribute (`Module.Entity.Attribute`), an enumeration, | `type` | an attribute, parameter or variable is typed as the enumeration | | `value` | an expression names the enumeration value | | `mapping` | an import or export mapping maps the entity | +| `split` | a microflow, nanoflow or rule branches on an object being the entity (an inheritance split) | `IMPACT OF` an enumeration includes the uses of each of its values, with a `Target` column naming which one. diff --git a/docs-site/src/tools/refresh-catalog.md b/docs-site/src/tools/refresh-catalog.md index 15ae297e5a..01b35c8414 100644 --- a/docs-site/src/tools/refresh-catalog.md +++ b/docs-site/src/tools/refresh-catalog.md @@ -71,7 +71,24 @@ mxcli -p app.mpr -c "REFRESH CATALOG FULL FORCE" The catalog is cached in `.mxcli/catalog.db` next to the MPR file. On subsequent refreshes, mxcli checks whether the project has changed and only rebuilds if necessary (unless `FORCE` is specified). -A catalog with the MDL source index (`REFRESH CATALOG FULL SOURCE`) keeps it on every later rebuild, including the ones `SEARCH` and `SHOW REFERENCES` start when the project has changed. Those rebuilds describe again only the documents that changed, and the documents that reference them. An entity counts as changed when its own definition does, not when another entity of its module does; a change to an association or annotation describes every entity of that module again. Every other row is carried over from the cache, so a rebuild after one edit costs about as much as `REFRESH CATALOG FULL`. Everything is described again when something outside the described documents changes (a folder, a module, the project settings, a Java action), when another mxcli build wrote the cache, and with `FORCE`. +### Incremental rebuilds + +A rebuild after a save does not start from scratch. The cache records which document every row came from, and the next rebuild of the same level (fast, `FULL` or `FULL SOURCE`) starts from the cached catalog and produces again only the rows of: + +- the documents that changed, were added or were deleted (a document moved to another existing folder counts as changed); +- the documents whose references name one of them, before or after the change. A document's rows can depend on what it names: what a delete removes depends on the association it retrieved over. A specialization counts as naming what its generalization holds. + +The rows of every other document stay as they are, and only the documents being rebuilt are read and decoded. On a 4,753-document project a refresh after one edit takes under a second, against several seconds for a full build. The progress output then shows one line, `incremental: N units rebuilt, M carried over`, instead of a count per table. + +The rebuild is a full one, as before, when: + +- something changes that no per-document pass reads: a folder, a module, the project settings, the security settings or the navigation; +- the cache was written by another mxcli build or at another level; +- `FORCE` is given. + +An incremental rebuild produces exactly the rows a full one would. Only the internal row IDs (the `Id` column of `refs` and `permissions`) and the order of rows in an unsorted query can differ. + +With the MDL source index, an entity in a document being rebuilt is described again only when its own definition changed, not when another entity of its module did. A change to an association or annotation describes every entity of that module again, and a change to a document that is never described (a Java action, a constant, an image collection) describes everything. ### The unit store (MPR v2) diff --git a/docs/13-decisions/0014-incremental-catalog-builds.md b/docs/13-decisions/0014-incremental-catalog-builds.md new file mode 100644 index 0000000000..c972f5e889 --- /dev/null +++ b/docs/13-decisions/0014-incremental-catalog-builds.md @@ -0,0 +1,94 @@ +# ADR-0014: Catalog rows belong to a unit, and a refresh rebuilds only the changed units + +- **Status**: Accepted +- **Date**: 2026-10-30 +- **Related**: ako/mxcli#1804 (this decision), #1790 and #1803 (the incremental source index it generalises), #1789 (the unit store) + +## Context + +The catalog (`.mxcli/catalog.db`) goes stale on every save, and every +command that reads it then rebuilds it. A rebuild produced every table from +every unit: on a 4,753-unit project about 4.5 s of decoding and inserting, +5.6–8 s end to end per refresh. Users reported waits of hours on Windows, +where every unit file a refresh opens is also scanned. Nearly all of that +work reproduces rows that are already in the cache. Only #1790's source +index reused anything, and it still decoded every document to decide. + +The builder is about 10,000 lines of passes, written over two years by many +contributors, with about 130 insert statements across 29 files. New passes +will keep arriving. Whatever makes refreshes incremental has to hold for +passes nobody has written yet. + +## Decision + +Every catalog row belongs to the unit it came from. A wrapper on the build's +transaction records each insert's rowid range under the pass's current owner +(`own(unitID)`), or under no unit for project-wide rows. The next build loads +the cached catalog and plans from the unit content hashes. It rebuilds the +changed units and the units whose `refs` name them, decoding only those, and +drops and rewrites the project-wide rows. Every other row is left as it is. +Anything the plan cannot vouch for makes it a full build: a change to a unit +no per-unit pass owns, another mxcli build or level, or a row written for a +skipped document. An incremental build must produce exactly the rows of a +full one. + +## Consequences + +- A refresh after one edit on that project takes 0.7 s end to end, against + 5.6–8 s, with or without the source index. Only the rebuilt units' files + are read. +- **Every new catalog pass carries obligations**: mark its rows' owner, list + documents through the scoped list, look other documents up in the catalog + tables, and give any cross-document dependency a `refs` edge. Breaking the + first two leaves the build correct but slower, and nothing would notice. So + three tests count the cost: rows with no owner + (`TestCatalog_RowsBelongToADocument`), documents decoded per incremental + refresh (`TestCatalog_IncrementalRefreshDecodesOnlyWhatItRebuilds`), and + direct reader listings (`TestBuilderListsDocumentsThroughTheScope`). Each + keeps an allowlist that says why its entries are exempt. Breaking the third + or fourth obligation is a correctness bug, and only the equivalence tests + can catch it, and only for the steps they include. `/mxcli-dev:review` + carries the checklist. +- **Correctness depends on `refs` as the model of what a document's rows + depend on.** A dependency `refs` does not record shows up as a stale row, + not an error. Two kinds of evidence back it: + - The equivalence tests compare every table with a forced full build over + 19 kinds of edit, chained incrementally. They cover only what their steps + exercise. + - The ref coverage trace (`builder_coverage.go`) records every name the + cross-document lookups read per document and checks it the way the plan + would. It must be at 100% on the fixture + (`TestCatalog_RefsCoverEveryCrossDocumentRead`), and it measures a real + project on demand (`TestCatalog_RefCoverageOfProject`): 99.4% on two + large ones when it was added. The gaps it found are refs edges that + impact analysis lacked too. + - It sees only reads through the traced lookups, which is why a pass may + read other documents only through the catalog tables. +- Row order in an unsorted query, and surrogate IDs (`refs.Id`, + `permissions.Id`), differ between an incremental and a full build. Nothing + may depend on them. +- The cache now holds bookkeeping (`catalog_row_owners`, `catalog_units`, + `catalog_unit_names`): about 25,000 ranges on that project. +- Folder, module, settings, security and navigation changes still cost a full + build. Making one of them incremental means a pass that owns its rows, and + that every row depending on it is rebuilt with it. + +## Alternatives considered + +- **Phase-level reuse** (skip a whole pass when none of its unit types + changed). Simple, but almost every pass reads domain models, so one entity + edit still rebuilds nearly everything. +- **An owner column on every table.** It changes 78 schemas and 130 inserts, + shows the bookkeeping in every user query, and a new table can forget it. + Recording rowid ranges at the transaction needs no change to any insert. +- **Copying the kept rows into a fresh catalog** (the first implementation). + It was correct, but re-tokenising the carried full-text rows cost more than + the rest of the build. Starting from the cached catalog and deleting the + rebuilt rows copies nothing. The same database connection must not be + loaded twice, though: modernc.org/sqlite crashed once its full-text tables + had been used, so the plan is made on a scratch copy, and a fallback empties + the tables instead of reloading them. +- **Trusting the per-unit hash alone** (no dependents). Wrong: a document's + rows can change while its own unit does not, as in a delete over an + association whose end moved. The equivalence test proves it with a step + that fails without the dependents. diff --git a/docs/13-decisions/README.md b/docs/13-decisions/README.md index 3818db141f..e8a252b535 100644 --- a/docs/13-decisions/README.md +++ b/docs/13-decisions/README.md @@ -107,6 +107,7 @@ This preserves the audit trail. | [0011](0011-mdl-language-versioning.md) | MDL evolves through deprecation aliases and a language header; meaning changes only across versions | Accepted | | [0012](0012-mdl-first-and-data-first-editing.md) | MDL-first and data-first editing share one syntax and one patch engine | Accepted | | [0013](0013-activity-settings-property-list.md) | Microflow activity settings: words for the statement, one property list for the dialog (refines R2/R3 of 0010 for activities) | Proposed | +| [0014](0014-incremental-catalog-builds.md) | Catalog rows belong to a unit, and a refresh rebuilds only the changed units | Accepted | ## Candidates to back-fill diff --git a/mdl/backend/modelsdk/backend.go b/mdl/backend/modelsdk/backend.go index 2fea532e8a..495a6ef2ac 100644 --- a/mdl/backend/modelsdk/backend.go +++ b/mdl/backend/modelsdk/backend.go @@ -56,6 +56,10 @@ type Backend struct { // sourceFiles lists the files and folders beside the model that writes // changed since the executor last took them (TakeSourceFileChanges). sourceFiles []backend.SourceFileChange + + // unitScope, on a view made by WithUnitScope, is the units its document + // lists hold; nil lists every unit. + unitScope func(unitID string) bool } // New constructs a modelsdk backend. diff --git a/mdl/backend/modelsdk/constant.go b/mdl/backend/modelsdk/constant.go index aa11a877a3..14e6e59497 100644 --- a/mdl/backend/modelsdk/constant.go +++ b/mdl/backend/modelsdk/constant.go @@ -6,7 +6,6 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/element" genConst "github.com/mendixlabs/mxcli/modelsdk/gen/constants" genDT "github.com/mendixlabs/mxcli/modelsdk/gen/datatypes" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/model" ) @@ -16,7 +15,7 @@ import ( // constantDataTypeToModel and registers them in the codec. func (b *Backend) ListConstants() ([]*model.Constant, error) { - units, err := mprread.ListUnitsWithContainer[*genConst.Constant](b.reader) + units, err := listUnits[*genConst.Constant](b) if err != nil { return nil, err } @@ -28,7 +27,7 @@ func (b *Backend) ListConstants() ([]*model.Constant, error) { } func (b *Backend) GetConstant(id model.ID) (*model.Constant, error) { - units, err := mprread.ListUnitsWithContainer[*genConst.Constant](b.reader) + units, err := listUnits[*genConst.Constant](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/data_transformer_write.go b/mdl/backend/modelsdk/data_transformer_write.go index e4cff3f1e6..bc9b958308 100644 --- a/mdl/backend/modelsdk/data_transformer_write.go +++ b/mdl/backend/modelsdk/data_transformer_write.go @@ -11,7 +11,6 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/element" genDt "github.com/mendixlabs/mxcli/modelsdk/gen/datatransformers" mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/modelsdk/property" ) @@ -19,7 +18,7 @@ import ( // plus source type and transform steps). Used by SHOW DATA TRANSFORMERS, the // CREATE OR MODIFY existence check, and DROP. func (b *Backend) ListDataTransformers() ([]*model.DataTransformer, error) { - units, err := mprread.ListUnitsWithContainer[*genDt.DataTransformer](b.reader) + units, err := listUnits[*genDt.DataTransformer](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/domainmodel.go b/mdl/backend/modelsdk/domainmodel.go index 0627f0f8ad..73dca645ee 100644 --- a/mdl/backend/modelsdk/domainmodel.go +++ b/mdl/backend/modelsdk/domainmodel.go @@ -29,7 +29,7 @@ import ( // ListDomainModels reads every domain model through the codec engine. func (b *Backend) ListDomainModels() ([]*domainmodel.DomainModel, error) { - units, err := mprread.ListUnitsWithContainer[*genDm.DomainModel](b.reader) + units, err := listUnits[*genDm.DomainModel](b) if err != nil { return nil, err } @@ -65,7 +65,7 @@ func (b *Backend) populateViewEntityOql(dms []*domainmodel.DomainModel) { if !need { return } - units, err := mprread.ListUnitsWithContainer[*genDm.ViewEntitySourceDocument](b.reader) + units, err := listUnits[*genDm.ViewEntitySourceDocument](b) if err != nil { return } diff --git a/mdl/backend/modelsdk/domainmodel_security_write.go b/mdl/backend/modelsdk/domainmodel_security_write.go index 2a4cbf96a1..b1ccee8c46 100644 --- a/mdl/backend/modelsdk/domainmodel_security_write.go +++ b/mdl/backend/modelsdk/domainmodel_security_write.go @@ -12,7 +12,6 @@ import ( "github.com/mendixlabs/mxcli/model" genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" "github.com/mendixlabs/mxcli/modelsdk/meta" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // AddEntityAccessRule adds (or upserts by matching module-role set) an entity @@ -803,7 +802,7 @@ type projectAssociations struct { // names its TO entity by qualified name, so the incoming half is a scan of every // other domain model — the only place the fact is stored. func (b *Backend) scanProjectAssociations(moduleName string) (*projectAssociations, error) { - units, err := mprread.ListUnitsWithContainer[*genDm.DomainModel](b.reader) + units, err := listUnits[*genDm.DomainModel](b) if err != nil { return nil, fmt.Errorf("list domain models: %w", err) } diff --git a/mdl/backend/modelsdk/enumeration.go b/mdl/backend/modelsdk/enumeration.go index 53ef568de3..1b1f2ac7f7 100644 --- a/mdl/backend/modelsdk/enumeration.go +++ b/mdl/backend/modelsdk/enumeration.go @@ -5,7 +5,6 @@ package modelsdkbackend import ( genEnum "github.com/mendixlabs/mxcli/modelsdk/gen/enumerations" "github.com/mendixlabs/mxcli/modelsdk/meta" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/model" ) @@ -15,7 +14,7 @@ import ( // relies on the gen/texts package being registered (blank-imported in page.go). func (b *Backend) ListEnumerations() ([]*model.Enumeration, error) { - units, err := mprread.ListUnitsWithContainer[*genEnum.Enumeration](b.reader) + units, err := listUnits[*genEnum.Enumeration](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/integration_read.go b/mdl/backend/modelsdk/integration_read.go index 019a120fc9..4a8bfb3fa3 100644 --- a/mdl/backend/modelsdk/integration_read.go +++ b/mdl/backend/modelsdk/integration_read.go @@ -15,7 +15,6 @@ import ( genImportMappings "github.com/mendixlabs/mxcli/modelsdk/gen/importmappings" genOp "github.com/mendixlabs/mxcli/modelsdk/gen/odatapublish" genRest "github.com/mendixlabs/mxcli/modelsdk/gen/rest" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // ListConsumedODataServices reads every Rest$ConsumedODataService unit and @@ -26,7 +25,7 @@ import ( // gen element as nested parts/by-name refs and are not surfaced here because // no catalog/describe path reads them through this method. func (b *Backend) ListConsumedODataServices() ([]*model.ConsumedODataService, error) { - units, err := mprread.ListUnitsWithContainer[*genRest.ConsumedODataService](b.reader) + units, err := listUnits[*genRest.ConsumedODataService](b) if err != nil { return nil, err } @@ -92,7 +91,7 @@ func (b *Backend) ListConsumedODataServices() ([]*model.ConsumedODataService, er // reaching this method walks svc.EntityTypes/EntitySets members, and the gen // element stores those as nested parts. func (b *Backend) ListPublishedODataServices() ([]*model.PublishedODataService, error) { - units, err := mprread.ListUnitsWithContainer[*genOp.PublishedODataService2](b.reader) + units, err := listUnits[*genOp.PublishedODataService2](b) if err != nil { return nil, err } @@ -140,7 +139,7 @@ func (b *Backend) ListPublishedODataServices() ([]*model.PublishedODataService, // scheme, Operations with Name/HttpMethod/Path/Timeout/ResponseType/BodyType // and parameter counts). func (b *Backend) ListConsumedRestServices() ([]*model.ConsumedRestService, error) { - units, err := mprread.ListUnitsWithContainer[*genRest.ConsumedRestService](b.reader) + units, err := listUnits[*genRest.ConsumedRestService](b) if err != nil { return nil, err } @@ -396,7 +395,7 @@ func httpMethodUpper(method string) string { // Resources with Name and Operations with HTTPMethod/Path/Summary/Microflow/ // Deprecated). func (b *Backend) ListPublishedRestServices() ([]*model.PublishedRestService, error) { - units, err := mprread.ListUnitsWithContainer[*genRest.PublishedRestService](b.reader) + units, err := listUnits[*genRest.PublishedRestService](b) if err != nil { return nil, err } @@ -473,7 +472,7 @@ func (b *Backend) ListPublishedRestServices() ([]*model.PublishedRestService, er // EventNamePrefix, Channels with Messages/Attributes) and the // OperationImplementations the catalog/contract builders consume. func (b *Backend) ListBusinessEventServices() ([]*model.BusinessEventService, error) { - units, err := mprread.ListUnitsWithContainer[*genBe.BusinessEventService](b.reader) + units, err := listUnits[*genBe.BusinessEventService](b) if err != nil { return nil, err } @@ -599,7 +598,7 @@ func businessEventAttrType(el element.Element) string { // and converts it to the semantic model, mirroring the legacy reader for // top-level fields and the Queries (Name/SQL/QueryType) the catalog consumes. func (b *Backend) ListDatabaseConnections() ([]*model.DatabaseConnection, error) { - units, err := mprread.ListUnitsWithContainer[*genDb.DatabaseConnection](b.reader) + units, err := listUnits[*genDb.DatabaseConnection](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/java.go b/mdl/backend/modelsdk/java.go index bdbc26368e..bf646c2e33 100644 --- a/mdl/backend/modelsdk/java.go +++ b/mdl/backend/modelsdk/java.go @@ -4,7 +4,6 @@ package modelsdkbackend import ( genJA "github.com/mendixlabs/mxcli/modelsdk/gen/javaactions" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" @@ -16,7 +15,7 @@ import ( // listing. The rich parsed form (sdk/javaactions.JavaAction, with parameters and // return types) is ListJavaActionsFull, deferred to a later phase. func (b *Backend) ListJavaActions() ([]*types.JavaAction, error) { - units, err := mprread.ListUnitsWithContainer[*genJA.JavaAction](b.reader) + units, err := listUnits[*genJA.JavaAction](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/java_read.go b/mdl/backend/modelsdk/java_read.go index 185aaed3a9..d5b0c8ae93 100644 --- a/mdl/backend/modelsdk/java_read.go +++ b/mdl/backend/modelsdk/java_read.go @@ -13,7 +13,6 @@ import ( genCa "github.com/mendixlabs/mxcli/modelsdk/gen/codeactions" genJa "github.com/mendixlabs/mxcli/modelsdk/gen/javaactions" "github.com/mendixlabs/mxcli/modelsdk/meta" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/sdk/javaactions" ) @@ -35,7 +34,7 @@ func (b *Backend) ReadJavaActionByName(qualifiedName string) (*javaactions.JavaA return nil, fmt.Errorf("java action not found: %s", qualifiedName) } containers := b.containerSetForModule(string(mod.ID)) // module + nested folders - units, err := mprread.ListUnitsWithContainer[*genJa.JavaAction](b.reader) + units, err := listUnits[*genJa.JavaAction](b) if err != nil { return nil, err } @@ -73,7 +72,7 @@ func systemJavaActionByName(moduleName, actionName string) *javaactions.JavaActi // legacy reader appends are not yet modelled in modelsdk/meta, so they're omitted; // the catalog simply indexes fewer rows, it does not error.) func (b *Backend) ListJavaActionsFull() ([]*javaactions.JavaAction, error) { - units, err := mprread.ListUnitsWithContainer[*genJa.JavaAction](b.reader) + units, err := listUnits[*genJa.JavaAction](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/mapping_read.go b/mdl/backend/modelsdk/mapping_read.go index 5a01b22d10..708ab2e626 100644 --- a/mdl/backend/modelsdk/mapping_read.go +++ b/mdl/backend/modelsdk/mapping_read.go @@ -13,7 +13,6 @@ import ( genJson "github.com/mendixlabs/mxcli/modelsdk/gen/jsonstructures" genPrj "github.com/mendixlabs/mxcli/modelsdk/gen/projects" genXml "github.com/mendixlabs/mxcli/modelsdk/gen/xmlschemas" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // ListImportMappings reads every ImportMappings$ImportMapping unit and converts @@ -24,7 +23,7 @@ import ( // catalog only reads the top-level element count, and no other path reaches // this method. func (b *Backend) ListImportMappings() ([]*model.ImportMapping, error) { - units, err := mprread.ListUnitsWithContainer[*genImp.ImportMapping](b.reader) + units, err := listUnits[*genImp.ImportMapping](b) if err != nil { return nil, err } @@ -66,7 +65,7 @@ func (b *Backend) ListImportMappings() ([]*model.ImportMapping, error) { // MessageDefinition/NullValueOption and the root mapping Elements). Deep // mapping-element trees are not surfaced (see ListImportMappings). func (b *Backend) ListExportMappings() ([]*model.ExportMapping, error) { - units, err := mprread.ListUnitsWithContainer[*genExp.ExportMapping](b.reader) + units, err := listUnits[*genExp.ExportMapping](b) if err != nil { return nil, err } @@ -112,7 +111,7 @@ func (b *Backend) ListExportMappings() ([]*model.ExportMapping, error) { // not parse, so only the fields that identify the document are converted. Its // one job is resolving a mapping's `with xml schema` reference (ako/mxcli#259). func (b *Backend) ListXmlSchemas() ([]*types.XmlSchema, error) { - units, err := mprread.ListUnitsWithContainer[*genXml.XmlSchema](b.reader) + units, err := listUnits[*genXml.XmlSchema](b) if err != nil { return nil, err } @@ -138,7 +137,7 @@ func (b *Backend) ListXmlSchemas() ([]*types.XmlSchema, error) { // catalog consumes (Name/Documentation/JsonSnippet/ExportLevel/Excluded and the // element tree with children). func (b *Backend) ListJsonStructures() ([]*types.JsonStructure, error) { - units, err := mprread.ListUnitsWithContainer[*genJson.JsonStructure](b.reader) + units, err := listUnits[*genJson.JsonStructure](b) if err != nil { return nil, err } @@ -196,7 +195,7 @@ func jsonElementFromGen(g *genJson.JsonElement) *types.JsonElement { // (ExportLevel→"Source", ProtectedModuleType→"AddOn", Version→"1.0.0") is // preserved. func (b *Backend) ListModuleSettings() ([]*types.ModuleSettings, error) { - units, err := mprread.ListUnitsWithContainer[*genPrj.ModuleSettings](b.reader) + units, err := listUnits[*genPrj.ModuleSettings](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/mcp_bypass_reads.go b/mdl/backend/modelsdk/mcp_bypass_reads.go index eeddd17a75..4028c94e46 100644 --- a/mdl/backend/modelsdk/mcp_bypass_reads.go +++ b/mdl/backend/modelsdk/mcp_bypass_reads.go @@ -65,7 +65,7 @@ func (b *Backend) GetDomainModelByID(id model.ID) (*domainmodel.DomainModel, err // single-item read return something subtly different from the same item in a // listing. func (b *Backend) GetWorkflow(id model.ID) (*workflows.Workflow, error) { - units, err := mprread.ListUnitsWithContainer[*genWf.Workflow](b.reader) + units, err := listUnits[*genWf.Workflow](b) if err != nil { return nil, err } @@ -85,7 +85,7 @@ func (b *Backend) GetWorkflow(id model.ID) (*workflows.Workflow, error) { // than about the convention; GetNavigation keeps its first-one shortcut, since // changing that is a separate decision with its own callers. func (b *Backend) ListNavigationDocuments() ([]*types.NavigationDocument, error) { - units, err := mprread.ListUnitsWithContainer[*genNav.NavigationDocument](b.reader) + units, err := listUnits[*genNav.NavigationDocument](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/menu_read.go b/mdl/backend/modelsdk/menu_read.go index 768db8ebb2..84c080620a 100644 --- a/mdl/backend/modelsdk/menu_read.go +++ b/mdl/backend/modelsdk/menu_read.go @@ -8,7 +8,6 @@ import ( "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" genMenus "github.com/mendixlabs/mxcli/modelsdk/gen/menus" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // ListMenuDocuments reads every standalone Menus$MenuDocument unit. @@ -22,7 +21,7 @@ func (b *Backend) ListMenuDocuments() ([]*types.MenuDocument, error) { if b.reader == nil { return nil, fmt.Errorf("ListMenuDocuments: not connected") } - units, err := mprread.ListUnitsWithContainer[*genMenus.MenuDocument](b.reader) + units, err := listUnits[*genMenus.MenuDocument](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/messagedefinition_document.go b/mdl/backend/modelsdk/messagedefinition_document.go index 00b2fd8de8..c5a69a9789 100644 --- a/mdl/backend/modelsdk/messagedefinition_document.go +++ b/mdl/backend/modelsdk/messagedefinition_document.go @@ -10,7 +10,6 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/element" genMsg "github.com/mendixlabs/mxcli/modelsdk/gen/messagedefinitions" mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // MessageDefinition2 documents (ako/mxcli#987). @@ -27,7 +26,7 @@ const messageDefinitionDocumentType = "MessageDefinitions$MessageDefinition2" // ListMessageDefinitionDocuments reads every MessageDefinition2 unit. func (b *Backend) ListMessageDefinitionDocuments() ([]*model.MessageDefinitionDocument, error) { - units, err := mprread.ListUnitsWithContainer[*genMsg.MessageDefinition2](b.reader) + units, err := listUnits[*genMsg.MessageDefinition2](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/messagedefinition_read.go b/mdl/backend/modelsdk/messagedefinition_read.go index 717c57d1f1..7ef32e210e 100644 --- a/mdl/backend/modelsdk/messagedefinition_read.go +++ b/mdl/backend/modelsdk/messagedefinition_read.go @@ -5,7 +5,6 @@ package modelsdkbackend import ( "github.com/mendixlabs/mxcli/model" genMsg "github.com/mendixlabs/mxcli/modelsdk/gen/messagedefinitions" - "github.com/mendixlabs/mxcli/modelsdk/mprread" ) // ListMessageDefinitionCollections reads every @@ -17,7 +16,7 @@ import ( // Email_Connector definitions — and none of these types appears in // modelsdk/gen/keyaudit_test.go's mismatch ledger. func (b *Backend) ListMessageDefinitionCollections() ([]*model.MessageDefinitionCollection, error) { - units, err := mprread.ListUnitsWithContainer[*genMsg.MessageDefinitionCollection](b.reader) + units, err := listUnits[*genMsg.MessageDefinitionCollection](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/microflow.go b/mdl/backend/modelsdk/microflow.go index c2965b1b48..9dcf735212 100644 --- a/mdl/backend/modelsdk/microflow.go +++ b/mdl/backend/modelsdk/microflow.go @@ -24,7 +24,7 @@ import ( // events drive complexity). Full per-activity detail (DESCRIBE) is a later phase. func (b *Backend) ListMicroflows() ([]*microflows.Microflow, error) { - units, err := mprread.ListUnitsWithContainer[*genMf.Microflow](b.reader) + units, err := listUnits[*genMf.Microflow](b) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (b *Backend) IsRule(qualifiedName string) (bool, error) { if qualifiedName == "" { return false, nil } - units, err := mprread.ListUnitsWithContainer[*genMf.Rule](b.reader) + units, err := listUnits[*genMf.Rule](b) if err != nil { return false, err } @@ -116,7 +116,7 @@ func (b *Backend) IsRule(qualifiedName string) (bool, error) { } func (b *Backend) ListNanoflows() ([]*microflows.Nanoflow, error) { - units, err := mprread.ListUnitsWithContainer[*genMf.Nanoflow](b.reader) + units, err := listUnits[*genMf.Nanoflow](b) if err != nil { return nil, err } @@ -128,7 +128,7 @@ func (b *Backend) ListNanoflows() ([]*microflows.Nanoflow, error) { } func (b *Backend) GetNanoflow(id model.ID) (*microflows.Nanoflow, error) { - units, err := mprread.ListUnitsWithContainer[*genMf.Nanoflow](b.reader) + units, err := listUnits[*genMf.Nanoflow](b) if err != nil { return nil, err } @@ -171,7 +171,7 @@ func nanoflowFromGen(nf *genMf.Nanoflow, containerID model.ID) *microflows.Nanof // so it is deliberately absent from ListMicroflows — SHOW MICROFLOWS lists // microflows only, as it already does for nanoflows and workflows. func (b *Backend) ListRules() ([]*microflows.Rule, error) { - units, err := mprread.ListUnitsWithContainer[*genMf.Rule](b.reader) + units, err := listUnits[*genMf.Rule](b) if err != nil { return nil, err } @@ -183,7 +183,7 @@ func (b *Backend) ListRules() ([]*microflows.Rule, error) { } func (b *Backend) GetRule(id model.ID) (*microflows.Rule, error) { - units, err := mprread.ListUnitsWithContainer[*genMf.Rule](b.reader) + units, err := listUnits[*genMf.Rule](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/move_view_write.go b/mdl/backend/modelsdk/move_view_write.go index 2d7e232984..1701cc38ee 100644 --- a/mdl/backend/modelsdk/move_view_write.go +++ b/mdl/backend/modelsdk/move_view_write.go @@ -12,7 +12,6 @@ import ( genDm "github.com/mendixlabs/mxcli/modelsdk/gen/domainmodels" "github.com/mendixlabs/mxcli/modelsdk/meta" mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -47,7 +46,7 @@ func (b *Backend) UpdateOqlQueriesForMovedEntity(oldQualifiedName, newQualifiedN if b.writer == nil { return 0, fmt.Errorf("UpdateOqlQueriesForMovedEntity: not connected for writing") } - units, err := mprread.ListUnitsWithContainer[*genDm.ViewEntitySourceDocument](b.reader) + units, err := listUnits[*genDm.ViewEntitySourceDocument](b) if err != nil { return 0, fmt.Errorf("UpdateOqlQueriesForMovedEntity: list source docs: %w", err) } @@ -268,7 +267,7 @@ func (b *Backend) FindAllViewEntitySourceDocumentIDs(moduleName, docName string) if err != nil || mod == nil { return nil, nil } - units, err := mprread.ListUnitsWithContainer[*genDm.ViewEntitySourceDocument](b.reader) + units, err := listUnits[*genDm.ViewEntitySourceDocument](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/navigation_read.go b/mdl/backend/modelsdk/navigation_read.go index 2ff5bf78ef..a8b4a1faea 100644 --- a/mdl/backend/modelsdk/navigation_read.go +++ b/mdl/backend/modelsdk/navigation_read.go @@ -14,7 +14,6 @@ import ( genPages "github.com/mendixlabs/mxcli/modelsdk/gen/pages" genTexts "github.com/mendixlabs/mxcli/modelsdk/gen/texts" mmpr "github.com/mendixlabs/mxcli/modelsdk/mpr" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "go.mongodb.org/mongo-driver/v2/bson" ) @@ -27,7 +26,7 @@ import ( // metamodel uses canonical names (e.g. page actions read PageSettings, not // FormSettings; captions read Texts$Translation items). func (b *Backend) GetNavigation() (*types.NavigationDocument, error) { - units, err := mprread.ListUnitsWithContainer[*genNav.NavigationDocument](b.reader) + units, err := listUnits[*genNav.NavigationDocument](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/page.go b/mdl/backend/modelsdk/page.go index 580f899b12..38dd938e74 100644 --- a/mdl/backend/modelsdk/page.go +++ b/mdl/backend/modelsdk/page.go @@ -10,6 +10,7 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" "github.com/mendixlabs/mxcli/sdk/pages" ) @@ -23,8 +24,22 @@ import ( // by ListPageTemplates — this used to include them, to match legacy's // prefix-matched `Forms$Page` query, which made `show modules` report 46 pages // for a module with none. +// ListPageHeaders lists every page's ID, container, name and Excluded flag +// without decoding any page. +func (b *Backend) ListPageHeaders() ([]types.DocumentHeader, error) { + hs, err := mprread.ListUnitHeaders[*genPg.Page](b.reader) + if err != nil { + return nil, err + } + out := make([]types.DocumentHeader, len(hs)) + for i, h := range hs { + out[i] = types.DocumentHeader(h) + } + return out, nil +} + func (b *Backend) ListPages() ([]*pages.Page, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.Page](b.reader) + units, err := listUnits[*genPg.Page](b) if err != nil { return nil, err } @@ -41,7 +56,7 @@ func (b *Backend) ListPages() ([]*pages.Page, error) { // rather than FormCall, nothing consumes it yet, and a half-converted tree would // compare equal to a different one. func (b *Backend) ListPageTemplates() ([]*pages.PageTemplate, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.PageTemplate](b.reader) + units, err := listUnits[*genPg.PageTemplate](b) if err != nil { return nil, err } @@ -64,7 +79,7 @@ func pageTemplateFromGen(pt *genPg.PageTemplate, containerID model.ID) *pages.Pa } func (b *Backend) GetPage(id model.ID) (*pages.Page, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.Page](b.reader) + units, err := listUnits[*genPg.Page](b) if err != nil { return nil, err } @@ -79,7 +94,7 @@ func (b *Backend) GetPage(id model.ID) (*pages.Page, error) { // ListSnippets reads snippet units (count + identity is what SHOW MODULES needs; // the widget tree is not converted). func (b *Backend) ListSnippets() ([]*pages.Snippet, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.Snippet](b.reader) + units, err := listUnits[*genPg.Snippet](b) if err != nil { return nil, err } @@ -124,7 +139,7 @@ func (b *Backend) ListSnippets() ([]*pages.Snippet, error) { // lists them (name + scalar metadata) so SHOW / DESCRIBE BUILDING BLOCK work on // this engine too; the widget tree is read separately via GetRawUnit (shared). func (b *Backend) ListBuildingBlocks() ([]*pages.BuildingBlock, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.BuildingBlock](b.reader) + units, err := listUnits[*genPg.BuildingBlock](b) if err != nil { return nil, err } @@ -229,7 +244,7 @@ func textElementToModel(el element.Element) *model.Text { // builder's resolveLayout matches a layout by name + module to bind a page's // FormCall, so the header is all it needs. func (b *Backend) ListLayouts() ([]*pages.Layout, error) { - units, err := mprread.ListUnitsWithContainer[*genPg.Layout](b.reader) + units, err := listUnits[*genPg.Layout](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/security_read.go b/mdl/backend/modelsdk/security_read.go index c47a27f8b4..5d679b1c59 100644 --- a/mdl/backend/modelsdk/security_read.go +++ b/mdl/backend/modelsdk/security_read.go @@ -14,7 +14,7 @@ import ( // ListModuleSecurity reads every module's security document (its module roles) // through the codec engine. Mirrors the legacy reader.ListModuleSecurity. func (b *Backend) ListModuleSecurity() ([]*security.ModuleSecurity, error) { - units, err := mprread.ListUnitsWithContainer[*genSec.ModuleSecurity](b.reader) + units, err := listUnits[*genSec.ModuleSecurity](b) if err != nil { return nil, err } @@ -28,7 +28,7 @@ func (b *Backend) ListModuleSecurity() ([]*security.ModuleSecurity, error) { // GetModuleSecurity returns the module-security document whose container is // moduleID (its module roles). func (b *Backend) GetModuleSecurity(moduleID model.ID) (*security.ModuleSecurity, error) { - units, err := mprread.ListUnitsWithContainer[*genSec.ModuleSecurity](b.reader) + units, err := listUnits[*genSec.ModuleSecurity](b) if err != nil { return nil, err } diff --git a/mdl/backend/modelsdk/units.go b/mdl/backend/modelsdk/units.go index b3fd740785..0ac3afa2fa 100644 --- a/mdl/backend/modelsdk/units.go +++ b/mdl/backend/modelsdk/units.go @@ -5,6 +5,10 @@ package modelsdkbackend import ( "fmt" + "github.com/mendixlabs/mxcli/mdl/backend" + "github.com/mendixlabs/mxcli/modelsdk/element" + "github.com/mendixlabs/mxcli/modelsdk/mprread" + "github.com/mendixlabs/mxcli/mdl/types" "github.com/mendixlabs/mxcli/model" ) @@ -56,7 +60,30 @@ func (b *Backend) UpdateRawUnitOwningStorageGUIDs(unitID string, contents []byte // dedicated typed reader (e.g. JavaScript actions, data transformers). Delegates // to the codec reader. func (b *Backend) ListRawUnitsByType(typePrefix string) ([]*types.RawUnit, error) { - return b.reader.ListRawUnitsByType(typePrefix) + return b.reader.ListRawUnitsByTypeWhere(typePrefix, b.unitScope) +} + +// WithUnitScope returns a read view of the backend whose document lists hold +// only the units keep accepts, and which reads no other unit's contents on +// MPR v2. Everything else — single-document reads, settings, the unit +// table — is the backend's. A catalog build that rebuilds a few units lists +// through it, so it decodes those and not the project (ako/mxcli#1804). +// +// The view shares the backend's reader; it is for reading, and writes +// through it are not counted in the backend's write statistics. +func (b *Backend) WithUnitScope(keep func(unitID string) bool) backend.FullBackend { + view := *b + view.unitScope = keep + return &view +} + +// listUnits decodes the units of type T the backend reads: every one, or +// those in its unit scope (WithUnitScope). +func listUnits[T element.Element](b *Backend) ([]mprread.Unit[T], error) { + if b.unitScope != nil { + return mprread.ListUnitsWithContainerIn[T](b.reader, b.unitScope) + } + return mprread.ListUnitsWithContainer[T](b.reader) } func (b *Backend) ListUnits() ([]*types.UnitInfo, error) { diff --git a/mdl/backend/modelsdk/workflow_read.go b/mdl/backend/modelsdk/workflow_read.go index 31e79e9f3b..9e5ce73ad2 100644 --- a/mdl/backend/modelsdk/workflow_read.go +++ b/mdl/backend/modelsdk/workflow_read.go @@ -8,7 +8,6 @@ import ( "github.com/mendixlabs/mxcli/modelsdk/element" genMf "github.com/mendixlabs/mxcli/modelsdk/gen/microflows" genWf "github.com/mendixlabs/mxcli/modelsdk/gen/workflows" - "github.com/mendixlabs/mxcli/modelsdk/mprread" "github.com/mendixlabs/mxcli/sdk/workflows" ) @@ -26,7 +25,7 @@ import ( // path that reaches this method reads them. Unrecognised activity types decode to // a GenericWorkflowActivity carrying their $Type, so no activity is silently dropped. func (b *Backend) ListWorkflows() ([]*workflows.Workflow, error) { - units, err := mprread.ListUnitsWithContainer[*genWf.Workflow](b.reader) + units, err := listUnits[*genWf.Workflow](b) if err != nil { return nil, err } diff --git a/mdl/catalog/builder.go b/mdl/catalog/builder.go index 4afce78d0d..ef0418360b 100644 --- a/mdl/catalog/builder.go +++ b/mdl/catalog/builder.go @@ -3,6 +3,9 @@ package catalog import ( + "context" + "database/sql" + "errors" "fmt" "strconv" "strings" @@ -89,11 +92,11 @@ type Builder struct { snapshot *Snapshot progress ProgressFunc hierarchy *hierarchy - tx CatalogTx // Transaction for batched inserts - fullMode bool // If true, do full parsing (activities/widgets) - sourceMode bool // If true, build source FTS table (implies full) - communitiesMode bool // If true, run the graph-analysis pass (reads refs) - resolution float64 // Leiden resolution for the graph-analysis pass + tx *builderTx // Transaction for batched inserts; records row owners + fullMode bool // If true, do full parsing (activities/widgets) + sourceMode bool // If true, build source FTS table (implies full) + communitiesMode bool // If true, run the graph-analysis pass (reads refs) + resolution float64 // Leiden resolution for the graph-analysis pass describeFunc DescribeFunc // prevSource is the last source build's rows and basis, and @@ -101,6 +104,29 @@ type Builder struct { // unchanged documents keep their rows (builder_source_reuse.go). prevSource *PreviousSource sourceFingerprint string + entityDM map[string]string // entity ID -> its domain model's, from entityBasis + + // The incremental build (builder_incremental.go): the cached catalog to + // carry rows from, the mxcli build writing this one, every unit's basis + // and type, the units this build rebuilds (nil: all of them), the row + // owners it records, and how it went. + prevCatalogPath string + fingerprint string + basis map[string]string + unitTypes map[string]string + scope map[string]bool + rows *rowOwners + stats BuildStats + prevOwned map[string]bool // the previous build's per-unit passes' units + deps map[string]map[string]string // traced reads: unit -> name -> lookup (builder_coverage.go) + + // list is what the per-unit passes list documents from: the reader, or + // in an incremental build a view of it holding only the units in scope, + // so the others are neither read nor decoded. scopedReader makes that + // view; without one the passes list everything and skip what is out of + // scope. + list CatalogReader + scopedReader func(keep func(unitID string) bool) CatalogReader // The project's DefaultLanguageCode, resolved once per build by // defaultLanguage(). Texts (enumeration value captions today) are read in @@ -276,15 +302,25 @@ func NewBuilder(catalog *Catalog, reader CatalogReader) *Builder { return &Builder{ catalog: catalog, reader: reader, + list: reader, } } +// lister is what the per-unit passes list documents from (b.list), or the +// reader for a Builder made without NewBuilder, as a pass run on its own is. +func (b *Builder) lister() CatalogReader { + if b.list == nil { + return b.reader + } + return b.list +} + // Cached document accessors — each parses from BSON at most once per build. func (b *Builder) cachedMicroflows() ([]*microflows.Microflow, error) { if b.microflowCache == nil { var err error - b.microflowCache, err = b.reader.ListMicroflows() + b.microflowCache, err = b.lister().ListMicroflows() if err != nil { return nil, err } @@ -295,7 +331,7 @@ func (b *Builder) cachedMicroflows() ([]*microflows.Microflow, error) { func (b *Builder) cachedNanoflows() ([]*microflows.Nanoflow, error) { if b.nanoflowCache == nil { var err error - b.nanoflowCache, err = b.reader.ListNanoflows() + b.nanoflowCache, err = b.lister().ListNanoflows() if err != nil { return nil, err } @@ -306,7 +342,7 @@ func (b *Builder) cachedNanoflows() ([]*microflows.Nanoflow, error) { func (b *Builder) cachedRules() ([]*microflows.Rule, error) { if b.ruleCache == nil { var err error - b.ruleCache, err = b.reader.ListRules() + b.ruleCache, err = b.lister().ListRules() if err != nil { return nil, err } @@ -317,7 +353,7 @@ func (b *Builder) cachedRules() ([]*microflows.Rule, error) { func (b *Builder) cachedPages() ([]*pages.Page, error) { if b.pageCache == nil { var err error - b.pageCache, err = b.reader.ListPages() + b.pageCache, err = b.lister().ListPages() if err != nil { return nil, err } @@ -328,7 +364,7 @@ func (b *Builder) cachedPages() ([]*pages.Page, error) { func (b *Builder) cachedDomainModels() ([]*domainmodel.DomainModel, error) { if b.domainModelCache == nil { var err error - b.domainModelCache, err = b.reader.ListDomainModels() + b.domainModelCache, err = b.lister().ListDomainModels() if err != nil { return nil, err } @@ -339,7 +375,7 @@ func (b *Builder) cachedDomainModels() ([]*domainmodel.DomainModel, error) { func (b *Builder) cachedEnumerations() ([]*model.Enumeration, error) { if b.enumerationCache == nil { var err error - b.enumerationCache, err = b.reader.ListEnumerations() + b.enumerationCache, err = b.lister().ListEnumerations() if err != nil { return nil, err } @@ -350,7 +386,7 @@ func (b *Builder) cachedEnumerations() ([]*model.Enumeration, error) { func (b *Builder) cachedWorkflows() ([]*workflows.Workflow, error) { if b.workflowCache == nil { var err error - b.workflowCache, err = b.reader.ListWorkflows() + b.workflowCache, err = b.lister().ListWorkflows() if err != nil { return nil, err } @@ -361,7 +397,7 @@ func (b *Builder) cachedWorkflows() ([]*workflows.Workflow, error) { func (b *Builder) cachedBusinessEventServices() ([]*model.BusinessEventService, error) { if b.businessEventCache == nil { var err error - b.businessEventCache, err = b.reader.ListBusinessEventServices() + b.businessEventCache, err = b.lister().ListBusinessEventServices() if err != nil { return nil, err } @@ -372,7 +408,7 @@ func (b *Builder) cachedBusinessEventServices() ([]*model.BusinessEventService, func (b *Builder) cachedDatabaseConnections() ([]*model.DatabaseConnection, error) { if b.databaseConnectionCache == nil { var err error - b.databaseConnectionCache, err = b.reader.ListDatabaseConnections() + b.databaseConnectionCache, err = b.lister().ListDatabaseConnections() if err != nil { return nil, err } @@ -383,7 +419,7 @@ func (b *Builder) cachedDatabaseConnections() ([]*model.DatabaseConnection, erro func (b *Builder) cachedModuleSettings() ([]*types.ModuleSettings, error) { if b.moduleSettingsCache == nil { var err error - b.moduleSettingsCache, err = b.reader.ListModuleSettings() + b.moduleSettingsCache, err = b.lister().ListModuleSettings() if err != nil { return nil, err } @@ -391,9 +427,94 @@ func (b *Builder) cachedModuleSettings() ([]*types.ModuleSettings, error) { return b.moduleSettingsCache, nil } -// Build populates all catalog tables from MPR data. +// buildPhase is one pass of a build: the tables it writes, in dependency +// order. +type buildPhase struct { + name string + run func() error +} + +func (b *Builder) phases() []buildPhase { + return []buildPhase{ + {"modules", b.buildModules}, + {"entities", b.buildEntities}, + {"entity event handlers", b.buildEntityEventHandlers}, + {"associations", b.buildAssociations}, + {"enumerations", b.buildEnumerations}, + {"constants", b.buildConstants}, + {"constant values", b.buildConstantValues}, + {"java actions", b.buildJavaActions}, + {"javascript actions", func() error { + return b.buildSimpleNamedDocs("JavaScriptActions$JavaScriptAction", "javascript_actions", "JavaScript Actions") + }}, + {"image collections", func() error { + return b.buildSimpleNamedDocs("Images$ImageCollection", "image_collections", "Image Collections") + }}, + {"icon collections", func() error { + return b.buildSimpleNamedDocs("CustomIcons$CustomIconCollection", "icon_collections", "Icon Collections") + }}, + {"menus", func() error { return b.buildSimpleNamedDocs("Menus$MenuDocument", "menus", "Menus") }}, + // Page templates are indexed in their own right rather than as pages. They + // were previously swept into pages_data by a prefix-matched Forms$Page query; + // dropping them from the catalog entirely instead would make 46 documents in + // a stock Atlas project invisible to anything that enumerates a module. + {"page templates", func() error { return b.buildSimpleNamedDocs("Forms$PageTemplate", "page_templates", "Page Templates") }}, + {"data transformers", func() error { + return b.buildSimpleNamedDocs("DataTransformers$DataTransformer", "data_transformers", "Data Transformers") + }}, + {"message definition collections", func() error { + return b.buildSimpleNamedDocs("MessageDefinitions$MessageDefinitionCollection", "message_definition_collections", "Message Definition Collections") + }}, + {"message definitions", func() error { + return b.buildSimpleNamedDocs("MessageDefinitions$MessageDefinition2", "message_definitions", "Message Definitions") + }}, + {"scheduled events", b.buildScheduledEvents}, + {"queues", b.buildQueues}, + {"regular expressions", b.buildRegularExpressions}, + {"agent-editor documents", b.buildAgentEditorDocs}, + {"microflows", b.buildMicroflows}, + {"pages", b.buildPages}, + {"snippets", b.buildSnippets}, + {"building blocks", b.buildBuildingBlocks}, + {"layouts", b.buildLayouts}, + {"OData clients", b.buildODataClients}, + {"OData services", b.buildODataServices}, + {"workflows", b.buildWorkflows}, + {"business event services", b.buildBusinessEventServices}, + {"widget definitions", b.buildWidgetDefinitions}, + {"database connections", b.buildDatabaseConnections}, + {"jar dependencies", b.buildJarDependencies}, + {"REST clients", b.buildRestClients}, + {"published REST services", b.buildPublishedRestServices}, + {"external entities", b.buildExternalEntities}, + {"external actions", b.buildExternalActions}, + {"business events", b.buildBusinessEvents}, + {"contract entities", b.buildContractEntities}, + {"contract entity usage", b.buildContractEntityUsage}, + {"contract messages", b.buildContractMessages}, + {"navigation", b.buildNavigation}, + {"role mappings", b.buildRoleMappings}, + {"JSON structures", b.buildJsonStructures}, + {"import mappings", b.buildImportMappings}, + {"export mappings", b.buildExportMappings}, + // Before buildReferences, which resolves the attributes and associations + // each constraint names. + {"xpath expressions", b.buildXPathExpressions}, + {"references", b.buildReferences}, + {"permissions", b.buildPermissions}, + {"strings", b.buildStrings}, + {"source", b.buildSource}, + // Only for REFRESH CATALOG COMMUNITIES; reads the refs built above. + {"graph analysis", b.buildGraphAnalysis}, + } +} + +// Build populates all catalog tables from MPR data. With a previous catalog +// (SetPreviousCatalog) it carries over the rows of every unit nothing has +// changed under and rebuilds the rest (builder_incremental.go). func (b *Builder) Build(progress ProgressFunc) error { b.progress = progress + b.stats = BuildStats{} // Create snapshot b.snapshot = b.catalog.CreateSnapshot("", SnapshotSourceUnknown) @@ -403,239 +524,159 @@ func (b *Builder) Build(progress ProgressFunc) error { if err := b.buildHierarchy(); err != nil { return fmt.Errorf("failed to build hierarchy: %w", err) } - - // Start transaction for all inserts (major performance boost) - tx, err := b.catalog.db.Begin() - if err != nil { - return fmt.Errorf("failed to begin transaction: %w", err) - } - defer func() { - if tx != nil { - tx.Rollback() - } - }() - b.tx = tx - - // Build project record - if err := b.buildProject(); err != nil { - return fmt.Errorf("failed to build project: %w", err) - } - - // Build tables in dependency order - if err := b.buildModules(); err != nil { - return fmt.Errorf("failed to build modules: %w", err) - } - - if err := b.buildEntities(); err != nil { - return fmt.Errorf("failed to build entities: %w", err) - } - - if err := b.buildEntityEventHandlers(); err != nil { - return fmt.Errorf("failed to build entity event handlers: %w", err) - } - - if err := b.buildAssociations(); err != nil { - return fmt.Errorf("failed to build associations: %w", err) - } - - if err := b.buildEnumerations(); err != nil { - return fmt.Errorf("failed to build enumerations: %w", err) - } - - if err := b.buildConstants(); err != nil { - return fmt.Errorf("failed to build constants: %w", err) - } - - if err := b.buildConstantValues(); err != nil { - return fmt.Errorf("failed to build constant values: %w", err) - } - - if err := b.buildJavaActions(); err != nil { - return fmt.Errorf("failed to build java actions: %w", err) - } - - if err := b.buildSimpleNamedDocs("JavaScriptActions$JavaScriptAction", "javascript_actions", "JavaScript Actions"); err != nil { - return fmt.Errorf("failed to build javascript actions: %w", err) - } - - if err := b.buildSimpleNamedDocs("Images$ImageCollection", "image_collections", "Image Collections"); err != nil { - return fmt.Errorf("failed to build image collections: %w", err) - } - - if err := b.buildSimpleNamedDocs("CustomIcons$CustomIconCollection", "icon_collections", "Icon Collections"); err != nil { - return fmt.Errorf("failed to build icon collections: %w", err) - } - - if err := b.buildSimpleNamedDocs("Menus$MenuDocument", "menus", "Menus"); err != nil { - return fmt.Errorf("failed to build menus: %w", err) - } - - // Page templates are indexed in their own right rather than as pages. They - // were previously swept into pages_data by a prefix-matched Forms$Page query; - // dropping them from the catalog entirely instead would make 46 documents in - // a stock Atlas project invisible to anything that enumerates a module. - if err := b.buildSimpleNamedDocs("Forms$PageTemplate", "page_templates", "Page Templates"); err != nil { - return fmt.Errorf("failed to build page templates: %w", err) - } - - if err := b.buildSimpleNamedDocs("DataTransformers$DataTransformer", "data_transformers", "Data Transformers"); err != nil { - return fmt.Errorf("failed to build data transformers: %w", err) - } - - if err := b.buildSimpleNamedDocs("MessageDefinitions$MessageDefinitionCollection", "message_definition_collections", "Message Definition Collections"); err != nil { - return fmt.Errorf("failed to build message definition collections: %w", err) - } - - if err := b.buildSimpleNamedDocs("MessageDefinitions$MessageDefinition2", "message_definitions", "Message Definitions"); err != nil { - return fmt.Errorf("failed to build message definitions: %w", err) - } - - if err := b.buildScheduledEvents(); err != nil { - return fmt.Errorf("failed to build scheduled events: %w", err) - } - - if err := b.buildQueues(); err != nil { - return fmt.Errorf("failed to build queues: %w", err) - } - - if err := b.buildRegularExpressions(); err != nil { - return fmt.Errorf("failed to build regular expressions: %w", err) + if b.fingerprint != "" { + b.basis, b.unitTypes = b.unitBasis() } - if err := b.buildAgentEditorDocs(); err != nil { - return fmt.Errorf("failed to build agent-editor documents: %w", err) - } - - if err := b.buildMicroflows(); err != nil { - return fmt.Errorf("failed to build microflows: %w", err) - } - - if err := b.buildPages(); err != nil { - return fmt.Errorf("failed to build pages: %w", err) - } - - if err := b.buildSnippets(); err != nil { - return fmt.Errorf("failed to build snippets: %w", err) - } - - if err := b.buildBuildingBlocks(); err != nil { - return fmt.Errorf("failed to build building blocks: %w", err) - } - - if err := b.buildLayouts(); err != nil { - return fmt.Errorf("failed to build layouts: %w", err) - } - - if err := b.buildODataClients(); err != nil { - return fmt.Errorf("failed to build OData clients: %w", err) - } - - if err := b.buildODataServices(); err != nil { - return fmt.Errorf("failed to build OData services: %w", err) - } - - if err := b.buildWorkflows(); err != nil { - return fmt.Errorf("failed to build workflows: %w", err) - } - - if err := b.buildBusinessEventServices(); err != nil { - return fmt.Errorf("failed to build business event services: %w", err) - } - - if err := b.buildWidgetDefinitions(); err != nil { - return fmt.Errorf("failed to build widget definitions: %w", err) - } - - if err := b.buildDatabaseConnections(); err != nil { - return fmt.Errorf("failed to build database connections: %w", err) - } - - if err := b.buildJarDependencies(); err != nil { - return fmt.Errorf("failed to build jar dependencies: %w", err) - } - - if err := b.buildRestClients(); err != nil { - return fmt.Errorf("failed to build REST clients: %w", err) - } - - if err := b.buildPublishedRestServices(); err != nil { - return fmt.Errorf("failed to build published REST services: %w", err) - } - - if err := b.buildExternalEntities(); err != nil { - return fmt.Errorf("failed to build external entities: %w", err) - } - - if err := b.buildExternalActions(); err != nil { - return fmt.Errorf("failed to build external actions: %w", err) + sdb, ok := b.catalog.db.(*SqliteCatalogDB) + if !ok { + // A catalog not on SQLite (js/wasm) has no ATTACH to carry rows with. + tx, err := b.catalog.db.Begin() + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + b.stats.Reason = "the catalog is not on SQLite" + if err := b.buildAttempt(tx, nil, nil); err != nil { + return err + } + return b.markGraphAnalysis() } - - if err := b.buildBusinessEvents(); err != nil { - return fmt.Errorf("failed to build business events: %w", err) + if err := b.buildOn(sdb.RawDB()); err != nil { + return err } + // After the build's connection is back in the pool: an in-memory + // catalog lives on that one connection. + return b.markGraphAnalysis() +} - if err := b.buildContractEntities(); err != nil { - return fmt.Errorf("failed to build contract entities: %w", err) +// buildOn builds on one connection of db: an in-memory catalog lives on +// one connection, and an incremental build loads the previous catalog into it. +func (b *Builder) buildOn(db *sql.DB) error { + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + return fmt.Errorf("failed to open catalog connection: %w", err) + } + defer conn.Close() + + plan, reason := (*incrementalPlan)(nil), "no cache to carry from" + if b.prevCatalogPath != "" && b.fingerprint != "" && b.basis != nil { + var data []byte + if data, plan, reason = b.loadPrevious(ctx); plan != nil { + // Loaded once, and only to build on: a connection whose + // full-text tables were in use crashed SQLite when it was + // loaded again (modernc.org/sqlite), so a fallback empties the + // tables instead (clearCatalog). + if err := deserializeConn(conn, data); err != nil { + plan, reason = nil, "cannot load the cache: "+err.Error() + } + } } + loaded := plan != nil - if err := b.buildContractEntityUsage(); err != nil { - return fmt.Errorf("failed to build contract entity usage: %w", err) + // An incremental attempt whose result shows the plan missed a unit is + // rolled back and run again with the wider scope, once; after that, or + // when it fails any other way, the build is a full one. + for round := 1; plan != nil && round <= 2; round++ { + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) + } + b.stats.Rounds = round + err = b.buildAttempt(tx, plan, func() (map[string]bool, string) { return b.verifyIncremental(plan) }) + if err == nil { + return nil + } + var widen *widenScope + if !errors.As(err, &widen) { + var fallback *incrementalFallback + if !errors.As(err, &fallback) { + return err + } + reason = fallback.reason + break + } + reason = "the plan missed a document that names a changed one" + for id := range widen.units { + plan.scope[id] = true + } } - - if err := b.buildContractMessages(); err != nil { - return fmt.Errorf("failed to build contract messages: %w", err) + b.stats = BuildStats{Reason: reason} + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin transaction: %w", err) } - - if err := b.buildNavigation(); err != nil { - return fmt.Errorf("failed to build navigation: %w", err) + if loaded { + // The attempts were rolled back to the previous catalog. + if err := clearCatalog(tx); err != nil { + tx.Rollback() + return fmt.Errorf("failed to empty the catalog: %w", err) + } } + return b.buildAttempt(tx, nil, nil) +} - if err := b.buildRoleMappings(); err != nil { - return fmt.Errorf("failed to build role mappings: %w", err) - } +// widenScope rolls an incremental attempt back to run it again with more +// units in scope. +type widenScope struct{ units map[string]bool } - if err := b.buildJsonStructures(); err != nil { - return fmt.Errorf("failed to build JSON structures: %w", err) - } +func (w *widenScope) Error() string { return fmt.Sprintf("%d more units to rebuild", len(w.units)) } - if err := b.buildImportMappings(); err != nil { - return fmt.Errorf("failed to build import mappings: %w", err) - } +// incrementalFallback rolls an incremental attempt back to run a full build. +type incrementalFallback struct{ reason string } - if err := b.buildExportMappings(); err != nil { - return fmt.Errorf("failed to build export mappings: %w", err) - } +func (f *incrementalFallback) Error() string { return f.reason } - // Build XPath expressions table (full mode only). Before buildReferences, - // which resolves the attributes and associations each constraint names. - if err := b.buildXPathExpressions(); err != nil { - return fmt.Errorf("failed to build xpath expressions: %w", err) - } - - // Build cross-references (only in full mode) - if err := b.buildReferences(); err != nil { - return fmt.Errorf("failed to build references: %w", err) - } - - // Build permissions (only in full mode) - if err := b.buildPermissions(); err != nil { - return fmt.Errorf("failed to build permissions: %w", err) +// buildAttempt runs every phase in tx and commits. With a plan the catalog +// holds the previous build's rows: it first drops those of the units in +// scope and the project-wide ones, and verify checks the result before the +// commit. +func (b *Builder) buildAttempt(tx CatalogTx, plan *incrementalPlan, verify func() (map[string]bool, string)) error { + defer func() { + if tx != nil { + tx.Rollback() + } + }() + b.tx = newBuilderTx(tx, b) + b.rows = newRowOwners() + b.resetCollected() + if b.deps != nil { + b.deps = map[string]map[string]string{} + } + b.scope = nil + b.list = b.reader + if plan != nil { + b.scope = plan.scope + if b.scopedReader != nil { + scope := plan.scope + b.list = b.scopedReader(func(id string) bool { return scope[id] }) + } + carried, err := b.dropRebuiltRows() + if err != nil { + return &incrementalFallback{"cannot reuse the cached rows: " + err.Error()} + } + b.stats.Incremental = true + b.stats.Carried = carried + b.stats.Rebuilt = len(plan.scope) } - // Build strings FTS table (full mode only) - if err := b.buildStrings(); err != nil { - return fmt.Errorf("failed to build strings: %w", err) + // Build project record + if err := b.buildProject(); err != nil { + return fmt.Errorf("failed to build project: %w", err) } - // Build source FTS table (source mode only) - if err := b.buildSource(); err != nil { - return fmt.Errorf("failed to build source: %w", err) + for _, p := range b.phases() { + b.disown() + if err := p.run(); err != nil { + return fmt.Errorf("failed to build %s: %w", p.name, err) + } } - - // Graph analysis (communities/cycles/layers/centrality) — only when requested - // via REFRESH CATALOG COMMUNITIES; reads the refs table built above. - if err := b.buildGraphAnalysis(); err != nil { - return fmt.Errorf("failed to build graph analysis: %w", err) + b.disown() + if b.rows.err != nil { + if plan == nil { + return fmt.Errorf("failed to build catalog: %w", b.rows.err) + } + return &incrementalFallback{b.rows.err.Error()} } // Update snapshot with object count @@ -646,32 +687,79 @@ func (b *Builder) Build(progress ProgressFunc) error { return fmt.Errorf("failed to build snapshot: %w", err) } + // Last, so the ranges include every row written above. + if err := b.persistOwners(); err != nil { + return fmt.Errorf("failed to record row owners: %w", err) + } + if verify != nil { + if more, reason := verify(); reason != "" { + return &incrementalFallback{reason} + } else if len(more) > 0 { + return &widenScope{more} + } + } + if plan != nil { + b.say(incrementalSummary(b.stats), b.stats.Rebuilt) + } + // Commit transaction if err := tx.Commit(); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } tx = nil // Prevent rollback in defer + b.tx = nil + return nil +} - // Same flag AddGraphAnalysis writes, for the build-it-in-one-pass route. - // Both routes must set it or an empty cycles table stays ambiguous down - // whichever one the caller took (mendixlabs/mxcli#1060). - if b.communitiesMode { - if err := b.catalog.SetMeta(MetaGraphAnalysis, time.Now().Format(time.RFC3339)); err != nil { - return err - } - if err := b.catalog.SetMeta(MetaGraphResolution, - strconv.FormatFloat(effectiveResolution(b.resolution), 'g', -1, 64)); err != nil { - return err - } +// markGraphAnalysis records that the build ran the graph pass: the same flag +// AddGraphAnalysis writes, for the build-it-in-one-pass route. Both routes +// must set it or an empty cycles table stays ambiguous down whichever one +// the caller took (mendixlabs/mxcli#1060). +func (b *Builder) markGraphAnalysis() error { + if !b.communitiesMode { + return nil } - b.tx = nil + if err := b.catalog.SetMeta(MetaGraphAnalysis, time.Now().Format(time.RFC3339)); err != nil { + return err + } + return b.catalog.SetMeta(MetaGraphResolution, + strconv.FormatFloat(effectiveResolution(b.resolution), 'g', -1, 64)) +} - return nil +// resetCollected clears what the passes collect for later ones and the +// document lists, so a second attempt does not inherit the first one's. +func (b *Builder) resetCollected() { + b.microflowCache = nil + b.nanoflowCache = nil + b.ruleCache = nil + b.pageCache = nil + b.domainModelCache = nil + b.enumerationCache = nil + b.workflowCache = nil + b.businessEventCache = nil + b.databaseConnectionCache = nil + b.moduleSettingsCache = nil + b.scheduledEventRefs = nil + b.regexRuleRefs = nil + b.publishedRestRefs = nil + b.eventHandlerRefs = nil + b.serviceMicroflowRefs = nil + b.widgetExtraRefs = nil } +// report is a pass's progress line: the rows it wrote to a table. An +// incremental build writes only the rebuilt units' rows, so a count would +// read as the table's size and is left out; say reports regardless. func (b *Builder) report(table string, count int) { + if b.scope == nil { + b.say(table, count) + } +} + +// say reports a progress line in every build. +func (b *Builder) say(line string, count int) { if b.progress != nil { - b.progress(table, count) + b.progress(line, count) } } diff --git a/mdl/catalog/builder_activities_loops_test.go b/mdl/catalog/builder_activities_loops_test.go index d78a0fc530..e8d906992a 100644 --- a/mdl/catalog/builder_activities_loops_test.go +++ b/mdl/catalog/builder_activities_loops_test.go @@ -103,7 +103,7 @@ func buildFlowsForTest(t *testing.T, mfs []*microflows.Microflow, nfs []*microfl if err != nil { t.Fatal(err) } - b.tx = tx + b.tx = wrapTx(tx) if err := b.buildMicroflows(); err != nil { t.Fatalf("buildMicroflows: %v", err) } diff --git a/mdl/catalog/builder_agenteditor.go b/mdl/catalog/builder_agenteditor.go index 2b25697cd5..f7a06e6a58 100644 --- a/mdl/catalog/builder_agenteditor.go +++ b/mdl/catalog/builder_agenteditor.go @@ -3,8 +3,6 @@ package catalog import ( - "database/sql" - "go.mongodb.org/mongo-driver/v2/bson" ) @@ -24,18 +22,18 @@ var agentEditorTable = map[string]string{ // which the catalog does not need), so this reads through the raw-unit surface // and decodes only the wrapper fields — no CatalogReader/backend change. func (b *Builder) buildAgentEditorDocs() error { - units, err := b.reader.ListRawUnitsByType("CustomBlobDocuments$CustomBlobDocument") + units, err := b.lister().ListRawUnitsByType("CustomBlobDocuments$CustomBlobDocument") if err != nil { return err } - stmts := map[string]*sql.Stmt{} + stmts := map[string]*Stmt{} defer func() { for _, s := range stmts { s.Close() } }() - stmtFor := func(table string) (*sql.Stmt, error) { + stmtFor := func(table string) (*Stmt, error) { if s, ok := stmts[table]; ok { return s, nil } @@ -53,6 +51,9 @@ func (b *Builder) buildAgentEditorDocs() error { count := 0 for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc struct { Name string `bson:"Name"` Documentation string `bson:"Documentation"` @@ -82,6 +83,7 @@ func (b *Builder) buildAgentEditorDocs() error { } count++ } + b.disown() b.report("Agent-editor documents", count) return nil diff --git a/mdl/catalog/builder_associations.go b/mdl/catalog/builder_associations.go index cb06a3d3dd..387338200c 100644 --- a/mdl/catalog/builder_associations.go +++ b/mdl/catalog/builder_associations.go @@ -23,16 +23,19 @@ func (b *Builder) buildAssociations() error { return err } - // Build entity ID -> qualified name lookup (reuse already-parsed domain models). - moduleNames := make(map[model.ID]string) - entityNames := make(map[model.ID]string) - for _, dm := range domainModels { - modID := b.hierarchy.findModuleID(dm.ContainerID) - modName := b.hierarchy.getModuleName(modID) - moduleNames[dm.ContainerID] = modName - for _, entity := range dm.Entities { - entityNames[entity.ID] = modName + "." + entity.Name + // Entity ID -> qualified name. In an incremental build the domain + // models listed are only the rebuilt ones, so the names come from the + // entities buildEntities wrote, which hold every module's. + entityNames := map[model.ID]string{} + if b.scope == nil { + for _, dm := range domainModels { + modName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(dm.ContainerID)) + for _, entity := range dm.Entities { + entityNames[entity.ID] = modName + "." + entity.Name + } } + } else if entityNames, err = b.entityNamesByID(); err != nil { + return err } stmt, err := b.tx.Prepare(` @@ -51,7 +54,10 @@ func (b *Builder) buildAssociations() error { count := 0 for _, dm := range domainModels { - modName := moduleNames[dm.ContainerID] + if !b.own(string(dm.ID)) { + continue + } + modName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(dm.ContainerID)) for _, assoc := range dm.Associations { from := entityNames[assoc.ParentID] @@ -62,6 +68,8 @@ func (b *Builder) buildAssociations() error { if to == "" { to = string(assoc.ChildID) } + b.dependsOn(siteEntityByID, from) + b.dependsOn(siteEntityByID, to) // Mendix's pointer names are inverted relative to MDL (CLAUDE.md): // the Child* delete behaviour belongs to the TO end, Parent* to FROM. toDB, toMsg := deleteBehaviorColumns(assoc.ChildDeleteBehavior) @@ -91,6 +99,7 @@ func (b *Builder) buildAssociations() error { if from == "" { from = string(ca.ParentID) } + b.dependsOn(siteEntityByID, from) // Same inversion as above; ChildRef is the TO end by qualified name. toDB, toMsg := deleteBehaviorColumns(ca.ChildDeleteBehavior) fromDB, fromMsg := deleteBehaviorColumns(ca.ParentDeleteBehavior) @@ -114,7 +123,26 @@ func (b *Builder) buildAssociations() error { count++ } } + b.disown() b.report("Associations", count) return nil } + +// entityNamesByID maps every catalogued entity's ID to its qualified name. +func (b *Builder) entityNamesByID() (map[model.ID]string, error) { + rows, err := b.tx.Query(`SELECT Id, QualifiedName FROM entities_data`) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[model.ID]string{} + for rows.Next() { + var id, qn string + if err := rows.Scan(&id, &qn); err != nil { + return nil, err + } + out[model.ID(id)] = qn + } + return out, rows.Err() +} diff --git a/mdl/catalog/builder_audit_members_test.go b/mdl/catalog/builder_audit_members_test.go index 3bd4e9d426..419644ff05 100644 --- a/mdl/catalog/builder_audit_members_test.go +++ b/mdl/catalog/builder_audit_members_test.go @@ -55,7 +55,7 @@ func TestEntitiesCarryTheAuditMembers(t *testing.T) { if err != nil { t.Fatal(err) } - b.tx = tx + b.tx = wrapTx(tx) if err := b.buildEntities(); err != nil { t.Fatalf("buildEntities: %v", err) } diff --git a/mdl/catalog/builder_constants.go b/mdl/catalog/builder_constants.go index 9cabb53ff1..96e3130892 100644 --- a/mdl/catalog/builder_constants.go +++ b/mdl/catalog/builder_constants.go @@ -5,7 +5,7 @@ package catalog import "strings" func (b *Builder) buildConstants() error { - constants, err := b.reader.ListConstants() + constants, err := b.lister().ListConstants() if err != nil { return err } @@ -24,6 +24,9 @@ func (b *Builder) buildConstants() error { projectID, snapshotID := b.snapshotMeta() for _, c := range constants { + if !b.own(string(c.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(c.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + c.Name @@ -55,6 +58,7 @@ func (b *Builder) buildConstants() error { return err } } + b.disown() b.report("Constants", len(constants)) return nil diff --git a/mdl/catalog/builder_contract.go b/mdl/catalog/builder_contract.go index 202a26e29d..679783aef8 100644 --- a/mdl/catalog/builder_contract.go +++ b/mdl/catalog/builder_contract.go @@ -13,7 +13,7 @@ import ( // buildContractEntities parses cached $metadata from consumed OData services // and populates the contract_entities and contract_actions catalog tables. func (b *Builder) buildContractEntities() error { - services, err := b.reader.ListConsumedODataServices() + services, err := b.lister().ListConsumedODataServices() if err != nil { return err } @@ -47,6 +47,9 @@ func (b *Builder) buildContractEntities() error { actionCount := 0 for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } if svc.Metadata == "" { continue } @@ -121,6 +124,7 @@ func (b *Builder) buildContractEntities() error { actionCount++ } } + b.disown() b.report("Contract Entities", entityCount) if actionCount > 0 { @@ -180,6 +184,9 @@ func (b *Builder) buildContractMessages() error { count := 0 for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } if svc.Document == "" { continue } @@ -231,6 +238,7 @@ func (b *Builder) buildContractMessages() error { count++ } } + b.disown() b.report("Contract Messages", count) return nil diff --git a/mdl/catalog/builder_coverage.go b/mdl/catalog/builder_coverage.go new file mode 100644 index 0000000000..10262a271a --- /dev/null +++ b/mdl/catalog/builder_coverage.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "regexp" + "slices" + "sort" + "strings" +) + +// Lookup sites, and the unit types whose change can alter what each one +// answers: a name a member lookup probes can only start or stop resolving +// when a domain model or an enumeration changes, whatever unit happens to +// hold a name with that prefix (an icon collection's `Mod.Icons.pencil` is +// no member). +const ( + siteMemberRefs = "member refs" + siteAssocEnds = "association ends" + siteEntityByID = "entity by ID" + siteFlowByID = "flow by ID" +) + +var siteHolderTypes = map[string][]string{ + siteMemberRefs: {"DomainModels$DomainModel", "Enumerations$Enumeration"}, + siteAssocEnds: {"DomainModels$DomainModel"}, + siteEntityByID: {"DomainModels$DomainModel"}, + siteFlowByID: {"Microflows$Microflow", "Microflows$Nanoflow", "Microflows$Rule"}, +} + +// qualifiedName is what a lookup of a model name can be: identifiers joined +// by dots. A path (`../datasource`) or a sentence is not one. +var qualifiedName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)+$`) + +// Ref coverage (ako/mxcli#1804). +// +// An incremental build rebuilds a document when a unit it names in refs +// changes. That is only correct if refs records every other document a pass +// reads while writing the document's rows: a read refs does not cover leaves +// stale rows behind. Those reads happen in a few lookups — the member names +// a unit's strings resolve against, the ends of the association a retrieve +// follows, the names of entities and flows found by ID — and each of them +// reports what it looked up, found or not (a name that is missing today +// changes the rows when it appears). With tracing on, DependencyCoverage +// then asks of every read what the plan asks of a change: would a change to +// the unit holding that name rebuild the reader? + +// SetTraceDependencies records, during the next Build, every name a pass +// looks up for a document. It is for tests and diagnosis: DependencyCoverage +// reports the result. +func (b *Builder) SetTraceDependencies(on bool) { + b.deps = nil + if on { + b.deps = map[string]map[string]string{} + } +} + +// dependsOn records that the document being built read name, at site. +func (b *Builder) dependsOn(site, name string) { + if b.deps == nil || !qualifiedName.MatchString(name) { + return + } + owner := b.currentOwner() + if owner == "" || owner == skippedOwner { + return // a project-wide row is written again by every build + } + reads := b.deps[owner] + if reads == nil { + reads = map[string]string{} + b.deps[owner] = reads + } + if _, seen := reads[name]; !seen { + reads[name] = site + } +} + +// probe is dependsOn for one site, for the lookups that take a callback. +func (b *Builder) probe(site string) func(string) { + if b.deps == nil { + return nil + } + return func(name string) { b.dependsOn(site, name) } +} + +// DependencyCoverage is how many of the traced reads the plan covers. +type DependencyCoverage struct { + Reads int // reads of a name some unit holds, in another unit + Covered int // ... that a change to that unit would rebuild the reader for + Self int // reads of the reader's own unit + Unowned int // reads of a name no unit holds (a missing name; the build's check covers its appearance) + Uncovered []UncoveredRead +} + +// UncoveredRead is a read the plan would not follow. +type UncoveredRead struct { + Reader string // the reading document's qualified name (or unit ID) + Name string // what it read + Holder string // a qualified name of the unit that holds it + Site string // which lookup read it +} + +// DependencyCoverage checks the reads traced during the last Build against +// the catalog it built. +func (b *Builder) DependencyCoverage() (*DependencyCoverage, error) { + g, err := loadRefGraph(b.catalog.db.Query) + if err != nil { + return nil, err + } + // holder is the unit holding name, or the nearest member-prefix that a + // unit holds (`M.E.Attr` is held by M.E's domain model), among the units + // whose change can alter what the site's lookup answers. A name no + // unit holds (a System entity) has no holder. + holder := func(site, name string) string { + types := siteHolderTypes[site] + for n := name; n != ""; { + for _, u := range g.unitOf[n] { + t, isUnit := b.unitTypes[u] + if isUnit && (len(types) == 0 || slices.Contains(types, t)) { + return u + } + } + i := strings.LastIndexByte(n, '.') + if i < 0 { + return "" + } + n = n[:i] + } + return "" + } + label := func(unit string) string { + if names := g.namesOf[unit]; len(names) > 0 { + sorted := append([]string(nil), names...) + sort.Strings(sorted) + return sorted[0] + } + return unit + } + cov := &DependencyCoverage{} + rebuilds := map[string]map[string]bool{} // holder -> the units a change to it rebuilds + for reader, reads := range b.deps { + for name, site := range reads { + h := holder(site, name) + switch { + case h == "": + cov.Unowned++ + continue + case h == reader: + cov.Self++ + continue + } + cov.Reads++ + deps, ok := rebuilds[h] + if !ok { + deps = g.dependents(g.namesOf[h]) + rebuilds[h] = deps + } + if deps[reader] { + cov.Covered++ + continue + } + cov.Uncovered = append(cov.Uncovered, UncoveredRead{label(reader), name, label(h), site}) + } + } + sort.Slice(cov.Uncovered, func(i, j int) bool { + a, c := cov.Uncovered[i], cov.Uncovered[j] + if a.Site != c.Site { + return a.Site < c.Site + } + if a.Reader != c.Reader { + return a.Reader < c.Reader + } + return a.Name < c.Name + }) + return cov, nil +} diff --git a/mdl/catalog/builder_entity_events.go b/mdl/catalog/builder_entity_events.go index 1117953376..0e0e1e601f 100644 --- a/mdl/catalog/builder_entity_events.go +++ b/mdl/catalog/builder_entity_events.go @@ -18,7 +18,7 @@ import ( // eventHandlerRef is one entity → microflow edge for an event handler, held // until buildReferences (a later pass) emits it. -type eventHandlerRef struct{ entityQualifiedName, moduleName, microflow string } +type eventHandlerRef struct{ owner, entityQualifiedName, moduleName, microflow string } // entityEventHandlerRow is one row of entity_event_handlers_data, plus the // fields the edge needs. Built by a pure function so the mapping — and the @@ -88,6 +88,9 @@ func (b *Builder) buildEntityEventHandlers() error { count := 0 for _, dm := range domainModels { + if !b.own(string(dm.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -102,6 +105,7 @@ func (b *Builder) buildEntityEventHandlers() error { return err } b.eventHandlerRefs = append(b.eventHandlerRefs, eventHandlerRef{ + owner: b.currentOwner(), entityQualifiedName: row.entityQualifiedName, moduleName: row.moduleName, microflow: row.microflow, @@ -110,6 +114,7 @@ func (b *Builder) buildEntityEventHandlers() error { } } } + b.disown() b.report("Entity Event Handlers", count) return nil diff --git a/mdl/catalog/builder_external.go b/mdl/catalog/builder_external.go index 59628b6700..3c3df3cbd7 100644 --- a/mdl/catalog/builder_external.go +++ b/mdl/catalog/builder_external.go @@ -3,6 +3,7 @@ package catalog import ( + "bytes" "crypto/sha256" "fmt" "strings" @@ -13,7 +14,7 @@ import ( // buildExternalEntities populates the external_entities catalog table // from domain model entities that have an OData remote entity source. func (b *Builder) buildExternalEntities() error { - domainModels, err := b.reader.ListDomainModels() + domainModels, err := b.cachedDomainModels() if err != nil { return err } @@ -34,6 +35,9 @@ func (b *Builder) buildExternalEntities() error { count := 0 for _, dm := range domainModels { + if !b.own(string(dm.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -83,6 +87,7 @@ func (b *Builder) buildExternalEntities() error { count++ } } + b.disown() b.report("External Entities", count) return nil @@ -91,11 +96,7 @@ func (b *Builder) buildExternalEntities() error { // buildExternalActions populates the external_actions catalog table // by scanning all microflows and nanoflows for CallExternalAction activities. func (b *Builder) buildExternalActions() error { - mfs, err := b.reader.ListMicroflows() - if err != nil { - return err - } - nfs, err := b.reader.ListNanoflows() + mfs, nfs, err := b.externalActionFlows() if err != nil { return err } @@ -209,6 +210,43 @@ func (b *Builder) buildExternalActions() error { return nil } +// callExternalActionType is the storage name of a call-external-action action. +const callExternalActionType = "Microflows$CallExternalAction" + +// externalActionFlows lists the flows that may call an external action. The +// table aggregates over every flow, so it is rebuilt every time; an +// incremental build decodes only the flows whose stored bytes name the action +// type, keeping their order, instead of every flow in the project. +func (b *Builder) externalActionFlows() ([]*microflows.Microflow, []*microflows.Nanoflow, error) { + if b.list == b.reader || b.scopedReader == nil { + mfs, err := b.cachedMicroflows() + if err != nil { + return nil, nil, err + } + nfs, err := b.cachedNanoflows() + return mfs, nfs, err + } + calling := map[string]bool{} + for _, typeName := range []string{"Microflows$Microflow", "Microflows$Nanoflow"} { + units, err := b.reader.ListRawUnitsByType(typeName) + if err != nil { + return nil, nil, err + } + for _, u := range units { + if bytes.Contains(u.Contents, []byte(callExternalActionType)) { + calling[string(u.ID)] = true + } + } + } + view := b.scopedReader(func(id string) bool { return calling[id] }) + mfs, err := view.ListMicroflows() + if err != nil { + return nil, nil, err + } + nfs, err := view.ListNanoflows() + return mfs, nfs, err +} + // isODataEntitySource reports whether an entity's Source marks it as consumed // from an OData service. // diff --git a/mdl/catalog/builder_graph.go b/mdl/catalog/builder_graph.go index f5009eed09..486616aa93 100644 --- a/mdl/catalog/builder_graph.go +++ b/mdl/catalog/builder_graph.go @@ -85,11 +85,12 @@ func (c *Catalog) AddGraphAnalysis(resolution float64) error { } b := &Builder{ catalog: c, - tx: tx, snapshot: &Snapshot{ID: snapID}, communitiesMode: true, resolution: resolution, + rows: newRowOwners(), } + b.tx = newBuilderTx(tx, b) if err := b.buildGraphAnalysis(); err != nil { _ = tx.Rollback() return err diff --git a/mdl/catalog/builder_incremental.go b/mdl/catalog/builder_incremental.go new file mode 100644 index 0000000000..371dc4fcf1 --- /dev/null +++ b/mdl/catalog/builder_incremental.go @@ -0,0 +1,659 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "context" + "database/sql" + "fmt" + "os" + "sort" + "strings" +) + +// deserializeConn replaces conn's main database with data. +func deserializeConn(conn *sql.Conn, data []byte) error { + return conn.Raw(func(dc any) error { + d, ok := dc.(interface{ Deserialize([]byte) error }) + if !ok { + return fmt.Errorf("the SQLite driver cannot deserialize") + } + return d.Deserialize(data) + }) +} + +// Incremental catalog builds (ako/mxcli#1804). +// +// A refresh rebuilt every table from every unit after any save, which on a +// large project is seconds of decoding and inserting rows that are exactly +// the rows already in the cache. A build now records which unit each row came +// from (builderTx), and the next build rebuilds only: +// +// - the units whose contents or container changed, were added or removed; +// - the units holding a document whose refs name one of them, before or +// after the change: a pass resolves names against other documents (an +// association's ends, a called flow, an attribute's type), so a document +// that names a changed one may write different rows. A specialization +// counts as naming whatever its generalization holds, transitively, +// because member names resolve through the inheritance chain. +// +// The build starts from the cached catalog itself, loaded into memory, and +// drops the rows of those units and the rows written by no unit (project-wide +// passes, projections over other tables), which it produces again. Every +// other unit's rows stay where they are: nothing is copied, decoded or +// re-indexed for them. +// +// Anything the plan cannot vouch for makes it a full build, as before: a +// change to a unit no per-unit pass reads (project settings, security, a +// module or folder), another build mode or mxcli build, a schema change, a +// row written for a document the pass was skipping. Reuse is only ever an +// optimisation of a full build, never a different answer — the tests compare +// every table of an incremental build with a forced full one. + +// MetaCatalogFingerprint records which mxcli build wrote the catalog's rows. +const MetaCatalogFingerprint = "catalog_fingerprint" + +// SetPreviousCatalog gives the build the cached catalog it may carry rows +// over from. +func (b *Builder) SetPreviousCatalog(path string) { + b.prevCatalogPath = path +} + +// SetScopedReader supplies the view an incremental build lists documents +// from: one holding only the units keep accepts, which reads and decodes no +// other. Without it the build lists every document and skips the ones whose +// rows it carries. +func (b *Builder) SetScopedReader(f func(keep func(unitID string) bool) CatalogReader) { + b.scopedReader = f +} + +// SetFingerprint identifies the mxcli build writing the catalog: rows written +// by another build are never carried over, since it may write them +// differently. +func (b *Builder) SetFingerprint(fp string) { + b.fingerprint = fp +} + +// BuildStats says how the last Build went. +type BuildStats struct { + Incremental bool // rows of unchanged units were carried over + Reason string // why the build was full, when it was + Rebuilt int // units whose rows were produced again + Carried int // units whose rows were copied + Rounds int // incremental passes run (a second one widens the scope) +} + +// Stats reports how the last Build went. +func (b *Builder) Stats() BuildStats { + return b.stats +} + +// own marks the rows written from here on as coming from unit, and reports +// whether this build writes them. It is false for a unit whose rows were +// carried over: the pass skips the document. +func (b *Builder) own(unit string) bool { + if b.rows == nil { + b.rows = newRowOwners() + } + b.rows.owned[unit] = true + if b.scope != nil && !b.scope[unit] { + b.rows.owner = skippedOwner + return false + } + b.rows.owner = unit + return true +} + +// currentOwner is the unit the rows being written now come from, for a pass +// that collects rows a later one writes. +func (b *Builder) currentOwner() string { + if b.rows == nil { + return "" + } + return b.rows.owner +} + +// as marks the rows written from here on as coming from owner, which a pass +// recorded when it collected them: a unit in scope, or "" for project-wide. +func (b *Builder) as(owner string) { + if b.rows != nil { + b.rows.owner = owner + } +} + +// ownRows is own for a pass that walks units of every type (the strings +// index): it attributes the unit's rows to it without counting the unit as +// one a per-unit pass reads. A unit only such a walk visits — the project +// settings, a module, a folder — is read by project-wide passes too, and a +// change to it still makes the build a full one. +func (b *Builder) ownRows(unit string) bool { + if b.rows == nil { + b.rows = newRowOwners() + } + if b.scope != nil && !b.scope[unit] { + b.rows.owner = skippedOwner + return false + } + b.rows.owner = unit + return true +} + +// disown marks the rows written from here on as project-wide. +func (b *Builder) disown() { + if b.rows != nil { + b.rows.owner = "" + } +} + +// buildMode is the mode name the cache records for this build. +func (b *Builder) buildMode() string { + switch { + case b.sourceMode: + return "source" + case b.fullMode: + return "full" + } + return "fast" +} + +// unitBasis is what a unit's rows depend on of the unit itself: its content +// hash and its container. The container is a column of the Unit table, not +// part of the bytes, and rows print the folder and module. Nil when the +// reader cannot hash units, which carries and records nothing. +func (b *Builder) unitBasis() (map[string]string, map[string]string) { + h, ok := b.reader.(unitHasher) + if !ok { + return nil, nil + } + hashes, err := h.UnitContentHashes() + if err != nil { + return nil, nil + } + units, err := b.reader.ListUnits() + if err != nil { + return nil, nil + } + types := map[string]string{} + for _, u := range units { + id := string(u.ID) + types[id] = u.Type + if hash, ok := hashes[id]; ok { + hashes[id] = hash + "|" + string(u.ContainerID) + } + } + return hashes, types +} + +// incrementalPlan is what an incremental build rebuilds. +type incrementalPlan struct { + scope map[string]bool // units whose rows are produced again + changed map[string]bool // units changed, added or removed + added map[string]bool // units the previous build did not know +} + +// prevCatalog is the previous catalog's basis. +type prevCatalog struct { + basis map[string]string + owned map[string]bool +} + +// loadPrevious reads the cached catalog and plans the build from it, on a +// scratch in-memory copy; it returns the bytes the build loads to build on, +// or says why the build has to be a full one. Planning from the bytes the +// build then keeps means a cache another process replaces meanwhile cannot +// be planned from one version and built on another. +func (b *Builder) loadPrevious(ctx context.Context) ([]byte, *incrementalPlan, string) { + data, err := os.ReadFile(b.prevCatalogPath) + if err != nil { + return nil, nil, "no cache to carry from" + } + scratch, err := sql.Open("sqlite", ":memory:") + if err != nil { + return nil, nil, err.Error() + } + defer scratch.Close() + conn, err := scratch.Conn(ctx) + if err != nil { + return nil, nil, err.Error() + } + defer conn.Close() + if err := deserializeConn(conn, data); err != nil { + return nil, nil, "cannot load the cache: " + err.Error() + } + plan, reason := b.planFrom(ctx, conn) + return data, plan, reason +} + +// planFrom plans the build from the previous catalog on conn. +func (b *Builder) planFrom(ctx context.Context, conn *sql.Conn) (*incrementalPlan, string) { + meta := func(key string) string { + var v string + _ = conn.QueryRowContext(ctx, `SELECT Value FROM catalog_meta WHERE Key = ?`, key).Scan(&v) + return v + } + switch { + case meta(MetaSchemaVersion) != CatalogSchemaVersion: + return nil, "the cache has another schema version" + case meta(MetaCatalogFingerprint) != b.fingerprint: + return nil, "the cache was written by another mxcli build" + case meta(MetaBuildMode) != b.buildMode(): + return nil, "the cache was built in another mode" + } + var snap string + _ = conn.QueryRowContext(ctx, `SELECT SnapshotId FROM snapshots`).Scan(&snap) + if b.snapshot == nil || snap != b.snapshot.ID { + return nil, "the cache records another snapshot" + } + + p := &prevCatalog{basis: map[string]string{}, owned: map[string]bool{}} + rows, err := conn.QueryContext(ctx, `SELECT UnitId, Basis, Owned FROM catalog_units`) + if err != nil { + return nil, "the cache records no unit basis" + } + for rows.Next() { + var id, basis string + var owned int + if err := rows.Scan(&id, &basis, &owned); err != nil { + rows.Close() + return nil, err.Error() + } + p.basis[id] = basis + p.owned[id] = owned != 0 + } + rows.Close() + if len(p.basis) == 0 { + return nil, "the cache records no unit basis" + } + b.prevOwned = p.owned + return b.planIncremental(querier(ctx, conn.QueryContext), p) +} + +// planIncremental decides which units to rebuild, or says why the build has +// to be full. q reads the previous catalog. +func (b *Builder) planIncremental(q queryFunc, prev *prevCatalog) (*incrementalPlan, string) { + plan := &incrementalPlan{scope: map[string]bool{}, changed: map[string]bool{}, added: map[string]bool{}} + for id, basis := range b.basis { + old, ok := prev.basis[id] + if !ok { + plan.changed[id] = true + plan.added[id] = true + } else if old != basis { + plan.changed[id] = true + } + } + for id := range prev.basis { + if _, ok := b.basis[id]; !ok { + plan.changed[id] = true + } + } + for id := range plan.changed { + if !plan.added[id] && !prev.owned[id] { + return nil, fmt.Sprintf("a %s changed", b.unitKind(id)) + } + plan.scope[id] = true + } + if len(plan.changed) == 0 { + return plan, "" + } + names, err := unitNames(q, plan.changed) + if err != nil { + return nil, "the cache records no unit names: " + err.Error() + } + deps, err := dependentUnits(q, names) + if err != nil { + return nil, "the cache's references are unreadable: " + err.Error() + } + for id := range deps { + plan.scope[id] = true + } + return plan, "" +} + +// queryFunc runs a query on the connection or transaction a step has. +type queryFunc func(query string, args ...any) (*sql.Rows, error) + +func querier(ctx context.Context, f func(context.Context, string, ...any) (*sql.Rows, error)) queryFunc { + return func(query string, args ...any) (*sql.Rows, error) { return f(ctx, query, args...) } +} + +// verifyIncremental checks an incremental attempt before it commits. It +// returns why the build has to be a full one, or the units the plan missed: +// those holding a document that names what a changed unit is called now (a +// rename, or a document added under a name others already used). +func (b *Builder) verifyIncremental(plan *incrementalPlan) (map[string]bool, string) { + for id := range plan.changed { + if _, exists := b.basis[id]; exists && !b.rows.owned[id] { + return nil, fmt.Sprintf("a %s changed", b.unitKind(id)) + } + } + q := queryFunc(b.tx.CatalogTx.Query) + names, err := unitNames(q, plan.changed) + if err != nil { + return nil, err.Error() + } + deps, err := dependentUnits(q, names) + if err != nil { + return nil, err.Error() + } + more := map[string]bool{} + for id := range deps { + if !plan.scope[id] { + more[id] = true + } + } + return more, "" +} + +// incrementalSummary is the progress line for a build that carried rows. +func incrementalSummary(s BuildStats) string { + return fmt.Sprintf("incremental: %d units rebuilt, %d carried over", s.Rebuilt, s.Carried) +} + +// unitKind names a unit's type for a progress line. +func (b *Builder) unitKind(id string) string { + if t := b.unitTypes[id]; t != "" { + return t + } + return "unit" +} + +// unitNames is the qualified names the given units' rows carry (an +// entity's, an association's, a microflow's). +func unitNames(q queryFunc, units map[string]bool) ([]string, error) { + rows, err := q(`SELECT UnitId, QualifiedName FROM catalog_unit_names`) + if err != nil { + return nil, err + } + defer rows.Close() + var out []string + for rows.Next() { + var id, qn string + if err := rows.Scan(&id, &qn); err != nil { + return nil, err + } + if units[id] { + out = append(out, qn) + } + } + return out, rows.Err() +} + +// dependentUnits is the units holding a document whose refs name one of +// names or a member of one (`M.E.Attr` under `M.E`). Specializations are +// followed first: a document naming `M.Sub.Attr` may resolve it on Sub's +// generalization. +func dependentUnits(q queryFunc, names []string) (map[string]bool, error) { + g, err := loadRefGraph(q) + if err != nil { + return nil, err + } + return g.dependents(names), nil +} + +// refGraph is the refs and unit names the plan follows, read once. +type refGraph struct { + refs []graphRef + unitOf map[string][]string // qualified name -> units whose rows carry it + namesOf map[string][]string // unit -> qualified names its rows carry +} + +type graphRef struct{ source, target, kind string } + +func loadRefGraph(q queryFunc) (*refGraph, error) { + g := &refGraph{unitOf: map[string][]string{}, namesOf: map[string][]string{}} + rows, err := q(`SELECT SourceName, TargetName, RefKind FROM refs`) + if err != nil { + return nil, err + } + for rows.Next() { + var r graphRef + if err := rows.Scan(&r.source, &r.target, &r.kind); err != nil { + rows.Close() + return nil, err + } + g.refs = append(g.refs, r) + } + rows.Close() + + owners, err := q(`SELECT UnitId, QualifiedName FROM catalog_unit_names`) + if err != nil { + return nil, err + } + defer owners.Close() + for owners.Next() { + var id, qn string + if err := owners.Scan(&id, &qn); err != nil { + return nil, err + } + g.unitOf[qn] = append(g.unitOf[qn], id) + g.namesOf[id] = append(g.namesOf[id], qn) + } + return g, owners.Err() +} + +// dependents is dependentUnits on a loaded graph. +func (g *refGraph) dependents(names []string) map[string]bool { + named := map[string]bool{} + for _, n := range names { + if n != "" { + named[n] = true + } + } + for grew := true; grew; { + grew = false + for _, r := range g.refs { + if r.kind == RefKindGeneralize && named[r.target] && !named[r.source] { + named[r.source] = true + grew = true + } + } + } + isNamed := func(target string) bool { + for { + if named[target] { + return true + } + i := strings.LastIndexByte(target, '.') + if i < 0 { + return false + } + target = target[:i] + } + } + sources := map[string]bool{} + for _, r := range g.refs { + if isNamed(r.target) { + sources[r.source] = true + } + } + for n := range named { + sources[n] = true // a specialization is rebuilt itself + } + out := map[string]bool{} + for qn := range sources { + for _, id := range g.unitOf[qn] { + out[id] = true + } + } + return out +} + +// clearCatalog empties every table of a catalog loaded from the cache, so a +// full build starts from what a fresh one would: no rows, no AUTOINCREMENT +// counters, and only the schema version in catalog_meta. +func clearCatalog(tx CatalogTx) error { + rows, err := tx.Query(`SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`) + if err != nil { + return err + } + type table struct { + name string + virtual bool + } + var tables []table + for rows.Next() { + var name string + var ddl sql.NullString + if err := rows.Scan(&name, &ddl); err != nil { + rows.Close() + return err + } + tables = append(tables, table{name, strings.HasPrefix(strings.ToUpper(ddl.String), "CREATE VIRTUAL TABLE")}) + } + rows.Close() + // A full-text table's shadow tables are emptied through the table. + shadow := map[string]bool{} + for _, t := range tables { + if t.virtual { + for _, suffix := range []string{"_data", "_idx", "_content", "_docsize", "_config"} { + shadow[t.name+suffix] = true + } + } + } + for _, t := range tables { + if shadow[t.name] { + continue + } + stmt := `DELETE FROM "` + t.name + `"` + if t.name == "catalog_meta" { + stmt += ` WHERE Key != '` + MetaSchemaVersion + `'` + } + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("empty %s: %w", t.name, err) + } + } + var seq int + if tx.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE name = 'sqlite_sequence'`).Scan(&seq) == nil && seq > 0 { + if _, err := tx.Exec(`DELETE FROM sqlite_sequence`); err != nil { + return err + } + } + return nil +} + +// dropRebuiltRows deletes, from the previous catalog the build starts from, +// the rows of the units in scope and the rows no unit wrote, and records the +// ranges of every row it keeps under its unit. It returns how many units' +// rows were kept. +func (b *Builder) dropRebuiltRows() (int, error) { + type rng struct { + table, owner string + first, last int64 + } + rows, err := b.txQuery(`SELECT TableName, UnitId, FirstRow, LastRow FROM catalog_row_owners ORDER BY TableName, FirstRow`) + if err != nil { + return 0, err + } + var all []rng + for rows.Next() { + var r rng + if err := rows.Scan(&r.table, &r.owner, &r.first, &r.last); err != nil { + rows.Close() + return 0, err + } + all = append(all, r) + } + rows.Close() + + kept := map[string]bool{} + for _, r := range all { + if r.owner != "" && !b.scope[r.owner] { + b.rows.add(r.table, r.owner, r.first, r.last) + kept[r.owner] = true + continue + } + if _, err := b.tx.CatalogTx.Exec(`DELETE FROM "`+r.table+`" WHERE rowid BETWEEN ? AND ?`, r.first, r.last); err != nil { + return 0, fmt.Errorf("drop rows of %s: %w", r.table, err) + } + } + // What records the build rather than the project is written again, and + // the graph analysis, which AddGraphAnalysis writes outside a build. + for _, stmt := range []string{ + `DELETE FROM communities_data`, + `DELETE FROM graph_cycles_data`, + `DELETE FROM graph_module_cycles_data`, + `DELETE FROM graph_layers_data`, + `DELETE FROM graph_centrality_data`, + `DELETE FROM projects`, + `DELETE FROM snapshots`, + `DELETE FROM catalog_row_owners`, + `DELETE FROM catalog_units`, + `DELETE FROM catalog_unit_names`, + `DELETE FROM catalog_meta WHERE Key IN ('` + MetaGraphAnalysis + `', '` + MetaGraphResolution + `', '` + MetaSourceFingerprint + `', '` + MetaCatalogFingerprint + `')`, + } { + if _, err := b.tx.CatalogTx.Exec(stmt); err != nil { + return 0, err + } + } + return len(kept), nil +} + +func (b *Builder) txQuery(query string, args ...any) (*sql.Rows, error) { + return b.tx.CatalogTx.Query(query, args...) +} + +// persistOwners records what the next build needs to carry from this one: +// the unit each row range came from, every unit's basis and whether a +// per-unit pass reads it, the names each unit's rows carry, and which mxcli +// build wrote them. +func (b *Builder) persistOwners() error { + if b.basis == nil || b.fingerprint == "" { + return nil + } + exec := b.tx.CatalogTx.Exec + ins, err := b.tx.CatalogTx.Prepare(`INSERT INTO catalog_row_owners (TableName, UnitId, FirstRow, LastRow) VALUES (?, ?, ?, ?)`) + if err != nil { + return err + } + defer ins.Close() + tables := make([]string, 0, len(b.rows.ranges)) + for t := range b.rows.ranges { + tables = append(tables, t) + } + sort.Strings(tables) + for _, t := range tables { + for _, r := range b.rows.ranges[t] { + // A project-wide range ("") is recorded too: the next build + // drops those rows and writes them again. + if _, err := ins.Exec(t, r.owner, r.first, r.last); err != nil { + return err + } + } + } + + units, err := b.tx.CatalogTx.Prepare(`INSERT INTO catalog_units (UnitId, Basis, Owned) VALUES (?, ?, ?)`) + if err != nil { + return err + } + defer units.Close() + for id, basis := range b.basis { + owned := 0 + // A unit outside the scope was not listed, so this build did not + // visit it; it is unchanged, and so is whether a pass reads it. + if b.rows.owned[id] || (b.scope != nil && !b.scope[id] && b.prevOwned[id]) { + owned = 1 + } + if _, err := units.Exec(id, basis, owned); err != nil { + return err + } + } + + // The names each unit's rows carry, from every table with a + // QualifiedName column. + for _, t := range tables { + var has int + if err := b.tx.CatalogTx.QueryRow(`SELECT COUNT(*) FROM pragma_table_info(?, 'main') WHERE name = 'QualifiedName'`, t).Scan(&has); err != nil { + return err + } + if has == 0 { + continue + } + if _, err := exec(`INSERT INTO catalog_unit_names (UnitId, QualifiedName) + SELECT DISTINCT o.UnitId, t.QualifiedName FROM catalog_row_owners o CROSS JOIN main.` + t + ` t + WHERE o.TableName = '` + t + `' AND o.UnitId != '' AND t.rowid BETWEEN o.FirstRow AND o.LastRow AND t.QualifiedName IS NOT NULL`); err != nil { + return err + } + } + _, err = exec(`INSERT OR REPLACE INTO catalog_meta (Key, Value) VALUES (?, ?)`, MetaCatalogFingerprint, b.fingerprint) + return err +} diff --git a/mdl/catalog/builder_incremental_test.go b/mdl/catalog/builder_incremental_test.go new file mode 100644 index 0000000000..4fd4a7ed77 --- /dev/null +++ b/mdl/catalog/builder_incremental_test.go @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "database/sql" + "reflect" + "sort" + "testing" +) + +// fakeResult is the sql.Result of an INSERT that wrote n rows ending at last. +type fakeResult struct{ last, n int64 } + +func (r fakeResult) LastInsertId() (int64, error) { return r.last, nil } +func (r fakeResult) RowsAffected() (int64, error) { return r.n, nil } + +func TestRowOwners_RecordsRangesPerOwner(t *testing.T) { + o := newRowOwners() + o.owner = "a" + o.record("refs", fakeResult{1, 1}) + o.record("refs", fakeResult{3, 2}) // an INSERT ... SELECT of rows 2-3 + o.owner = "b" + o.record("refs", fakeResult{4, 1}) + o.owner = "a" + o.record("refs", fakeResult{5, 1}) + o.record("refs", fakeResult{5, 0}) // ignored by INSERT OR IGNORE + o.record("catalog_units", fakeResult{9, 1}) // the build's bookkeeping + + want := []rowRange{{"a", 1, 3}, {"b", 4, 4}, {"a", 5, 5}} + if got := o.ranges["refs"]; !reflect.DeepEqual(got, want) { + t.Errorf("ranges = %+v, want %+v", got, want) + } + if _, ok := o.ranges["catalog_units"]; ok { + t.Error("the build's own bookkeeping was recorded as rows of a unit") + } + if o.err != nil { + t.Errorf("unexpected error: %v", o.err) + } +} + +// A row written while a pass skips a document belongs to no unit the build +// can name; the build must notice, or the next one carries a row nobody +// rebuilds. +func TestRowOwners_RowForASkippedDocumentIsAnError(t *testing.T) { + b := &Builder{scope: map[string]bool{"in": true}} + if !b.own("in") { + t.Fatal("a unit in scope was skipped") + } + if b.own("out") { + t.Fatal("a unit out of scope was not skipped") + } + b.rows.record("refs", fakeResult{1, 1}) + if b.rows.err == nil { + t.Fatal("a row written for a skipped document was not reported") + } + if !b.rows.owned["in"] || !b.rows.owned["out"] { + t.Error("own did not record the units the pass visited") + } +} + +func TestBuilderTx_InsertTable(t *testing.T) { + tx := newBuilderTx(nil, nil) + for q, want := range map[string]string{ + "INSERT INTO refs (A) VALUES (?)": "refs", + "\n\t\tinsert or ignore into Contract_X values": "contract_x", + "INSERT OR REPLACE INTO source_units VALUES": "source_units", + "UPDATE contract_entities_data SET X = 1": "", + "SELECT 1": "", + } { + if got := tx.insertTable(q); got != want { + t.Errorf("insertTable(%q) = %q, want %q", q, got, want) + } + } +} + +func TestDependentUnits(t *testing.T) { + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + for _, s := range []string{ + `CREATE TABLE refs (SourceName TEXT, TargetName TEXT, RefKind TEXT)`, + `CREATE TABLE catalog_unit_names (UnitId TEXT, QualifiedName TEXT)`, + // M.Sub specializes M.Base; Other.Sub2 specializes M.Sub. + `INSERT INTO refs VALUES ('M.Sub', 'M.Base', 'generalize')`, + `INSERT INTO refs VALUES ('Other.Sub2', 'M.Sub', 'generalize')`, + // A flow naming a member of the changed entity, one naming an + // inherited member, and one naming something else. + `INSERT INTO refs VALUES ('M.FlowA', 'M.Base.Name', 'member')`, + `INSERT INTO refs VALUES ('X.FlowB', 'Other.Sub2.Name', 'member')`, + `INSERT INTO refs VALUES ('X.FlowC', 'M.Basement', 'call')`, + `INSERT INTO catalog_unit_names VALUES ('dm-m', 'M.Base'), ('dm-m', 'M.Sub'), ('dm-other', 'Other.Sub2'), + ('flow-a', 'M.FlowA'), ('flow-b', 'X.FlowB'), ('flow-c', 'X.FlowC')`, + } { + if _, err := db.Exec(s); err != nil { + t.Fatal(err) + } + } + got, err := dependentUnits(db.Query, []string{"M.Base"}) + if err != nil { + t.Fatal(err) + } + var units []string + for u := range got { + units = append(units, u) + } + sort.Strings(units) + // M.Basement shares a prefix but is not a member: FlowC is not one. + want := []string{"dm-m", "dm-other", "flow-a", "flow-b"} + if !reflect.DeepEqual(units, want) { + t.Errorf("dependents = %v, want %v", units, want) + } +} diff --git a/mdl/catalog/builder_java_actions_test.go b/mdl/catalog/builder_java_actions_test.go index 4bd6e4eb04..a60f35dd7b 100644 --- a/mdl/catalog/builder_java_actions_test.go +++ b/mdl/catalog/builder_java_actions_test.go @@ -82,7 +82,7 @@ func TestJavaActionTypeParameterNamedAfterPrimitive(t *testing.T) { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } if err := b.buildJavaActions(); err != nil { t.Fatalf("buildJavaActions: %v", err) diff --git a/mdl/catalog/builder_layout_platform_test.go b/mdl/catalog/builder_layout_platform_test.go index cae00a57b6..680c1dd85e 100644 --- a/mdl/catalog/builder_layout_platform_test.go +++ b/mdl/catalog/builder_layout_platform_test.go @@ -49,7 +49,7 @@ func TestLayoutPlatformColumn_Issue953(t *testing.T) { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } if err := b.buildLayouts(); err != nil { t.Fatalf("buildLayouts: %v", err) diff --git a/mdl/catalog/builder_lint_columns_test.go b/mdl/catalog/builder_lint_columns_test.go index 7e4357b143..9df02fb22f 100644 --- a/mdl/catalog/builder_lint_columns_test.go +++ b/mdl/catalog/builder_lint_columns_test.go @@ -90,7 +90,7 @@ func lintColumnsBuilder(t *testing.T) (*Builder, *Catalog) { }, {ContainerID: empty}, }, - tx: tx, + tx: wrapTx(tx), } return b, cat } diff --git a/mdl/catalog/builder_listing_test.go b/mdl/catalog/builder_listing_test.go new file mode 100644 index 0000000000..2fd8e80816 --- /dev/null +++ b/mdl/catalog/builder_listing_test.go @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// readerListings are the passes allowed to list documents through b.reader, +// which holds every unit, rather than b.lister() or the cached accessors, +// which in an incremental build hold only the units it rebuilds. A listing +// through the reader decodes or reads the whole project on every refresh +// (ako/mxcli#1804), so each one here says why it has to. +var readerListings = map[string]string{ + "builder.go ListModules": "the hierarchy: every document's module name", + "builder.go ListUnits": "the hierarchy: every document's container (no contents)", + "builder.go ListFolders": "the hierarchy: every document's folder path", + "builder_incremental.go ListUnits": "every unit's basis, to find what changed (no contents)", + "builder_modules.go ListModules": "the module rows, rebuilt every time", + "builder_modules.go ListRawUnitsByType": "the domain model documentation of every module row, read raw, not decoded", + "builder_external.go ListRawUnitsByType": "finds the flows that call an external action by their raw bytes, then decodes only those", + "builder_source_reuse.go ListUnits": "every unit's source basis (no contents)", + "builder_source_reuse.go ListRawUnitsByType": "every entity's source basis, hashed from the raw domain models", +} + +var readerListing = regexp.MustCompile(`b\.reader\.(List\w+)\(`) + +func TestBuilderListsDocumentsThroughTheScope(t *testing.T) { + files, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + found := map[string]bool{} + var stray []string + for _, f := range files { + if strings.HasSuffix(f, "_test.go") { + continue + } + src, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, m := range readerListing.FindAllStringSubmatch(string(src), -1) { + key := f + " " + m[1] + found[key] = true + if _, ok := readerListings[key]; !ok { + stray = append(stray, key) + } + } + } + sort.Strings(stray) + if len(stray) > 0 { + t.Errorf("passes list documents through b.reader, which decodes the whole project on every incremental refresh:\n %s\n"+ + "List through b.lister() or a cached accessor, and look other documents up in the catalog tables; "+ + "if the pass really needs every document, add it to readerListings with the reason.", + strings.Join(stray, "\n ")) + } + if len(found) == 0 { + t.Fatal("control: no listing found at all; the pattern no longer matches the code") + } + for key := range readerListings { + if !found[key] { + t.Errorf("readerListings names %q, which no longer lists through the reader: remove it", key) + } + } +} diff --git a/mdl/catalog/builder_loop_var_refs_test.go b/mdl/catalog/builder_loop_var_refs_test.go index 266c7e562d..c4054972af 100644 --- a/mdl/catalog/builder_loop_var_refs_test.go +++ b/mdl/catalog/builder_loop_var_refs_test.go @@ -54,7 +54,7 @@ func TestBuildVarEntityMap_LoopIteratorAndAssociationRetrieve(t *testing.T) { // A loop over a list whose entity is unknown maps nothing. iterLoop("l3", "$Untyped", "X"), }} - got := buildVarEntityMap([]*microflows.MicroflowParameter{listParam("Orders", "T.Order")}, oc, ends) + got := buildVarEntityMap([]*microflows.MicroflowParameter{listParam("Orders", "T.Order")}, oc, ends, nil) want := map[string]string{ "Orders": "T.Order", "Order": "T.Order", "Cust": "T.Customer", "CustOrders": "T.Order", "Sibling": "T.Order", diff --git a/mdl/catalog/builder_mappings_excluded_test.go b/mdl/catalog/builder_mappings_excluded_test.go index 26b7019703..af3b1cb944 100644 --- a/mdl/catalog/builder_mappings_excluded_test.go +++ b/mdl/catalog/builder_mappings_excluded_test.go @@ -38,7 +38,7 @@ func twinBuilder(t *testing.T, reader *mock.MockBackend, describe DescribeFunc) containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), sourceMode: describe != nil, describeFunc: describe, } diff --git a/mdl/catalog/builder_member_refs.go b/mdl/catalog/builder_member_refs.go index fb2398af34..fa252ad7b1 100644 --- a/mdl/catalog/builder_member_refs.go +++ b/mdl/catalog/builder_member_refs.go @@ -74,6 +74,17 @@ type memberRefIndex struct { enumValues map[string]bool // "Mod.Enum.Value" entities map[string]bool generalization map[string]string // entity -> the entity it specializes + + // probe, when set, is told every name looked up, found or not: the + // document being walked depends on it (builder_coverage.go). + probe func(name string) +} + +// read reports a lookup of name to the probe. +func (idx *memberRefIndex) read(name string) { + if idx.probe != nil { + idx.probe(name) + } } // loadMemberRefIndex reads the name sets from the tables buildEntities, @@ -125,6 +136,7 @@ func (idx *memberRefIndex) resolveAttribute(entity, name string) (string, bool) seen := map[string]bool{} for e := entity; e != "" && !seen[e]; e = idx.generalization[e] { seen[e] = true + idx.read(e + "." + name) if _, ok := idx.attributes[e+"."+name]; ok { return e + "." + name, true } @@ -184,6 +196,7 @@ func memberRefsInUnit(contents []byte, sourceType string, idx *memberRefIndex) [ // A whole-string match is a structured reference: MemberChange.Attribute, // AttributeRef.Attribute, EntityRefStep.Association, EnumerationType.Enumeration, // ObjectMappingElement.Entity. + idx.read(v) switch { case hasKey(idx.attributes, v): set.add(RefObjectAttribute, v, RefKindMember) @@ -295,6 +308,9 @@ func scanPaths(text, context string, idx *memberRefIndex, emit func(targetType, lit.WriteByte(text[j]) j++ } + if pendingEnum != "" { + idx.read(pendingEnum + "." + lit.String()) + } if pendingEnum != "" && idx.enumValues[pendingEnum+"."+lit.String()] { emit(RefObjectEnumerationValue, pendingEnum+"."+lit.String()) } @@ -363,6 +379,7 @@ func resolvePath(chunk, context string, idx *memberRefIndex, emit func(targetTyp segs := strings.Split(chunk, "/") for n, seg := range segs { pendingEnum, endEntity = "", "" + idx.read(seg) switch { case seg == "": cur = "" @@ -396,7 +413,9 @@ func resolvePath(chunk, context string, idx *memberRefIndex, emit func(targetTyp // extractMemberRefs walks every document of the types in memberRefSourceTypes // and emits its member, enumeration and mapping-entity edges. -func (b *Builder) extractMemberRefs(stmt *sql.Stmt, idx *memberRefIndex, projectID, snapshotID string) int { +func (b *Builder) extractMemberRefs(stmt *Stmt, idx *memberRefIndex, projectID, snapshotID string) int { + idx.probe = b.probe(siteMemberRefs) + defer func() { idx.probe = nil }() unitTypes := make([]string, 0, len(memberRefSourceTypes)) for t := range memberRefSourceTypes { unitTypes = append(unitTypes, t) @@ -406,11 +425,14 @@ func (b *Builder) extractMemberRefs(stmt *sql.Stmt, idx *memberRefIndex, project count := 0 for _, unitType := range unitTypes { sourceType := memberRefSourceTypes[unitType] - units, err := b.reader.ListRawUnitsByType(unitType) + units, err := b.lister().ListRawUnitsByType(unitType) if err != nil { continue } for _, u := range units { + if !b.ownRows(string(u.ID)) { + continue + } // ListRawUnitsByType matches a PREFIX: Forms$Page also returns // Forms$PageTemplate units, which are not pages. if u.Type != unitType || len(u.Contents) == 0 { @@ -432,6 +454,7 @@ func (b *Builder) extractMemberRefs(stmt *sql.Stmt, idx *memberRefIndex, project } } } + b.disown() } return count } @@ -468,7 +491,7 @@ var xpathSourceTypes = map[string]string{ // enumeration values each recorded XPath constraint names, resolved against the // constraint's target entity. buildXPathExpressions runs before buildReferences // so the table is populated. -func (b *Builder) extractXPathRefs(stmt *sql.Stmt, idx *memberRefIndex, projectID, snapshotID string) int { +func (b *Builder) extractXPathRefs(stmt *Stmt, idx *memberRefIndex, projectID, snapshotID string) int { rows, err := b.tx.Query(`SELECT DocumentType, DocumentId, DocumentQualifiedName, COALESCE(TargetEntity, ''), XPathExpression, COALESCE(ModuleName, '') FROM xpath_expressions_data ORDER BY Id`) if err != nil { diff --git a/mdl/catalog/builder_member_refs_test.go b/mdl/catalog/builder_member_refs_test.go index aa06ffce9f..3f9cb5321a 100644 --- a/mdl/catalog/builder_member_refs_test.go +++ b/mdl/catalog/builder_member_refs_test.go @@ -235,7 +235,7 @@ func memberRefsFixture(t *testing.T) *Catalog { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), fullMode: true, } diff --git a/mdl/catalog/builder_microflows.go b/mdl/catalog/builder_microflows.go index ad376a02c3..1e19493d33 100644 --- a/mdl/catalog/builder_microflows.go +++ b/mdl/catalog/builder_microflows.go @@ -3,7 +3,6 @@ package catalog import ( - "database/sql" "fmt" "sort" "strings" @@ -71,7 +70,7 @@ func (b *Builder) buildMicroflows() error { defer paramStmt.Close() // Prepare activity statement only in full mode - var actStmt *sql.Stmt + var actStmt *Stmt if b.fullMode { actStmt, err = b.tx.Prepare(` INSERT INTO activities_data (Id, Name, Caption, ActivityType, Sequence, MicroflowId, MicroflowQualifiedName, @@ -137,6 +136,9 @@ func (b *Builder) buildMicroflows() error { // Process microflows for _, mf := range mfs { + if !b.own(string(mf.ID)) { + continue + } // Get module name moduleID := b.hierarchy.findModuleID(mf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -181,16 +183,20 @@ func (b *Builder) buildMicroflows() error { // Insert activities only in full mode if b.fullMode { n, err := insertFlowActivities(actStmt, string(mf.ID), qualifiedName, moduleName, mf.ObjectCollection, - buildVarEntityMap(mf.Parameters, mf.ObjectCollection, assocs), projectID, snapshotID) + buildVarEntityMap(mf.Parameters, mf.ObjectCollection, assocs, b.probe(siteAssocEnds)), projectID, snapshotID) if err != nil { return err } actCount += n } } + b.disown() // Process nanoflows for _, nf := range nfs { + if !b.own(string(nf.ID)) { + continue + } // Get module name moduleID := b.hierarchy.findModuleID(nf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -234,13 +240,14 @@ func (b *Builder) buildMicroflows() error { // Insert activities only in full mode if b.fullMode { n, err := insertFlowActivities(actStmt, string(nf.ID), qualifiedName, moduleName, nf.ObjectCollection, - buildVarEntityMap(nf.Parameters, nf.ObjectCollection, assocs), projectID, snapshotID) + buildVarEntityMap(nf.Parameters, nf.ObjectCollection, assocs, b.probe(siteAssocEnds)), projectID, snapshotID) if err != nil { return err } actCount += n } } + b.disown() // Process rules. Their bodies are walked for activities in full mode like // any other flow; the reference edges out of those bodies are emitted by @@ -248,6 +255,9 @@ func (b *Builder) buildMicroflows() error { // object but whose body is not walked reports every document it calls as // dead, which is worse than not knowing about rules at all. for _, rule := range rules { + if !b.own(string(rule.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(rule.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + rule.Name @@ -283,13 +293,14 @@ func (b *Builder) buildMicroflows() error { if b.fullMode { n, err := insertFlowActivities(actStmt, string(rule.ID), qualifiedName, moduleName, rule.ObjectCollection, - buildVarEntityMap(rule.Parameters, rule.ObjectCollection, assocs), projectID, snapshotID) + buildVarEntityMap(rule.Parameters, rule.ObjectCollection, assocs, b.probe(siteAssocEnds)), projectID, snapshotID) if err != nil { return err } actCount += n } } + b.disown() b.report("Microflows", mfCount) b.report("Nanoflows", nfCount) @@ -528,7 +539,7 @@ func calculateRuleComplexity(rule *microflows.Rule) int { // // varEntity is the flow's variable→entity map (buildVarEntityMap), which a // delete's EntityRef is resolved through. -func insertFlowActivities(stmt *sql.Stmt, flowID, qualifiedName, moduleName string, +func insertFlowActivities(stmt *Stmt, flowID, qualifiedName, moduleName string, oc *microflows.MicroflowObjectCollection, varEntity map[string]string, projectID, snapshotID string) (int, error) { if stmt == nil || oc == nil { return 0, nil diff --git a/mdl/catalog/builder_modules.go b/mdl/catalog/builder_modules.go index d301fd484d..78d7ef0b0f 100644 --- a/mdl/catalog/builder_modules.go +++ b/mdl/catalog/builder_modules.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + "go.mongodb.org/mongo-driver/v2/bson" + "github.com/mendixlabs/mxcli/sdk/domainmodel" "github.com/mendixlabs/mxcli/sdk/javaactions" ) @@ -19,11 +21,22 @@ func (b *Builder) buildModules() error { // The domain model's documentation, by module. A module has no // documentation of its own; its domain model does, and nothing read it. // A failed domain-model read costs the column, not the module list. + // The modules are rebuilt every time, and an incremental build decodes + // only the domain models it rebuilds: it reads every domain model's + // stored Documentation instead. dmDocs := map[string]string{} - if dms, err := b.cachedDomainModels(); err == nil { - for _, dm := range dms { - if dm.Documentation != "" { - dmDocs[string(b.hierarchy.findModuleID(dm.ContainerID))] = dm.Documentation + if b.scope == nil { + if dms, err := b.cachedDomainModels(); err == nil { + for _, dm := range dms { + if dm.Documentation != "" { + dmDocs[string(b.hierarchy.findModuleID(dm.ContainerID))] = dm.Documentation + } + } + } + } else if units, err := b.reader.ListRawUnitsByType("DomainModels$DomainModel"); err == nil { + for _, u := range units { + if doc, ok := bson.Raw(u.Contents).Lookup("Documentation").StringValueOK(); ok && doc != "" { + dmDocs[string(b.hierarchy.findModuleID(u.ContainerID))] = doc } } } @@ -109,6 +122,9 @@ func (b *Builder) buildEntities() error { entityCount := 0 attrCount := 0 for _, dm := range domainModels { + if !b.own(string(dm.ID)) { + continue + } // Get module name using hierarchy moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -253,6 +269,7 @@ func (b *Builder) buildEntities() error { } } } + b.disown() b.report("Entities", entityCount) b.report("Attributes", attrCount) @@ -301,6 +318,9 @@ func (b *Builder) buildEnumerations() error { lang := b.defaultLanguage() for _, enum := range enums { + if !b.own(string(enum.ID)) { + continue + } // Get module name using hierarchy moduleID := b.hierarchy.findModuleID(enum.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -361,6 +381,7 @@ func (b *Builder) buildEnumerations() error { valueCount++ } } + b.disown() b.report("Enumerations", len(enums)) b.report("Enumeration values", valueCount) @@ -396,7 +417,7 @@ func catalogCodeActionType(t interface{ TypeString() string }) string { } func (b *Builder) buildJavaActions() error { - actions, err := b.reader.ListJavaActionsFull() + actions, err := b.lister().ListJavaActionsFull() if err != nil { return err } @@ -427,6 +448,9 @@ func (b *Builder) buildJavaActions() error { paramCount := 0 for _, ja := range actions { + if !b.own(string(ja.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(ja.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + ja.Name @@ -479,6 +503,7 @@ func (b *Builder) buildJavaActions() error { paramCount++ } } + b.disown() b.report("Java Actions", len(actions)) if paramCount > 0 { @@ -488,7 +513,7 @@ func (b *Builder) buildJavaActions() error { } func (b *Builder) buildODataClients() error { - services, err := b.reader.ListConsumedODataServices() + services, err := b.lister().ListConsumedODataServices() if err != nil { return err } @@ -507,6 +532,9 @@ func (b *Builder) buildODataClients() error { projectID, snapshotID := b.snapshotMeta() for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -537,13 +565,14 @@ func (b *Builder) buildODataClients() error { return err } } + b.disown() b.report("OData Clients", len(services)) return nil } func (b *Builder) buildODataServices() error { - services, err := b.reader.ListPublishedODataServices() + services, err := b.lister().ListPublishedODataServices() if err != nil { return err } @@ -562,6 +591,9 @@ func (b *Builder) buildODataServices() error { projectID, snapshotID := b.snapshotMeta() for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -591,6 +623,7 @@ func (b *Builder) buildODataServices() error { return err } } + b.disown() b.report("OData Services", len(services)) return nil @@ -617,6 +650,9 @@ func (b *Builder) buildBusinessEventServices() error { projectID, snapshotID := b.snapshotMeta() for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + svc.Name @@ -668,6 +704,7 @@ func (b *Builder) buildBusinessEventServices() error { return err } } + b.disown() b.report("Business Event Services", len(services)) return nil @@ -696,6 +733,9 @@ func (b *Builder) buildBusinessEvents() error { count := 0 for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) svcQN := moduleName + "." + svc.Name @@ -768,6 +808,7 @@ func (b *Builder) buildBusinessEvents() error { } } } + b.disown() b.report("Business Events", count) return nil @@ -793,6 +834,9 @@ func (b *Builder) buildDatabaseConnections() error { projectID, snapshotID := b.snapshotMeta() for _, conn := range connections { + if !b.own(string(conn.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(conn.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + conn.Name @@ -812,6 +856,7 @@ func (b *Builder) buildDatabaseConnections() error { return err } } + b.disown() b.report("Database Connections", len(connections)) return nil @@ -837,6 +882,9 @@ func (b *Builder) buildJarDependencies() error { count := 0 for _, ms := range allSettings { + if !b.own(string(ms.ID)) { + continue + } moduleName := b.hierarchy.getModuleName(ms.ContainerID) for _, dep := range ms.JarDependencies { coordinate := dep.GroupID + ":" + dep.ArtifactID @@ -860,13 +908,14 @@ func (b *Builder) buildJarDependencies() error { count++ } } + b.disown() b.report("JAR Dependencies", count) return nil } func (b *Builder) buildJsonStructures() error { - structures, err := b.reader.ListJsonStructures() + structures, err := b.lister().ListJsonStructures() if err != nil { return err } @@ -885,6 +934,9 @@ func (b *Builder) buildJsonStructures() error { projectID, snapshotID := b.snapshotMeta() for _, js := range structures { + if !b.own(string(js.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(js.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + js.Name @@ -915,13 +967,14 @@ func (b *Builder) buildJsonStructures() error { return err } } + b.disown() b.report("JSON Structures", len(structures)) return nil } func (b *Builder) buildImportMappings() error { - mappings, err := b.reader.ListImportMappings() + mappings, err := b.lister().ListImportMappings() if err != nil { return err } @@ -940,6 +993,9 @@ func (b *Builder) buildImportMappings() error { projectID, snapshotID := b.snapshotMeta() for _, im := range mappings { + if !b.own(string(im.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(im.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + im.Name @@ -969,13 +1025,14 @@ func (b *Builder) buildImportMappings() error { return err } } + b.disown() b.report("Import Mappings", len(mappings)) return nil } func (b *Builder) buildExportMappings() error { - mappings, err := b.reader.ListExportMappings() + mappings, err := b.lister().ListExportMappings() if err != nil { return err } @@ -994,6 +1051,9 @@ func (b *Builder) buildExportMappings() error { projectID, snapshotID := b.snapshotMeta() for _, em := range mappings { + if !b.own(string(em.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(em.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + em.Name @@ -1024,6 +1084,7 @@ func (b *Builder) buildExportMappings() error { return err } } + b.disown() b.report("Export Mappings", len(mappings)) return nil diff --git a/mdl/catalog/builder_navigation.go b/mdl/catalog/builder_navigation.go index 2b50700288..dbdfb93a13 100644 --- a/mdl/catalog/builder_navigation.go +++ b/mdl/catalog/builder_navigation.go @@ -3,7 +3,6 @@ package catalog import ( - "database/sql" "fmt" "strings" @@ -170,7 +169,7 @@ func countMenuItems(items []*types.NavMenuItem) int { } // insertMenuItems recursively inserts menu items with hierarchical path encoding. -func insertMenuItems(stmt *sql.Stmt, profileName string, items []*types.NavMenuItem, parentPath string, depth int, projectID, snapshotID string) int { +func insertMenuItems(stmt *Stmt, profileName string, items []*types.NavMenuItem, parentPath string, depth int, projectID, snapshotID string) int { count := 0 for i, item := range items { itemPath := fmt.Sprintf("%d", i) diff --git a/mdl/catalog/builder_page_title_test.go b/mdl/catalog/builder_page_title_test.go index 6efd5d6dae..c2312f55d9 100644 --- a/mdl/catalog/builder_page_title_test.go +++ b/mdl/catalog/builder_page_title_test.go @@ -73,7 +73,7 @@ func TestPageTitleIsDefaultLanguage_Issue1262(t *testing.T) { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } if err := b.buildPages(); err != nil { t.Fatalf("buildPages: %v", err) diff --git a/mdl/catalog/builder_pages.go b/mdl/catalog/builder_pages.go index 4da4213677..29cc7cab26 100644 --- a/mdl/catalog/builder_pages.go +++ b/mdl/catalog/builder_pages.go @@ -3,7 +3,6 @@ package catalog import ( - "database/sql" "encoding/base64" "fmt" "go.mongodb.org/mongo-driver/v2/bson" @@ -33,7 +32,7 @@ func (b *Builder) buildPages() error { defer pageStmt.Close() // Prepare widget statement only in full mode - var widgetStmt *sql.Stmt + var widgetStmt *Stmt if b.fullMode { widgetStmt, err = b.tx.Prepare(` INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, @@ -54,6 +53,9 @@ func (b *Builder) buildPages() error { widgetCount := 0 for _, pg := range pageList { + if !b.own(string(pg.ID)) { + continue + } // Get module name moduleID := b.hierarchy.findModuleID(pg.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -131,6 +133,7 @@ func (b *Builder) buildPages() error { } } } + b.disown() b.report("Pages", pageCount) if b.fullMode { @@ -141,7 +144,7 @@ func (b *Builder) buildPages() error { func (b *Builder) buildSnippets() error { // Get all snippets - snippetList, err := b.reader.ListSnippets() + snippetList, err := b.lister().ListSnippets() if err != nil { return err } @@ -160,7 +163,7 @@ func (b *Builder) buildSnippets() error { // Snippet widgets feed the same widgets_data table as page widgets (full mode // only), with ContainerType=SNIPPET, so the refs projection in buildReferences // emits snippet → entity/microflow/nanoflow edges automatically. - var widgetStmt *sql.Stmt + var widgetStmt *Stmt if b.fullMode { widgetStmt, err = b.tx.Prepare(` INSERT INTO widgets_data (Id, Name, WidgetType, ContainerId, ContainerQualifiedName, ContainerType, @@ -180,6 +183,9 @@ func (b *Builder) buildSnippets() error { count := 0 widgetCount := 0 for _, sn := range snippetList { + if !b.own(string(sn.ID)) { + continue + } // Get module name moduleID := b.hierarchy.findModuleID(sn.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -226,6 +232,7 @@ func (b *Builder) buildSnippets() error { } } } + b.disown() b.report("Snippets", count) if b.fullMode { @@ -241,7 +248,7 @@ func (b *Builder) buildSnippets() error { // the raw unit's top-level Widgets array (same shape as a snippet), so it is // only available in full mode. func (b *Builder) buildBuildingBlocks() error { - blocks, err := b.reader.ListBuildingBlocks() + blocks, err := b.lister().ListBuildingBlocks() if err != nil { return err } @@ -261,6 +268,9 @@ func (b *Builder) buildBuildingBlocks() error { count := 0 for _, bb := range blocks { + if !b.own(string(bb.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(bb.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + bb.Name @@ -290,6 +300,7 @@ func (b *Builder) buildBuildingBlocks() error { } count++ } + b.disown() b.report("Building blocks", count) return nil @@ -914,7 +925,7 @@ const ( func (b *Builder) buildLayouts() error { // Get all layouts - layoutList, err := b.reader.ListLayouts() + layoutList, err := b.lister().ListLayouts() if err != nil { return err } @@ -933,6 +944,9 @@ func (b *Builder) buildLayouts() error { count := 0 for _, l := range layoutList { + if !b.own(string(l.ID)) { + continue + } // Get module name moduleID := b.hierarchy.findModuleID(l.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -955,6 +969,7 @@ func (b *Builder) buildLayouts() error { } count++ } + b.disown() b.report("Layouts", count) return nil @@ -963,6 +978,7 @@ func (b *Builder) buildLayouts() error { // widgetExtraRef is a page/snippet → document edge a widget makes that its // single widgets_data column does not carry (see rawWidgetInfo.AllRefs). type widgetExtraRef struct { + owner string containerType, containerQN, moduleName string targetType, targetName, refKind string } @@ -977,7 +993,7 @@ func (b *Builder) collectWidgetExtraRefs(containerType, containerQN, moduleName continue } b.widgetExtraRefs = append(b.widgetExtraRefs, widgetExtraRef{ - containerType, containerQN, moduleName, targetType, n, refKind, + b.currentOwner(), containerType, containerQN, moduleName, targetType, n, refKind, }) } } diff --git a/mdl/catalog/builder_permissions.go b/mdl/catalog/builder_permissions.go index 6e0dc8d90b..fcf056c7d0 100644 --- a/mdl/catalog/builder_permissions.go +++ b/mdl/catalog/builder_permissions.go @@ -3,8 +3,6 @@ package catalog import ( - "database/sql" - "github.com/mendixlabs/mxcli/sdk/domainmodel" ) @@ -86,15 +84,18 @@ func (b *Builder) buildPermissions() error { } // buildEntityPermissions extracts entity-level and member-level access permissions. -func (b *Builder) buildEntityPermissions(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) buildEntityPermissions(stmt *Stmt, projectID, snapshotID string) int { count := 0 - dms, err := b.reader.ListDomainModels() + dms, err := b.cachedDomainModels() if err != nil { return 0 } for _, dm := range dms { + if !b.own(string(dm.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -146,6 +147,7 @@ func (b *Builder) buildEntityPermissions(stmt *sql.Stmt, projectID, snapshotID s } } } + b.disown() return count } @@ -180,7 +182,7 @@ func entityAccessFromMemberRights(rule *domainmodel.AccessRule) (hasRead, hasWri // emitMemberPermissions emits MEMBER_READ/MEMBER_WRITE rows for entity attributes and associations. // When MemberAccesses is non-empty, use explicit per-member rights. // When MemberAccesses is empty, expand DefaultMemberAccessRights to all attributes. -func (b *Builder) emitMemberPermissions(stmt *sql.Stmt, rule *domainmodel.AccessRule, ent *domainmodel.Entity, +func (b *Builder) emitMemberPermissions(stmt *Stmt, rule *domainmodel.AccessRule, ent *domainmodel.Entity, roleName, entityQN, xpath, defaultRights, moduleName, projectID, snapshotID string) int { count := 0 @@ -222,15 +224,18 @@ func (b *Builder) emitMemberPermissions(stmt *sql.Stmt, rule *domainmodel.Access } // buildMicroflowPermissions extracts microflow execution permissions. -func (b *Builder) buildMicroflowPermissions(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) buildMicroflowPermissions(stmt *Stmt, projectID, snapshotID string) int { count := 0 - mfs, err := b.reader.ListMicroflows() + mfs, err := b.cachedMicroflows() if err != nil { return 0 } for _, mf := range mfs { + if !b.own(string(mf.ID)) { + continue + } if len(mf.AllowedModuleRoles) == 0 { continue } @@ -246,6 +251,7 @@ func (b *Builder) buildMicroflowPermissions(stmt *sql.Stmt, projectID, snapshotI count++ } } + b.disown() return count } @@ -253,7 +259,7 @@ func (b *Builder) buildMicroflowPermissions(stmt *sql.Stmt, projectID, snapshotI // buildNanoflowPermissions extracts nanoflow execute permissions. A nanoflow // carries AllowedModuleRoles exactly like a microflow, and a rule checking // exposed flows misses every one of them without these rows (ako/mxcli#1434). -func (b *Builder) buildNanoflowPermissions(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) buildNanoflowPermissions(stmt *Stmt, projectID, snapshotID string) int { count := 0 nfs, err := b.cachedNanoflows() @@ -262,6 +268,9 @@ func (b *Builder) buildNanoflowPermissions(stmt *sql.Stmt, projectID, snapshotID } for _, nf := range nfs { + if !b.own(string(nf.ID)) { + continue + } if len(nf.AllowedModuleRoles) == 0 { continue } @@ -277,20 +286,24 @@ func (b *Builder) buildNanoflowPermissions(stmt *sql.Stmt, projectID, snapshotID count++ } } + b.disown() return count } // buildPagePermissions extracts page view permissions. -func (b *Builder) buildPagePermissions(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) buildPagePermissions(stmt *Stmt, projectID, snapshotID string) int { count := 0 - pages, err := b.reader.ListPages() + pages, err := b.cachedPages() if err != nil { return 0 } for _, pg := range pages { + if !b.own(string(pg.ID)) { + continue + } if len(pg.AllowedRoles) == 0 { continue } @@ -306,20 +319,24 @@ func (b *Builder) buildPagePermissions(stmt *sql.Stmt, projectID, snapshotID str count++ } } + b.disown() return count } // buildODataServicePermissions extracts published OData service access permissions. -func (b *Builder) buildODataServicePermissions(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) buildODataServicePermissions(stmt *Stmt, projectID, snapshotID string) int { count := 0 - services, err := b.reader.ListPublishedODataServices() + services, err := b.lister().ListPublishedODataServices() if err != nil { return 0 } for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } if len(svc.AllowedModuleRoles) == 0 { continue } @@ -333,6 +350,7 @@ func (b *Builder) buildODataServicePermissions(stmt *sql.Stmt, projectID, snapsh count++ } } + b.disown() return count } diff --git a/mdl/catalog/builder_references.go b/mdl/catalog/builder_references.go index b0e66da663..bd144f2e83 100644 --- a/mdl/catalog/builder_references.go +++ b/mdl/catalog/builder_references.go @@ -3,7 +3,6 @@ package catalog import ( - "database/sql" "strings" "github.com/mendixlabs/mxcli/mdl/types" @@ -44,6 +43,7 @@ const ( RefKindType = "type" // An attribute, parameter or variable is typed as an enumeration RefKindValue = "value" // An expression names an enumeration value RefKindMapping = "mapping" // An import or export mapping maps an entity + RefKindSplit = "split" // A decision branches on an object being a specialization ) // RefKinds is every value that reaches refs.RefKind. It is the vocabulary a @@ -60,7 +60,7 @@ var RefKinds = []string{ RefKindCalculate, RefKindReturn, RefKindSchedule, RefKindValidate, RefKindWidget, RefKindSettings, RefKindSync, RefKindPublish, RefKindEvent, RefKindMember, RefKindXPath, RefKindType, - RefKindValue, RefKindMapping, + RefKindValue, RefKindMapping, RefKindSplit, } // Object types recorded in refs.SourceType and refs.TargetType — the catalog's @@ -195,6 +195,37 @@ func collectRuleCalls(oc *microflows.MicroflowObjectCollection) []string { return out } +// collectInheritanceCases returns the entity of every branch of an +// inheritance split (`split type $x when Module.Entity`), once each, +// recursing into loops. The split is no activity and names the entity only +// on its outgoing flows, so without this a flow that branches on a +// specialization had no edge to it: `show references` missed the flow, and +// the incremental catalog build could not tell it depends on the entity's +// module (found by the ref coverage trace, ako/mxcli#1804). +func collectInheritanceCases(oc *microflows.MicroflowObjectCollection) []string { + if oc == nil { + return nil + } + var out []string + seen := map[string]bool{} + var walk func(*microflows.MicroflowObjectCollection) + walk = func(c *microflows.MicroflowObjectCollection) { + for _, f := range c.Flows { + if ic, ok := f.CaseValue.(*microflows.InheritanceCase); ok && ic.EntityQualifiedName != "" && !seen[ic.EntityQualifiedName] { + seen[ic.EntityQualifiedName] = true + out = append(out, ic.EntityQualifiedName) + } + } + for _, obj := range c.Objects { + if l, ok := obj.(*microflows.LoopedActivity); ok && l.ObjectCollection != nil { + walk(l.ObjectCollection) + } + } + } + walk(oc) + return out +} + // microflowActionRef returns the cross-reference a microflow action makes to // another document: the target's catalog object type, its qualified name, and // the RefKind label. ok is false when the action references no resolvable @@ -347,7 +378,7 @@ type assocEnds struct{ From, To string } // A variable can be defined after the object that uses it in collection order // (Objects is not flow order), so the walk repeats until nothing new is mapped; // each pass only adds entries, so it ends within one pass per variable. -func buildVarEntityMap(params []*microflows.MicroflowParameter, oc *microflows.MicroflowObjectCollection, assocs map[string]assocEnds) map[string]string { +func buildVarEntityMap(params []*microflows.MicroflowParameter, oc *microflows.MicroflowObjectCollection, assocs map[string]assocEnds, probe func(string)) map[string]string { norm := func(v string) string { return strings.TrimPrefix(v, "$") } varEntity := map[string]string{} for _, p := range params { @@ -380,6 +411,9 @@ func buildVarEntityMap(params []*microflows.MicroflowParameter, oc *microflows.M case *microflows.DatabaseRetrieveSource: added = set(a.OutputVariable, src.EntityQualifiedName) || added case *microflows.AssociationRetrieveSource: + if probe != nil { + probe(src.AssociationQualifiedName) + } added = set(a.OutputVariable, associationTarget(assocs[src.AssociationQualifiedName], varEntity[norm(src.StartVariable)])) || added } } @@ -413,20 +447,45 @@ func associationTarget(ends assocEnds, start string) string { } // associationEnds maps every association's qualified name to its endpoint -// entities, for resolving an association retrieve's output variable. +// entities, for resolving an association retrieve's output variable. An +// incremental build reads the associations buildAssociations wrote, which +// hold every module's while the domain models listed are only the rebuilt +// ones. There, an end that did not resolve to an entity is stored as its ID, which +// has no dot, and is "" here; a cross-module association's TO end is stored +// as the qualified name the model holds. func (b *Builder) associationEnds() map[string]assocEnds { out := map[string]assocEnds{} - dms, err := b.cachedDomainModels() + if b.scope == nil { + dms, err := b.cachedDomainModels() + if err != nil { + return out + } + for _, dm := range dms { + moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(dm.ContainerID)) + for _, a := range dm.Associations { + out[moduleName+"."+a.Name] = assocEnds{From: b.resolveEntityID(a.ParentID), To: b.resolveEntityID(a.ChildID)} + } + for _, ca := range dm.CrossAssociations { + out[moduleName+"."+ca.Name] = assocEnds{From: b.resolveEntityID(ca.ParentID), To: ca.ChildRef} + } + } + return out + } + rows, err := b.tx.Query(`SELECT QualifiedName, FromEntity, ToEntity FROM associations_data`) if err != nil { return out } - for _, dm := range dms { - moduleName := b.hierarchy.getModuleName(b.hierarchy.findModuleID(dm.ContainerID)) - for _, a := range dm.Associations { - out[moduleName+"."+a.Name] = assocEnds{From: b.resolveEntityID(a.ParentID), To: b.resolveEntityID(a.ChildID)} + defer rows.Close() + end := func(name string) string { + if !strings.Contains(name, ".") { + return "" } - for _, ca := range dm.CrossAssociations { - out[moduleName+"."+ca.Name] = assocEnds{From: b.resolveEntityID(ca.ParentID), To: ca.ChildRef} + return name + } + for rows.Next() { + var qn, from, to string + if rows.Scan(&qn, &from, &to) == nil { + out[qn] = assocEnds{From: end(from), To: end(to)} } } return out @@ -484,10 +543,13 @@ func (b *Builder) buildReferences() error { acts := collectActionActivities(oc) // Intra-flow variable→entity map so change/delete (which operate on a // variable, not a named entity) can resolve their target. - varEntity := buildVarEntityMap(params, oc, assocs) + varEntity := buildVarEntityMap(params, oc, assocs, b.probe(siteAssocEnds)) for _, rule := range collectRuleCalls(oc) { emit(RefObjectRule, rule, RefKindCall) } + for _, entity := range collectInheritanceCases(oc) { + emit(RefObjectEntity, entity, RefKindSplit) + } for _, act := range acts { if tt, tn, rk, ok := microflowActionRef(act.Action); ok { emit(tt, tn, rk) @@ -507,16 +569,24 @@ func (b *Builder) buildReferences() error { return err } for _, mf := range mfs { + if !b.own(string(mf.ID)) { + continue + } emitActionRefs(RefObjectMicroflow, string(mf.ID), mf.ContainerID, mf.Name, mf.Parameters, mf.ReturnType, mf.ObjectCollection) } + b.disown() // Extract nanoflow references — nanoflows also call microflows/nanoflows, // show pages, retrieve, etc. nfs, err := b.cachedNanoflows() if err == nil { for _, nf := range nfs { + if !b.own(string(nf.ID)) { + continue + } emitActionRefs(RefObjectNanoflow, string(nf.ID), nf.ContainerID, nf.Name, nf.Parameters, nf.ReturnType, nf.ObjectCollection) } + b.disown() } // Extract rule references. A rule's body calls microflows, retrieves and @@ -527,14 +597,21 @@ func (b *Builder) buildReferences() error { rules, err := b.cachedRules() if err == nil { for _, rule := range rules { + if !b.own(string(rule.ID)) { + continue + } emitActionRefs(RefObjectRule, string(rule.ID), rule.ContainerID, rule.Name, rule.Parameters, rule.ReturnType, rule.ObjectCollection) } + b.disown() } // Extract entity references (generalization) — using cached list dms, err := b.cachedDomainModels() if err == nil { for _, dm := range dms { + if !b.own(string(dm.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -595,12 +672,16 @@ func (b *Builder) buildReferences() error { } } } + b.disown() } // Extract page references (layout, datasources, parameters) — using cached list pageList, err := b.cachedPages() if err == nil { for _, pg := range pageList { + if !b.own(string(pg.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(pg.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) sourceQN := moduleName + "." + pg.Name @@ -632,6 +713,7 @@ func (b *Builder) buildReferences() error { } } } + b.disown() // Widget references: page → entity (datasource) and page → microflow / // nanoflow (action). Projected from the widgets table, which buildPages @@ -667,6 +749,7 @@ func (b *Builder) buildReferences() error { // keeps the projection's one-edge-per-page shape when another widget's // column already made the edge. for _, r := range b.widgetExtraRefs { + b.as(r.owner) res, xerr := b.tx.Exec( `INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetId, TargetName, RefKind, ModuleName, ProjectId, SnapshotId) SELECT ?, '', ?, ?, '', ?, ?, ?, ?, ? @@ -679,6 +762,7 @@ func (b *Builder) buildReferences() error { } } } + b.disown() // Page/snippet -> widget definition. buildWidgetDefinitions runs before // this pass, so the join target is populated. @@ -776,6 +860,9 @@ func (b *Builder) buildReferences() error { wfs, wfErr := b.cachedWorkflows() if wfErr == nil { for _, wf := range wfs { + if !b.own(string(wf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(wf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) sourceQN := moduleName + "." + wf.Name @@ -790,6 +877,7 @@ func (b *Builder) buildReferences() error { } } } + b.disown() } // Scheduled events run a microflow. Without this edge the microflow looks @@ -852,9 +940,11 @@ func (b *Builder) buildReferences() error { // same transaction. // extractRegexRuleRefs emits one `validate` edge per attribute validation rule // that uses a regular expression, from the entity to the regex document. -func (b *Builder) extractRegexRuleRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractRegexRuleRefs(stmt *Stmt, projectID, snapshotID string) int { + defer b.disown() count := 0 for _, r := range b.regexRuleRefs { + b.as(r.owner) if _, err := stmt.Exec( RefObjectEntity, "", r.entityQualifiedName, RefObjectRegularExpression, "", r.regexQualifiedName, @@ -866,9 +956,11 @@ func (b *Builder) extractRegexRuleRefs(stmt *sql.Stmt, projectID, snapshotID str return count } -func (b *Builder) extractScheduledEventRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractScheduledEventRefs(stmt *Stmt, projectID, snapshotID string) int { + defer b.disown() count := 0 for _, r := range b.scheduledEventRefs { + b.as(r.owner) if _, err := stmt.Exec( RefObjectScheduledEvent, "", r.qualifiedName, RefObjectMicroflow, "", r.microflow, @@ -891,9 +983,11 @@ func (b *Builder) extractScheduledEventRefs(stmt *sql.Stmt, projectID, snapshotI // // The edges are collected by buildPublishedRestServices, which runs earlier in // the same transaction. -func (b *Builder) extractPublishedRestRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractPublishedRestRefs(stmt *Stmt, projectID, snapshotID string) int { + defer b.disown() count := 0 for _, r := range b.publishedRestRefs { + b.as(r.owner) if _, err := stmt.Exec( RefObjectPublishedRestOperation, r.sourceID, r.qualifiedName, RefObjectMicroflow, "", r.microflow, @@ -917,9 +1011,11 @@ func (b *Builder) extractPublishedRestRefs(stmt *sql.Stmt, projectID, snapshotID // // The edges are collected by buildEntityEventHandlers, which runs earlier in // the same transaction. -func (b *Builder) extractEventHandlerRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractEventHandlerRefs(stmt *Stmt, projectID, snapshotID string) int { + defer b.disown() count := 0 for _, r := range b.eventHandlerRefs { + b.as(r.owner) if _, err := stmt.Exec( RefObjectEntity, "", r.entityQualifiedName, RefObjectMicroflow, "", r.microflow, @@ -932,7 +1028,7 @@ func (b *Builder) extractEventHandlerRefs(stmt *sql.Stmt, projectID, snapshotID } // extractMenuItemRefs extracts page and microflow references from menu items recursively. -func (b *Builder) extractMenuItemRefs(stmt *sql.Stmt, items []*types.NavMenuItem, sourceName, projectID, snapshotID string) int { +func (b *Builder) extractMenuItemRefs(stmt *Stmt, items []*types.NavMenuItem, sourceName, projectID, snapshotID string) int { refCount := 0 for _, item := range items { if item.Page != "" { @@ -959,7 +1055,7 @@ func (b *Builder) extractMenuItemRefs(stmt *sql.Stmt, items []*types.NavMenuItem } // extractWidgetRefs extracts entity and microflow references from a widget and its children. -func (b *Builder) extractWidgetRefs(stmt *sql.Stmt, w pages.Widget, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { +func (b *Builder) extractWidgetRefs(stmt *Stmt, w pages.Widget, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { if w == nil { return 0 } @@ -1033,7 +1129,7 @@ func (b *Builder) extractWidgetRefs(stmt *sql.Stmt, w pages.Widget, sourceType, } // extractWidgetObjectRefs extracts refs from a pluggable widget's WidgetObject. -func (b *Builder) extractWidgetObjectRefs(stmt *sql.Stmt, obj *pages.WidgetObject, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { +func (b *Builder) extractWidgetObjectRefs(stmt *Stmt, obj *pages.WidgetObject, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { if obj == nil { return 0 } @@ -1098,7 +1194,7 @@ func (b *Builder) extractWidgetObjectRefs(stmt *sql.Stmt, obj *pages.WidgetObjec } // extractDataSourceRefs extracts entity and microflow references from a datasource. -func (b *Builder) extractDataSourceRefs(stmt *sql.Stmt, ds pages.DataSource, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { +func (b *Builder) extractDataSourceRefs(stmt *Stmt, ds pages.DataSource, sourceType, sourceID, sourceQN, moduleName, projectID, snapshotID string) int { if ds == nil { return 0 } @@ -1196,6 +1292,7 @@ func (b *Builder) resolveEntityID(entityID model.ID) string { if err != nil { return "" } + b.dependsOn(siteEntityByID, qualifiedName) return qualifiedName } @@ -1223,6 +1320,7 @@ func (b *Builder) resolveMicroflowID(mfID model.ID) string { if err != nil { return "" } + b.dependsOn(siteFlowByID, qualifiedName) return qualifiedName } @@ -1248,7 +1346,7 @@ var projectSettingsMicroflowRefs = []struct { // The source is the SETTING, not the project, so `show references to ` // names which setting depends on it — which is the thing you need before dropping // it, and what `describe settings` would otherwise be the only way to learn. -func (b *Builder) extractProjectSettingsRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractProjectSettingsRefs(stmt *Stmt, projectID, snapshotID string) int { ps, err := b.reader.GetProjectSettings() if err != nil || ps == nil || ps.Model == nil { return 0 // a project may have no settings document; not an error diff --git a/mdl/catalog/builder_references_test.go b/mdl/catalog/builder_references_test.go index b0dd58f66a..7c2a02cbea 100644 --- a/mdl/catalog/builder_references_test.go +++ b/mdl/catalog/builder_references_test.go @@ -3,6 +3,7 @@ package catalog import ( + "reflect" "testing" "github.com/mendixlabs/mxcli/model" @@ -129,7 +130,7 @@ func TestBuildVarEntityMap_And_VarActionRef(t *testing.T) { Source: µflows.DatabaseRetrieveSource{EntityQualifiedName: "M.Product"}, }), } - varEntity := buildVarEntityMap(params, µflows.MicroflowObjectCollection{Objects: []microflows.MicroflowObject{acts[0], acts[1]}}, nil) + varEntity := buildVarEntityMap(params, µflows.MicroflowObjectCollection{Objects: []microflows.MicroflowObject{acts[0], acts[1]}}, nil, nil) wantMap := map[string]string{"In": "M.Customer", "New": "M.Order", "List": "M.Product"} for k, v := range wantMap { if varEntity[k] != v { @@ -263,3 +264,22 @@ func TestMicroflowActionRef(t *testing.T) { }) } } + +// An inheritance split names its entities only on its outgoing flows; each +// is one edge, inside loops too (ako/mxcli#1804). +func TestCollectInheritanceCases(t *testing.T) { + flow := func(entity string) *microflows.SequenceFlow { + return µflows.SequenceFlow{CaseValue: µflows.InheritanceCase{EntityQualifiedName: entity}} + } + oc := µflows.MicroflowObjectCollection{ + Flows: []*microflows.SequenceFlow{flow("M.Dog"), flow("M.Cat"), flow("M.Dog"), {CaseValue: µflows.NoCase{}}}, + Objects: []microflows.MicroflowObject{µflows.LoopedActivity{ + ObjectCollection: µflows.MicroflowObjectCollection{Flows: []*microflows.SequenceFlow{flow("M.Bird")}}, + }}, + } + got := collectInheritanceCases(oc) + want := []string{"M.Dog", "M.Cat", "M.Bird"} + if !reflect.DeepEqual(got, want) { + t.Errorf("collectInheritanceCases = %v, want %v", got, want) + } +} diff --git a/mdl/catalog/builder_regularexpressions.go b/mdl/catalog/builder_regularexpressions.go index 8d2614860b..324680388b 100644 --- a/mdl/catalog/builder_regularexpressions.go +++ b/mdl/catalog/builder_regularexpressions.go @@ -16,7 +16,7 @@ import ( // from what is stored — notably the pattern's BSON key, which modelsdk/gen gets // wrong (see mdl/regularexpressions). func (b *Builder) buildRegularExpressions() error { - units, err := b.reader.ListRawUnitsByType(regex.TypeName) + units, err := b.lister().ListRawUnitsByType(regex.TypeName) if err != nil { return err } @@ -36,6 +36,9 @@ func (b *Builder) buildRegularExpressions() error { count := 0 for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc bson.M if err := bsoncompat.UnmarshalDocM(u.Contents, &doc); err != nil { continue @@ -56,6 +59,7 @@ func (b *Builder) buildRegularExpressions() error { } count++ } + b.disown() b.report("Regular Expressions", count) return b.collectRegexRuleRefs() @@ -63,7 +67,7 @@ func (b *Builder) buildRegularExpressions() error { // regexRuleRef is one entity → regular-expression edge, held until // buildReferences. -type regexRuleRef struct{ entityQualifiedName, moduleName, regexQualifiedName string } +type regexRuleRef struct{ owner, entityQualifiedName, moduleName, regexQualifiedName string } // collectRegexRuleRefs walks stored domain models for attribute validation // rules of type DomainModels$RegExRuleInfo and records the regex each one uses. @@ -78,11 +82,14 @@ type regexRuleRef struct{ entityQualifiedName, moduleName, regexQualifiedName st // restricted to ENTITY/MICROFLOW/NANOFLOW/PAGE/SNIPPET, so a regex was never // reported dead in the first place. func (b *Builder) collectRegexRuleRefs() error { - units, err := b.reader.ListRawUnitsByType("DomainModels$DomainModel") + units, err := b.lister().ListRawUnitsByType("DomainModels$DomainModel") if err != nil { return nil // domain models are catalogued elsewhere; a read failure is reported there } for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc bson.M if bsoncompat.UnmarshalDocM(u.Contents, &doc) != nil { continue @@ -100,6 +107,7 @@ func (b *Builder) collectRegexRuleRefs() error { } for _, target := range regexRuleTargets(em["ValidationRules"]) { b.regexRuleRefs = append(b.regexRuleRefs, regexRuleRef{ + owner: b.currentOwner(), entityQualifiedName: moduleName + "." + entityName, moduleName: moduleName, regexQualifiedName: target, @@ -107,6 +115,7 @@ func (b *Builder) collectRegexRuleRefs() error { } } } + b.disown() return nil } diff --git a/mdl/catalog/builder_rest.go b/mdl/catalog/builder_rest.go index 0de19f65ed..2356adbabd 100644 --- a/mdl/catalog/builder_rest.go +++ b/mdl/catalog/builder_rest.go @@ -10,7 +10,7 @@ import ( // buildRestClients populates the rest_clients and rest_operations catalog tables. func (b *Builder) buildRestClients() error { - services, err := b.reader.ListConsumedRestServices() + services, err := b.lister().ListConsumedRestServices() if err != nil { return err } @@ -41,6 +41,9 @@ func (b *Builder) buildRestClients() error { opCount := 0 for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + svc.Name @@ -97,6 +100,7 @@ func (b *Builder) buildRestClients() error { opCount++ } } + b.disown() b.report("REST Clients", len(services)) if opCount > 0 { @@ -108,7 +112,7 @@ func (b *Builder) buildRestClients() error { // publishedRestRef is one published REST operation → microflow edge. sourceID is // the operation's synthetic catalog id, so the emitted edge joins back to // published_rest_operations_data rather than only naming the operation in prose. -type publishedRestRef struct{ sourceID, qualifiedName, moduleName, microflow string } +type publishedRestRef struct{ owner, sourceID, qualifiedName, moduleName, microflow string } // publishedRestOpName renders the name a published operation is known by in the // reference graph. The empty parts are dropped rather than left as runs of @@ -126,7 +130,7 @@ func publishedRestOpName(service, resource, method, path string) string { // buildPublishedRestServices populates the published_rest_services and published_rest_operations catalog tables. func (b *Builder) buildPublishedRestServices() error { - services, err := b.reader.ListPublishedRestServices() + services, err := b.lister().ListPublishedRestServices() if err != nil { return err } @@ -157,6 +161,9 @@ func (b *Builder) buildPublishedRestServices() error { totalOps := 0 for _, svc := range services { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + svc.Name @@ -223,6 +230,7 @@ func (b *Builder) buildPublishedRestServices() error { // name would collide with every other unnamed target in refs. if op.Microflow != "" { b.publishedRestRefs = append(b.publishedRestRefs, publishedRestRef{ + owner: b.currentOwner(), sourceID: opID, // The resource is part of what makes an operation unique: // two resources of one service can both expose GET on the @@ -235,6 +243,7 @@ func (b *Builder) buildPublishedRestServices() error { } } } + b.disown() b.report("Published REST Services", len(services)) if totalOps > 0 { diff --git a/mdl/catalog/builder_rest_refs_test.go b/mdl/catalog/builder_rest_refs_test.go index 858251c569..4ca93073fd 100644 --- a/mdl/catalog/builder_rest_refs_test.go +++ b/mdl/catalog/builder_rest_refs_test.go @@ -52,14 +52,14 @@ func restFixture(t *testing.T, svc *model.PublishedRestService) *Catalog { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } if err := b.buildPublishedRestServices(); err != nil { t.Fatalf("buildPublishedRestServices: %v", err) } - stmt, err := tx.Prepare(` + stmt, err := b.tx.Prepare(` INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetId, TargetName, RefKind, ModuleName, ProjectId, SnapshotId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) diff --git a/mdl/catalog/builder_rest_timeout_test.go b/mdl/catalog/builder_rest_timeout_test.go index cd8f367d88..4c7f0d4b86 100644 --- a/mdl/catalog/builder_rest_timeout_test.go +++ b/mdl/catalog/builder_rest_timeout_test.go @@ -72,7 +72,7 @@ func TestActivitiesCarryTheRestCallTimeout(t *testing.T) { if err != nil { t.Fatal(err) } - b.tx = tx + b.tx = wrapTx(tx) if err := b.buildMicroflows(); err != nil { t.Fatalf("buildMicroflows: %v", err) } diff --git a/mdl/catalog/builder_scheduling.go b/mdl/catalog/builder_scheduling.go index 5268272c8f..442c75116a 100644 --- a/mdl/catalog/builder_scheduling.go +++ b/mdl/catalog/builder_scheduling.go @@ -22,11 +22,11 @@ import ( // enabled — are the whole reason to query them. // scheduledEventRef is one event → microflow edge, held until buildReferences. -type scheduledEventRef struct{ qualifiedName, moduleName, microflow string } +type scheduledEventRef struct{ owner, qualifiedName, moduleName, microflow string } // buildScheduledEvents catalogs ScheduledEvents$ScheduledEvent units. func (b *Builder) buildScheduledEvents() error { - units, err := b.reader.ListRawUnitsByType(sched.TypeName) + units, err := b.lister().ListRawUnitsByType(sched.TypeName) if err != nil { return err } @@ -47,6 +47,9 @@ func (b *Builder) buildScheduledEvents() error { count := 0 for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc bson.M if err := bsoncompat.UnmarshalDocM(u.Contents, &doc); err != nil { continue @@ -74,6 +77,7 @@ func (b *Builder) buildScheduledEvents() error { } if ev.MicroflowID != "" { b.scheduledEventRefs = append(b.scheduledEventRefs, scheduledEventRef{ + owner: b.currentOwner(), qualifiedName: moduleName + "." + ev.Name, moduleName: moduleName, microflow: string(ev.MicroflowID), @@ -81,6 +85,7 @@ func (b *Builder) buildScheduledEvents() error { } count++ } + b.disown() b.report("Scheduled Events", count) return nil @@ -88,7 +93,7 @@ func (b *Builder) buildScheduledEvents() error { // buildQueues catalogs Queues$Queue units. func (b *Builder) buildQueues() error { - units, err := b.reader.ListRawUnitsByType("Queues$Queue") + units, err := b.lister().ListRawUnitsByType("Queues$Queue") if err != nil { return err } @@ -108,6 +113,9 @@ func (b *Builder) buildQueues() error { count := 0 for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc bson.M if err := bsoncompat.UnmarshalDocM(u.Contents, &doc); err != nil { continue @@ -137,6 +145,7 @@ func (b *Builder) buildQueues() error { } count++ } + b.disown() b.report("Task Queues", count) return nil diff --git a/mdl/catalog/builder_service_refs.go b/mdl/catalog/builder_service_refs.go index 6bc6568e7c..3671f83439 100644 --- a/mdl/catalog/builder_service_refs.go +++ b/mdl/catalog/builder_service_refs.go @@ -3,7 +3,6 @@ package catalog import ( - "database/sql" "strings" "github.com/mendixlabs/mxcli/model" @@ -18,6 +17,7 @@ import ( // Collected while the service builders catalogue the services, and emitted by // buildReferences, like scheduledEventRefs and publishedRestRefs. type serviceMicroflowRef struct { + owner string sourceType, sourceID, sourceName, moduleName, microflow, kind string } @@ -35,6 +35,7 @@ func (b *Builder) addServiceMicroflowRefs(sourceType string, sourceID model.ID, } seen[mf] = true b.serviceMicroflowRefs = append(b.serviceMicroflowRefs, serviceMicroflowRef{ + owner: b.currentOwner(), sourceType: sourceType, sourceID: string(sourceID), sourceName: sourceName, @@ -73,9 +74,11 @@ func odataModeMicroflow(mode string) string { } // extractServiceMicroflowRefs emits the edges the service builders collected. -func (b *Builder) extractServiceMicroflowRefs(stmt *sql.Stmt, projectID, snapshotID string) int { +func (b *Builder) extractServiceMicroflowRefs(stmt *Stmt, projectID, snapshotID string) int { + defer b.disown() count := 0 for _, r := range b.serviceMicroflowRefs { + b.as(r.owner) if _, err := stmt.Exec( r.sourceType, r.sourceID, r.sourceName, RefObjectMicroflow, "", r.microflow, diff --git a/mdl/catalog/builder_service_refs_test.go b/mdl/catalog/builder_service_refs_test.go index a28a53c374..e7cd2a8c69 100644 --- a/mdl/catalog/builder_service_refs_test.go +++ b/mdl/catalog/builder_service_refs_test.go @@ -64,7 +64,7 @@ func serviceRefsFixture(t *testing.T, r *mock.MockBackend) *Catalog { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } for _, step := range []struct { name string @@ -80,7 +80,7 @@ func serviceRefsFixture(t *testing.T, r *mock.MockBackend) *Catalog { } } - stmt, err := tx.Prepare(` + stmt, err := b.tx.Prepare(` INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetId, TargetName, RefKind, ModuleName, ProjectId, SnapshotId) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `) diff --git a/mdl/catalog/builder_simpledocs.go b/mdl/catalog/builder_simpledocs.go index 6e49b6c836..04cb385aac 100644 --- a/mdl/catalog/builder_simpledocs.go +++ b/mdl/catalog/builder_simpledocs.go @@ -14,7 +14,7 @@ import "go.mongodb.org/mongo-driver/v2/bson" // call, and an objects-view union. Used for image collections, JavaScript // actions, and data transformers, which had no catalog table before. func (b *Builder) buildSimpleNamedDocs(typePrefix, table, reportLabel string) error { - units, err := b.reader.ListRawUnitsByType(typePrefix) + units, err := b.lister().ListRawUnitsByType(typePrefix) if err != nil { return err } @@ -33,6 +33,9 @@ func (b *Builder) buildSimpleNamedDocs(typePrefix, table, reportLabel string) er count := 0 for _, u := range units { + if !b.own(string(u.ID)) { + continue + } var doc struct { Name string `bson:"Name"` Documentation string `bson:"Documentation"` @@ -53,6 +56,7 @@ func (b *Builder) buildSimpleNamedDocs(typePrefix, table, reportLabel string) er } count++ } + b.disown() b.report(reportLabel, count) return nil diff --git a/mdl/catalog/builder_source.go b/mdl/catalog/builder_source.go index 97df5cb7ae..57443d030a 100644 --- a/mdl/catalog/builder_source.go +++ b/mdl/catalog/builder_source.go @@ -84,7 +84,9 @@ func (b *Builder) buildSource() error { return nil } - // Phase 1: Collect all items to describe (fast — uses cached lists) + // Phase 1: Collect all items to describe (fast — uses cached lists). In + // an incremental build the lists hold the units in scope, and every + // other document's rows are carried over (builder_incremental.go). var items []sourceItem // Entities. An entity is described from its own subtree of its module's @@ -144,7 +146,7 @@ func (b *Builder) buildSource() error { } // Snippets (not cached — only used here) - snippetList, _ := b.reader.ListSnippets() + snippetList, _ := b.lister().ListSnippets() for _, sn := range snippetList { moduleID := b.hierarchy.findModuleID(sn.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -175,7 +177,7 @@ func (b *Builder) buildSource() error { // #912 asked for. Each already had a catalog table and a Reader method; only // the source text was missing, so `search` could not reach a mapping's // element names or a structure's fields. - jsonList, err := b.reader.ListJsonStructures() + jsonList, err := b.lister().ListJsonStructures() if err == nil { for _, js := range jsonList { moduleID := b.hierarchy.findModuleID(js.ContainerID) @@ -184,7 +186,7 @@ func (b *Builder) buildSource() error { } } - importList, err := b.reader.ListImportMappings() + importList, err := b.lister().ListImportMappings() if err == nil { for _, im := range importList { moduleID := b.hierarchy.findModuleID(im.ContainerID) @@ -193,7 +195,7 @@ func (b *Builder) buildSource() error { } } - exportList, err := b.reader.ListExportMappings() + exportList, err := b.lister().ListExportMappings() if err == nil { for _, em := range exportList { moduleID := b.hierarchy.findModuleID(em.ContainerID) @@ -202,6 +204,18 @@ func (b *Builder) buildSource() error { } } + // An item outside the scope — one the lists hold that is no unit, such + // as a System enumeration — keeps its carried row. + if b.scope != nil { + kept := items[:0] + for _, it := range items { + if b.scope[sourceOwner(it, sourceParentOf(entityUnit, it))] { + kept = append(kept, it) + } + } + items = kept + } + if len(items) == 0 { b.report("source", 0) return nil @@ -217,10 +231,17 @@ func (b *Builder) buildSource() error { } hashes := b.sourceBasis() plan := b.planSourceReuse(items, units, parents, hashes) + if b.scope != nil && plan.reason == reasonOtherUnitChanged { + // Describing everything is the answer, and the carried rows are not + // everything: build it all, as before. + return &incrementalFallback{"source: " + plan.reason} + } toDescribe := make([]sourceItem, 0, len(items)) + owners := make([]string, 0, len(items)) // the unit each row comes from: an entity's domain model for i, it := range items { if _, ok := plan.reused[i]; !ok { toDescribe = append(toDescribe, it) + owners = append(owners, sourceOwner(it, parents[i])) } } @@ -239,7 +260,7 @@ func (b *Builder) buildSource() error { case <-stop: return case <-ticker.C: - b.report(fmt.Sprintf("source %d/%d documents", done.Load(), total), int(done.Load())) + b.say(fmt.Sprintf("source %d/%d documents", done.Load(), total), int(done.Load())) } } }() @@ -261,24 +282,27 @@ func (b *Builder) buildSource() error { defer stmt.Close() count := 0 - for _, res := range results { + for i, res := range results { if res.text == "" { continue } + b.own(owners[i]) stmt.Exec(res.item.qn, res.item.objType, res.text, res.item.moduleName, res.item.id) count++ } for i, row := range plan.reused { it := items[i] + b.own(sourceOwner(it, parents[i])) stmt.Exec(it.qn, it.objType, row.text, it.moduleName, it.id) count++ } + b.disown() if err := b.persistSourceBasis(items, units, parents, hashes); err != nil { return err } if len(plan.reused) > 0 { - b.report(sourceReuseSummary(len(plan.reused), len(toDescribe)), len(toDescribe)) + b.say(sourceReuseSummary(len(plan.reused), len(toDescribe)), len(toDescribe)) } b.report("source", count) @@ -293,7 +317,7 @@ func (b *Builder) buildSource() error { } for _, objType := range SourceObjectTypes { if n := byType[objType]; n > 0 { - b.report(fmt.Sprintf("source: %d %s document(s) could not be described (e.g. %s)", + b.say(fmt.Sprintf("source: %d %s document(s) could not be described (e.g. %s)", n, objType, firstFailureReason(failures, objType)), n) } } diff --git a/mdl/catalog/builder_source_reuse.go b/mdl/catalog/builder_source_reuse.go index 4c41d01766..c2953afef9 100644 --- a/mdl/catalog/builder_source_reuse.go +++ b/mdl/catalog/builder_source_reuse.go @@ -187,42 +187,35 @@ func (b *Builder) sourceBasis() map[string]string { // still re-describes every entity of the module: planSourceReuse treats the // domain model as each entity's parent. // -// An entity whose subtree cannot be found keeps the whole domain model's -// hash, so it is described again whenever anything in its module changes. +// The entities are found in the stored bytes, not a decoded list, so an +// incremental build that rebuilds one module does not decode every domain +// model. A domain model whose contents do not parse keeps its whole-unit +// basis, and its entities have none of their own: each is then described +// again whenever the domain model changes, through its parent. func (b *Builder) entityBasis(hashes map[string]string) { - dms, err := b.cachedDomainModels() - if err != nil || len(dms) == 0 { - return - } + b.entityDM = map[string]string{} units, err := b.reader.ListRawUnitsByType("DomainModels$DomainModel") if err != nil { return } - raw := map[string][]byte{} for _, u := range units { - raw[string(u.ID)] = u.Contents - } - for _, dm := range dms { - dmID := string(dm.ID) - unitHash, ok := hashes[dmID] - if !ok { + dmID := string(u.ID) + if _, ok := hashes[dmID]; !ok { continue } - residual, entities, ok := splitDomainModel(raw[dmID]) - for _, ent := range dm.Entities { - h, found := entities[ent.Name] - if !ok || !found { - h = unitHash - } - hashes[string(ent.ID)] = h + "|" + dmID + residual, entities, ok := splitDomainModel(u.Contents) + if !ok { + continue } - if ok { - hashes[dmID] = residual + "|" + string(dm.ContainerID) + for id, h := range entities { + hashes[id] = h + "|" + dmID + b.entityDM[id] = dmID } + hashes[dmID] = residual + "|" + string(u.ContainerID) } } -// splitDomainModel hashes a domain model's entities by name, and everything +// splitDomainModel hashes a domain model's entities by ID, and everything // else in it as one residual. ok is false when the contents do not parse. func splitDomainModel(contents []byte) (residual string, entities map[string]string, ok bool) { elems, err := bson.Raw(contents).Elements() @@ -249,14 +242,21 @@ func splitDomainModel(contents []byte) (residual string, entities map[string]str if !isDoc { continue // the array's leading version marker } - name, _ := doc.Lookup("Name").StringValueOK() + _, id, isBin := doc.Lookup("$ID").BinaryOK() + if !isBin { + return "", nil, false + } sum := sha256.Sum256(doc) - entities[name] = hex.EncodeToString(sum[:]) + entities[formatGUID(id)] = hex.EncodeToString(sum[:]) } } return hex.EncodeToString(rest.Sum(nil)), entities, true } +// reasonOtherUnitChanged is why a source build describes every document: a +// unit changed that no describe covers. +const reasonOtherUnitChanged = "a unit other than a described document changed" + // sourceReusePlan says, per item, whether its previous row can be carried // over; reused[i] is that row's text. type sourceReusePlan struct { @@ -301,7 +301,7 @@ func (b *Builder) planSourceReuse(items []sourceItem, units, parents []string, h } for id := range changed { if !itemUnit[id] && !prev.described[id] { - return sourceReusePlan{reason: "a unit other than a described document changed"} + return sourceReusePlan{reason: reasonOtherUnitChanged} } } @@ -390,7 +390,16 @@ func (b *Builder) persistSourceBasis(items []sourceItem, units, parents []string return err } defer ustmt.Close() + // Each row belongs to the unit that holds it — an entity's to its domain + // model — and an incremental build writes only the rebuilt units' rows. for id, h := range hashes { + owner := id + if dm := b.entityDM[id]; dm != "" { + owner = dm + } + if !b.ownRows(owner) { + continue + } d := 0 if described[id] { d = 1 @@ -399,16 +408,19 @@ func (b *Builder) persistSourceBasis(items []sourceItem, units, parents []string return err } } + b.disown() estmt, err := b.tx.Prepare(`INSERT INTO source_elements (ElementId, UnitId, QualifiedName, ObjectType) VALUES (?, ?, ?, ?)`) if err != nil { return err } defer estmt.Close() for i, it := range items { + b.own(sourceOwner(it, parents[i])) if _, err := estmt.Exec(it.id, units[i], it.qn, it.objType); err != nil { return err } } + b.disown() _, err = b.tx.Exec(`INSERT OR REPLACE INTO catalog_meta (Key, Value) VALUES (?, ?)`, MetaSourceFingerprint, b.sourceFingerprint) return err } @@ -418,6 +430,15 @@ func sourceReuseSummary(reused, described int) string { return fmt.Sprintf("source: %d reused, %d described", reused, described) } +// sourceOwner is the unit an item's rows come from: the unit that holds it +// (an entity's domain model) when that is another one, else the item. +func sourceOwner(it sourceItem, parent string) string { + if parent != "" { + return parent + } + return it.id +} + // sourceParentOf is the unit that holds an item when that is not the item // itself: an entity's domain model. Every other source document is its own // unit, and its basis is that unit. diff --git a/mdl/catalog/builder_strings.go b/mdl/catalog/builder_strings.go index 2c1bee5c4f..4490b8f3d6 100644 --- a/mdl/catalog/builder_strings.go +++ b/mdl/catalog/builder_strings.go @@ -49,6 +49,9 @@ func (b *Builder) buildStrings() error { pageList, err := b.cachedPages() if err == nil { for _, pg := range pageList { + if !b.own(string(pg.ID)) { + continue + } if pg.URL == "" { continue } @@ -56,12 +59,16 @@ func (b *Builder) buildStrings() error { moduleName := b.hierarchy.getModuleName(moduleID) insert(moduleName+"."+pg.Name, "PAGE", pg.URL, "page_url", "", string(pg.ID), moduleName) } + b.disown() } // Extract from microflows — using cached list mfList, err := b.cachedMicroflows() if err == nil { for _, mf := range mfList { + if !b.own(string(mf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(mf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qn := moduleName + "." + mf.Name @@ -76,12 +83,16 @@ func (b *Builder) buildStrings() error { } extractLogNodeNames(mf.ObjectCollection, qn, "MICROFLOW", moduleName, insert) } + b.disown() } // Extract from workflows — using cached list wfList, err := b.cachedWorkflows() if err == nil { for _, wf := range wfList { + if !b.own(string(wf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(wf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qn := moduleName + "." + wf.Name @@ -101,12 +112,16 @@ func (b *Builder) buildStrings() error { extractWorkflowFlowStrings(wf.Flow, qn, moduleName, insert) } } + b.disown() } // Extract from published REST services - prsServices, err := b.reader.ListPublishedRestServices() + prsServices, err := b.lister().ListPublishedRestServices() if err == nil { for _, svc := range prsServices { + if !b.own(string(svc.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(svc.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qn := moduleName + "." + svc.Name @@ -128,6 +143,7 @@ func (b *Builder) buildStrings() error { } } } + b.disown() } b.buildTranslatableStrings(insert) @@ -227,12 +243,15 @@ func (b *Builder) buildDocumentationStrings() (int, error) { // The caller's insert closure counts the rows it writes, so nothing is counted // here — doing both reported twice the rows the table actually holds. func (b *Builder) buildTranslatableStrings(insert func(string, string, string, string, string, string, string)) { - units, err := b.reader.ListRawUnitsByType("") + units, err := b.lister().ListRawUnitsByType("") if err != nil { return } for _, u := range units { + if !b.ownRows(string(u.ID)) { + continue + } if len(u.Contents) == 0 { continue } @@ -251,6 +270,7 @@ func (b *Builder) buildTranslatableStrings(insert func(string, string, string, s insert(r.QualifiedName, r.ObjectType, r.StringValue, r.StringContext, r.Language, r.ElementID, r.ModuleName) } } + b.disown() } // stringRow is one row of the strings index. diff --git a/mdl/catalog/builder_strings_documentation_test.go b/mdl/catalog/builder_strings_documentation_test.go index 203100c47b..d5af3e0c77 100644 --- a/mdl/catalog/builder_strings_documentation_test.go +++ b/mdl/catalog/builder_strings_documentation_test.go @@ -94,7 +94,7 @@ func docFixture(t *testing.T, word string) *Catalog { containerParent: map[model.ID]model.ID{"dm-sales": modID}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), } // In the order Build runs them: strings reads the tables the others fill. for _, step := range []struct { diff --git a/mdl/catalog/builder_tx.go b/mdl/catalog/builder_tx.go new file mode 100644 index 0000000000..0629f76eba --- /dev/null +++ b/mdl/catalog/builder_tx.go @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "database/sql" + "fmt" + "regexp" + "strings" + "sync" +) + +// builderTx is the transaction a build writes through. It records, for every +// row inserted, which unit the row came from (the builder's current owner), +// as rowid ranges per table. The next build carries an unchanged unit's rows +// over by those ranges instead of producing them again (ako/mxcli#1804; see +// builder_incremental.go). +type builderTx struct { + CatalogTx + b *Builder + + mu sync.Mutex + tables map[string]string // statement text -> table it inserts into ("" if none) +} + +// Stmt is a prepared statement of a build. Exec records the rows an INSERT +// wrote; everything else is the *Stmt. +type Stmt struct { + *sql.Stmt + table string + tx *builderTx +} + +func newBuilderTx(tx CatalogTx, b *Builder) *builderTx { + return &builderTx{CatalogTx: tx, b: b, tables: map[string]string{}} +} + +// insertInto matches the table an INSERT writes to. +var insertInto = regexp.MustCompile(`(?is)^\s*INSERT\s+(?:OR\s+\w+\s+)?INTO\s+([A-Za-z_][A-Za-z0-9_]*)`) + +// insertTable is the table query inserts into, or "" when it is not an INSERT. +func (t *builderTx) insertTable(query string) string { + t.mu.Lock() + defer t.mu.Unlock() + table, ok := t.tables[query] + if !ok { + if m := insertInto.FindStringSubmatch(query); m != nil { + table = strings.ToLower(m[1]) + } + t.tables[query] = table + } + return table +} + +func (t *builderTx) Prepare(query string) (*Stmt, error) { + s, err := t.CatalogTx.Prepare(query) + if err != nil { + return nil, err + } + return &Stmt{Stmt: s, table: t.insertTable(query), tx: t}, nil +} + +func (t *builderTx) Exec(query string, args ...any) (sql.Result, error) { + res, err := t.CatalogTx.Exec(query, args...) + if err == nil { + if table := t.insertTable(query); table != "" { + t.record(table, res) + } + } + return res, err +} + +func (s *Stmt) Exec(args ...any) (sql.Result, error) { + res, err := s.Stmt.Exec(args...) + if err == nil && s.table != "" { + s.tx.record(s.table, res) + } + return res, err +} + +// record notes the rows an INSERT wrote, when the transaction belongs to a +// build (a pass run on its own in a test has nobody to record for). +func (t *builderTx) record(table string, res sql.Result) { + if t.b != nil && t.b.rows != nil { + t.b.rows.record(table, res) + } +} + +// rowRange is a run of consecutive rowids of one table written for one unit. +type rowRange struct { + owner string + first, last int64 +} + +// rowOwners is what a build records about the rows it wrote: which unit each +// one came from, and which units the per-unit passes visited at all. +type rowOwners struct { + owner string // the unit the rows being written now come from; "" for project-wide rows + ranges map[string][]rowRange // by table + owned map[string]bool // every unit a per-unit pass visited, in scope or not + err error // the first row written for a unit this build skipped +} + +func newRowOwners() *rowOwners { + return &rowOwners{ranges: map[string][]rowRange{}, owned: map[string]bool{}} +} + +// skippedOwner is the owner while a pass is between documents it does not +// rebuild. A row written then belongs to no unit the build can name, which is +// a bug in that pass: the build records it and falls back to a full one. +const skippedOwner = "\x00skipped" + +func (o *rowOwners) record(table string, res sql.Result) { + if strings.HasPrefix(table, "catalog_") { + return // the build's own bookkeeping + } + n, err := res.RowsAffected() + if err != nil || n <= 0 { + return + } + last, err := res.LastInsertId() + if err != nil { + return + } + if o.owner == skippedOwner { + if o.err == nil { + o.err = fmt.Errorf("a row of %s was written while its pass was skipping a document", table) + } + return + } + o.add(table, o.owner, last-n+1, last) +} + +// add appends a range, extending the table's last one when it continues it. +func (o *rowOwners) add(table, owner string, first, last int64) { + rs := o.ranges[table] + if k := len(rs) - 1; k >= 0 && rs[k].owner == owner && rs[k].last+1 == first { + rs[k].last = last + return + } + o.ranges[table] = append(rs, rowRange{owner, first, last}) +} diff --git a/mdl/catalog/builder_tx_helpers_test.go b/mdl/catalog/builder_tx_helpers_test.go new file mode 100644 index 0000000000..f07a0e1e27 --- /dev/null +++ b/mdl/catalog/builder_tx_helpers_test.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +// wrapTx is the transaction a pass run on its own in a test writes through. +func wrapTx(tx CatalogTx) *builderTx { + return newBuilderTx(tx, nil) +} diff --git a/mdl/catalog/builder_widget_definitions.go b/mdl/catalog/builder_widget_definitions.go index 26f55c0b07..7de0d33e89 100644 --- a/mdl/catalog/builder_widget_definitions.go +++ b/mdl/catalog/builder_widget_definitions.go @@ -12,7 +12,6 @@ package catalog import ( "crypto/sha1" - "database/sql" "encoding/hex" "encoding/json" "fmt" @@ -227,7 +226,7 @@ func mpkProperties(mpkDef *mpk.WidgetDefinition) []WidgetDefinitionPropMeta { // widget_definition_properties rows for its properties, child slots, and // object lists. func insertWidgetDef( - defStmt, propStmt *sql.Stmt, + defStmt, propStmt *Stmt, widgetID, mdlName, displayName, kind, version, mpkPath, defPath string, properties []WidgetDefinitionPropMeta, childSlots []WidgetDefinitionSlotMeta, diff --git a/mdl/catalog/builder_widget_refs.go b/mdl/catalog/builder_widget_refs.go index 6a15b467ef..f4ab172a4f 100644 --- a/mdl/catalog/builder_widget_refs.go +++ b/mdl/catalog/builder_widget_refs.go @@ -57,7 +57,7 @@ package catalog // four sibling projections in buildReferences. Per-instance rows would say // nothing SHOW REFERENCES or SHOW IMPACT could use, and CATALOG.WIDGETS already // holds the instances. -func insertWidgetRefs(tx CatalogTx, projectID, snapshotID string) (int, error) { +func insertWidgetRefs(tx *builderTx, projectID, snapshotID string) (int, error) { res, err := tx.Exec( `INSERT INTO refs (SourceType, SourceId, SourceName, TargetType, TargetId, TargetName, RefKind, ModuleName, ProjectId, SnapshotId) SELECT DISTINCT w.ContainerType, '', w.ContainerQualifiedName, diff --git a/mdl/catalog/builder_widget_refs_test.go b/mdl/catalog/builder_widget_refs_test.go index 5f8b996d2b..99bd0fd6fa 100644 --- a/mdl/catalog/builder_widget_refs_test.go +++ b/mdl/catalog/builder_widget_refs_test.go @@ -63,7 +63,7 @@ func runInsertWidgetRefs(t *testing.T, cat *Catalog) int { if err != nil { t.Fatalf("Begin: %v", err) } - n, err := insertWidgetRefs(tx, "p", "s") + n, err := insertWidgetRefs(wrapTx(tx), "p", "s") if err != nil { tx.Rollback() t.Fatalf("insertWidgetRefs: %v", err) diff --git a/mdl/catalog/builder_workflows.go b/mdl/catalog/builder_workflows.go index 0ff45c8a74..ec1cd00bf8 100644 --- a/mdl/catalog/builder_workflows.go +++ b/mdl/catalog/builder_workflows.go @@ -32,6 +32,9 @@ func (b *Builder) buildWorkflows() error { count := 0 for _, wf := range wfs { + if !b.own(string(wf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(wf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) qualifiedName := moduleName + "." + wf.Name @@ -66,6 +69,7 @@ func (b *Builder) buildWorkflows() error { } count++ } + b.disown() b.report("Workflows", count) return nil diff --git a/mdl/catalog/builder_xpath.go b/mdl/catalog/builder_xpath.go index 8e5a8d7360..e5ad42caea 100644 --- a/mdl/catalog/builder_xpath.go +++ b/mdl/catalog/builder_xpath.go @@ -4,7 +4,6 @@ package catalog import ( "crypto/sha256" - "database/sql" "fmt" "github.com/mendixlabs/mxcli/modelsdk/bsoncompat" "strings" @@ -50,6 +49,9 @@ func (b *Builder) buildXPathExpressions() error { mfs, err := b.cachedMicroflows() if err == nil { for _, mf := range mfs { + if !b.own(string(mf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(mf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) sourceQN := moduleName + "." + mf.Name @@ -57,11 +59,15 @@ func (b *Builder) buildXPathExpressions() error { count += b.extractMicroflowXPath(stmt, mf.ObjectCollection, "MICROFLOW", string(mf.ID), sourceQN, moduleName, projectID, snapshotID) } + b.disown() } nfs, err := b.cachedNanoflows() if err == nil { for _, nf := range nfs { + if !b.own(string(nf.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(nf.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) sourceQN := moduleName + "." + nf.Name @@ -69,12 +75,16 @@ func (b *Builder) buildXPathExpressions() error { count += b.extractMicroflowXPath(stmt, nf.ObjectCollection, "NANOFLOW", string(nf.ID), sourceQN, moduleName, projectID, snapshotID) } + b.disown() } // 2. Extract from entity access rules dms, err := b.cachedDomainModels() if err == nil { for _, dm := range dms { + if !b.own(string(dm.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(dm.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -99,17 +109,21 @@ func (b *Builder) buildXPathExpressions() error { } } } + b.disown() } // 3. Extract from page/snippet data sources by scanning raw BSON // The page parser only reads metadata, not the widget tree, so we scan the // raw BSON documents for XPathConstraint fields at any nesting depth. for _, typePrefix := range []string{"Forms$Page", "Pages$Page", "Forms$Snippet", "Pages$Snippet"} { - rawPages, err := b.reader.ListRawUnitsByType(typePrefix) + rawPages, err := b.lister().ListRawUnitsByType(typePrefix) if err != nil { continue } for _, ru := range rawPages { + if !b.own(string(ru.ID)) { + continue + } moduleID := b.hierarchy.findModuleID(ru.ContainerID) moduleName := b.hierarchy.getModuleName(moduleID) @@ -130,6 +144,7 @@ func (b *Builder) buildXPathExpressions() error { count += scanBSONForXPath(stmt, raw, docType, string(ru.ID), sourceQN, moduleName, projectID, snapshotID) } + b.disown() } b.report("XPath Expressions", count) @@ -137,7 +152,7 @@ func (b *Builder) buildXPathExpressions() error { } // extractMicroflowXPath extracts XPath constraints from microflow/nanoflow actions. -func (b *Builder) extractMicroflowXPath(stmt *sql.Stmt, +func (b *Builder) extractMicroflowXPath(stmt *Stmt, oc *microflows.MicroflowObjectCollection, docType, docID, docQN, moduleName, projectID, snapshotID string) int { @@ -189,7 +204,7 @@ func (b *Builder) extractMicroflowXPath(stmt *sql.Stmt, // scanBSONForXPath recursively scans a BSON map for XPathConstraint fields // and inserts records into the xpath_expressions table. // This works on raw BSON data, avoiding the need for a full widget tree parser. -func scanBSONForXPath(stmt *sql.Stmt, raw map[string]any, +func scanBSONForXPath(stmt *Stmt, raw map[string]any, docType, docID, docQN, moduleName, projectID, snapshotID string) int { diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index 9e00de0cfb..4d2c3aeed6 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -1526,6 +1526,28 @@ func (c *Catalog) createTables() error { // document came from which unit. The next source build compares and // re-describes only what changed (builder_source_reuse.go, #1790). A // cache without them reuses nothing, so they need no schema bump. + // Which unit each row came from, as rowid ranges per table; every + // unit's basis (content hash and container) and whether a per-unit + // pass reads it; and the qualified names each unit's rows carry. The + // next build rebuilds only the changed units and the documents that + // name them, and copies the rest (builder_incremental.go, #1804). A + // cache without them carries nothing, so they need no schema bump. + `CREATE TABLE IF NOT EXISTS catalog_row_owners ( + TableName TEXT NOT NULL, + UnitId TEXT NOT NULL, + FirstRow INTEGER NOT NULL, + LastRow INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS catalog_units ( + UnitId TEXT PRIMARY KEY, + Basis TEXT NOT NULL, + Owned INTEGER NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS catalog_unit_names ( + UnitId TEXT NOT NULL, + QualifiedName TEXT NOT NULL + )`, + `CREATE TABLE IF NOT EXISTS source_units ( UnitId TEXT PRIMARY KEY, Hash TEXT NOT NULL, diff --git a/mdl/catalog/widget_multi_refs_test.go b/mdl/catalog/widget_multi_refs_test.go index b88a341956..7a72bc7ba6 100644 --- a/mdl/catalog/widget_multi_refs_test.go +++ b/mdl/catalog/widget_multi_refs_test.go @@ -112,7 +112,7 @@ func TestWidgetWithDatasourceAndClickRefsBothMicroflows(t *testing.T) { containerParent: map[model.ID]model.ID{}, folderNames: map[model.ID]string{}, }, - tx: tx, + tx: wrapTx(tx), fullMode: true, } if err := b.buildPages(); err != nil { diff --git a/mdl/executor/catalog_incremental_budget_pedapp_test.go b/mdl/executor/catalog_incremental_budget_pedapp_test.go new file mode 100644 index 0000000000..f2b1267578 --- /dev/null +++ b/mdl/executor/catalog_incremental_budget_pedapp_test.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "context" + "fmt" + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/catalog" + "github.com/mendixlabs/mxcli/modelsdk/mprread" +) + +// ako/mxcli#1804: an incremental catalog build is only fast while every pass +// keeps to its rules, and breaking one is silent — the build stays correct +// and gets slower. These tests make the two ways it gets slower fail: rows +// no document owns (rebuilt on every refresh) and documents decoded outside +// the scope (every refresh decodes the project again). + +// projectWideTables are the tables whose rows may belong to no document, +// and why. Such rows are dropped and written again on every refresh, so a +// table belongs here only when its rows cannot come from one document. +var projectWideTables = map[string]string{ + "projects": "the build's own record", + "snapshots": "the build's own record", + "modules_data": "a module is no per-document pass's unit; a module change rebuilds everything", + "navigation_profiles_data": "the navigation document; a change to it rebuilds everything", + "navigation_menu_items": "the navigation document", + "navigation_role_homes": "the navigation document", + "constant_values": "the project settings; a change to them rebuilds everything", + "user_roles": "the project security; a change to it rebuilds everything", + "role_mappings": "the project security", + "widget_definitions_data": "widgets/*.mpk and the built-in definitions, not the model", + "widget_definition_properties_data": "widgets/*.mpk and the built-in definitions, not the model", + "external_actions_data": "aggregated over every flow that calls an action", + "refs": "projections over other tables (widgets, XPath, enumeration types) and the navigation and settings edges", + "strings": "the documentation projection over the object tables", + "communities_data": "the graph analysis, over all refs", + "graph_cycles_data": "the graph analysis", + "graph_module_cycles_data": "the graph analysis", + "graph_layers_data": "the graph analysis", + "graph_centrality_data": "the graph analysis", +} + +func TestCatalog_RowsBelongToADocument(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + ctx := exec.newExecContext(context.Background()) + ctx.Output = out + ctx.catalogForce = true + if err := buildCatalog(ctx, true, true, false, 0); err != nil { + t.Fatalf("build: %v\n%s", err, out.String()) + } + defer func() { ctx.Catalog.Close(); ctx.Catalog = nil }() + res, err := ctx.Catalog.Query(`SELECT TableName, SUM(LastRow - FirstRow + 1) FROM catalog_row_owners + WHERE UnitId = '' GROUP BY TableName`) + if err != nil { + t.Fatal(err) + } + owned, err := ctx.Catalog.Query(`SELECT COUNT(*) FROM catalog_row_owners WHERE UnitId != ''`) + if err != nil || len(owned.Rows) == 0 || fmt.Sprint(owned.Rows[0][0]) == "0" { + t.Fatalf("control: the build recorded no document's rows (%v)", err) + } + var stray []string + for _, r := range res.Rows { + table := fmt.Sprint(r[0]) + if _, ok := projectWideTables[table]; !ok { + stray = append(stray, fmt.Sprintf("%s (%v rows)", table, r[1])) + } + } + sort.Strings(stray) + if len(stray) > 0 { + t.Errorf("rows that belong to no document, so every refresh writes them again:\n %s\n"+ + "Call b.own(unitID) at the head of the pass's per-document loop (builder_incremental.go), "+ + "or, if the rows really cannot come from one document, add the table to projectWideTables with the reason.", + strings.Join(stray, "\n ")) + } +} + +func TestCatalog_IncrementalRefreshDecodesOnlyWhatItRebuilds(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + if _, stats := catalogRows(t, exec, true, true); stats.Incremental { + t.Fatal("control: the forced build was incremental") + } + if err := agreeExec(t, exec, strings.Replace(redeclareChain(2, "create"), "length($Title) + $Prev", "length($Title) * 3 + $Prev", 1)); err != nil { + t.Fatalf("edit: %v\n%s", err, out.String()) + } + before := mprread.UnitsDecoded() + var stats catalog.BuildStats + catalogBuildHook = func(s catalog.BuildStats) { stats = s } + defer func() { catalogBuildHook = nil }() + ctx := exec.newExecContext(context.Background()) + ctx.Output = out + if err := buildCatalog(ctx, true, false, false, 0); err != nil { + t.Fatalf("build: %v\n%s", err, out.String()) + } + ctx.Catalog.Close() + ctx.Catalog = nil + decoded := mprread.UnitsDecoded() - before + if !stats.Incremental || stats.Rebuilt == 0 { + t.Fatalf("control: the refresh after an edit was not incremental (%+v)", stats) + } + // The project-wide passes read the project settings, the project + // security and the navigation, a few times between them: a constant, + // not a share of the project. + const singletons = 6 + if budget := int64(stats.Rebuilt + singletons); decoded > budget { + t.Errorf("an incremental refresh that rebuilt %d units decoded %d, want at most %d.\n"+ + "A pass lists documents through the reader instead of the build's scoped list: use b.lister() "+ + "or the cached accessors (builder.go), and look other documents up in the catalog tables.", + stats.Rebuilt, decoded, budget) + } + t.Logf("decoded %d units, rebuilt %d", decoded, stats.Rebuilt) +} diff --git a/mdl/executor/catalog_incremental_pedapp_test.go b/mdl/executor/catalog_incremental_pedapp_test.go new file mode 100644 index 0000000000..1c8130ef1c --- /dev/null +++ b/mdl/executor/catalog_incremental_pedapp_test.go @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "context" + "fmt" + "os" + "sort" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/catalog" +) + +// ako/mxcli#1804: a refresh rebuilt every catalog table from every unit after +// any save. It now rebuilds the changed units and the documents that name +// them, and copies every other unit's rows from the cached catalog. That is +// only safe if it never changes the answer, so every step below compares +// every table of the incremental build with a forced full one. + +// catalogRows builds the catalog, incremental unless force, and returns every +// row of every table by table, with how the build went. +func catalogRows(t *testing.T, exec *Executor, full, force bool) (map[string][]string, catalog.BuildStats) { + t.Helper() + var stats catalog.BuildStats + catalogBuildHook = func(s catalog.BuildStats) { stats = s } + defer func() { catalogBuildHook = nil }() + + ctx := exec.newExecContext(context.Background()) + var out bytes.Buffer + ctx.Output = &out + ctx.catalogForce = force + if err := buildCatalog(ctx, full, false, false, 0); err != nil { + t.Fatalf("build: %v\n%s", err, out.String()) + } + defer func() { ctx.Catalog.Close(); ctx.Catalog = nil }() + return dumpCatalog(t, ctx.Catalog), stats +} + +// dumpCatalog reads every row of every table the build writes. The rowid +// (an INTEGER PRIMARY KEY column) is left out: it numbers rows in the order +// they were written, and a carried row is written before a rebuilt one. +func dumpCatalog(t *testing.T, cat *catalog.Catalog) map[string][]string { + t.Helper() + tables, err := cat.Query(`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'`) + if err != nil { + t.Fatal(err) + } + isShadow := func(name string) bool { + for _, fts := range []string{"strings_", "source_"} { + if rest, ok := strings.CutPrefix(name, fts); ok { + switch rest { + case "data", "idx", "content", "docsize", "config": + return true + } + } + } + return false + } + rows := map[string][]string{} + for _, tr := range tables.Rows { + name := fmt.Sprint(tr[0]) + // What records when and how the build ran, not what it found. + if name == "catalog_meta" || name == "snapshots" || name == "projects" || + strings.HasPrefix(name, "catalog_") || isShadow(name) { + continue + } + info, err := cat.Query(`SELECT name, type, pk FROM pragma_table_info('` + name + `')`) + if err != nil { + t.Fatal(err) + } + var cols []string + for _, c := range info.Rows { + if fmt.Sprint(c[2]) == "1" && strings.EqualFold(fmt.Sprint(c[1]), "INTEGER") { + continue + } + cols = append(cols, `"`+fmt.Sprint(c[0])+`"`) + } + res, err := cat.Query("SELECT " + strings.Join(cols, ", ") + " FROM " + name) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + for _, r := range res.Rows { + rows[name] = append(rows[name], fmt.Sprint(r...)) + } + sort.Strings(rows[name]) + } + return rows +} + +// diffTables reports every table whose rows differ, with a few of the rows. +func diffTables(t *testing.T, step string, got, want map[string][]string) { + t.Helper() + names := map[string]bool{} + for n := range got { + names[n] = true + } + for n := range want { + names[n] = true + } + var sorted []string + for n := range names { + sorted = append(sorted, n) + } + sort.Strings(sorted) + for _, n := range sorted { + g, w := got[n], want[n] + if strings.Join(g, "\n") == strings.Join(w, "\n") { + continue + } + missing, extra := multisetDiff(w, g), multisetDiff(g, w) + t.Errorf("%s: table %s differs from a full build (%d rows, want %d)\n missing: %s\n extra: %s", + step, n, len(g), len(w), sample(missing), sample(extra)) + } +} + +// multisetDiff is the rows of a that b does not have, counting duplicates. +func multisetDiff(a, b []string) []string { + have := map[string]int{} + for _, r := range b { + have[r]++ + } + var out []string + for _, r := range a { + if have[r] > 0 { + have[r]-- + continue + } + out = append(out, r) + } + return out +} + +func sample(rows []string) string { + if len(rows) > 3 { + return fmt.Sprintf("%q … (%d more)", rows[:3], len(rows)-3) + } + return fmt.Sprintf("%q", rows) +} + +func TestCatalog_IncrementalBuildEqualsFullBuild(t *testing.T) { + incrementalSteps(t, true) +} + +// The fast build (a plain REFRESH CATALOG) writes no refs, so it has no +// dependents to follow: every row it writes must come from its own unit. +func TestCatalog_IncrementalFastBuildEqualsFullBuild(t *testing.T) { + incrementalSteps(t, false) +} + +func incrementalSteps(t *testing.T, full bool) { + exec, out, dir := openPedAppCopy(t) + setupIncrementalProbes(t, exec, out) + cachePath := getCachePath(exec.newExecContext(context.Background())) + base, _ := catalogRows(t, exec, full, true) + if full && (len(base["refs"]) < 50 || len(base["activities_data"]) < 50) { + t.Fatalf("control: the fixture has %d refs and %d activities", len(base["refs"]), len(base["activities_data"])) + } + if len(base["entities_data"]) < 20 || len(base["microflows_data"]) < 20 { + t.Fatalf("control: the fixture has %d entities and %d flows", len(base["entities_data"]), len(base["microflows_data"])) + } + + // Each step edits the project the way an agent does between two + // refreshes. full says the step must not be incremental: it changes a + // unit no per-unit pass reads. + steps := []struct { + name, mdl string + full bool + rounds int // 2: the plan must widen its scope once + }{ + {name: "no change"}, + {name: "edit a flow", mdl: strings.Replace(redeclareChain(2, "create or modify"), "length($Title) + $Prev", "length($Title) * 2 + $Prev", 1)}, + {name: "callee returns another type", mdl: `mdl 1; +create or modify microflow MyFirstModule.MF_Chain001 ($Title: String) returns Boolean as $B +begin + declare $B Boolean = length($Title) > 3; + return $B; +end; +`}, + {name: "data source flow returns another entity", mdl: `mdl 1; +create or modify microflow MyFirstModule.GetProbe () returns MyFirstModule.ProbeB as $B +begin + $B = create MyFirstModule.ProbeB (Name = 'b'); + return $B; +end; +`}, + {name: "add a flow", mdl: `mdl 1; +create microflow MyFirstModule.MF_Added ($Title: String) returns Integer as $N +begin + $N = call microflow MyFirstModule.MF_Chain004(Title = $Title); + return $N; +end; +`}, + {name: "rename a flow", mdl: "rename microflow MyFirstModule.MF_Chain004 to MF_Chain004b;"}, + {name: "alter an entity", mdl: "alter entity FeedbackModule.Feedback add attribute IncrementalProbe: String;"}, + {name: "rename an inherited attribute", mdl: "alter entity MyFirstModule.ProbeA rename attribute Name to Label;"}, + // Nothing in the flow changes, and what it deletes does: the + // variable's entity is the association's other end. + {name: "repoint an association", mdl: `drop association MyFirstModule.ProbeLink; +create association MyFirstModule.ProbeLink from MyFirstModule.ProbeD to MyFirstModule.ProbeA;`}, + {name: "add an association", mdl: "create association MyFirstModule.ProbeA_ProbeB from MyFirstModule.ProbeA to MyFirstModule.ProbeB;"}, + {name: "drop an association", mdl: "drop association MyFirstModule.ProbeA_ProbeB;"}, + {name: "rename an entity", mdl: "rename entity MyFirstModule.ProbeB to ProbeC;"}, + {name: "add an enumeration value", mdl: "alter enumeration MyFirstModule.ProbeColor add value Blue caption 'Blue';"}, + {name: "edit a page", mdl: `mdl 1; +create or modify page MyFirstModule.ProbePage (Title: 'P2', Layout: Atlas_Core.Atlas_Default) { + dataview dataView1 (DataSource: microflow MyFirstModule.GetProbe) { + textbox txtName (Attribute: Name) + actionbutton btnGo (Caption: 'Go', Action: microflow MyFirstModule.MF_Chain001) + } +}; +`}, + {name: "move a flow to a new folder", mdl: "move microflow MyFirstModule.MF_Chain001 to folder 'Probes';", full: true}, + {name: "move a flow into that folder", mdl: "move microflow MyFirstModule.MF_Chain002 to folder 'Probes';"}, + {name: "drop a flow", mdl: "drop microflow MyFirstModule.MF_Added;"}, + // The page still names the flow it no longer finds. Created again, + // the flow is a unit the previous catalog never knew, so only the + // built catalog shows the page naming it: the plan widens once. + {name: "drop a called flow", mdl: "drop microflow MyFirstModule.GetProbe;"}, + {name: "create it again", mdl: `mdl 1; +create microflow MyFirstModule.GetProbe () returns MyFirstModule.ProbeA as $A +begin + $A = create MyFirstModule.ProbeA (Label = 'a'); + return $A; +end; +`, rounds: 2}, + } + + incremental := 0 + for _, s := range steps { + if s.mdl != "" { + if err := agreeExec(t, exec, s.mdl); err != nil { + t.Fatalf("%s: %v\n%s", s.name, err, out.String()) + } + } + got, stats := catalogRows(t, exec, full, false) + // A second refresh with nothing changed carries from the first. + again, againStats := catalogRows(t, exec, full, false) + if !againStats.Incremental || againStats.Rebuilt != 0 { + t.Errorf("%s: a refresh with nothing changed rebuilt %d units (%+v)", s.name, againStats.Rebuilt, againStats) + } + // The forced full build is the reference. It replaces the cache, so + // the incremental one is put back: the next step carries from it, and + // every build in the sequence is incremental on incremental. + saved, err := os.ReadFile(cachePath) + if err != nil { + t.Fatal(err) + } + want, _ := catalogRows(t, exec, full, true) + if err := os.WriteFile(cachePath, saved, 0o644); err != nil { + t.Fatal(err) + } + + diffTables(t, s.name, got, want) + diffTables(t, s.name+" (again)", again, want) + switch { + case s.full && stats.Incremental: + t.Errorf("%s: the build was incremental, want a full one", s.name) + case !s.full && !stats.Incremental: + t.Errorf("%s: the build was full (%s), want an incremental one", s.name, stats.Reason) + } + if full && s.rounds != 0 && stats.Rounds != s.rounds { + t.Errorf("%s: the build took %d rounds, want %d", s.name, stats.Rounds, s.rounds) + } + if stats.Incremental { + incremental++ + } + t.Logf("%s: %+v", s.name, stats) + } + if incremental == 0 { + t.Fatalf("control: no build was incremental, so the comparison proves nothing about carrying rows") + } + _ = dir +} + +// setupIncrementalProbes adds the documents the incremental steps edit: a +// call chain, entities with a specialization and an association, a flow +// that deletes over that association, and a page whose data source is a +// flow. +func setupIncrementalProbes(t *testing.T, exec *Executor, out *bytes.Buffer) { + t.Helper() + if err := agreeExec(t, exec, redeclareChain(4, "create")); err != nil { + t.Fatalf("setup: %v\n%s", err, out.String()) + } + if err := agreeExec(t, exec, `mdl 1; +create persistent entity MyFirstModule.ProbeA (Name: String(50)); +create persistent entity MyFirstModule.ProbeB (Name: String(50)); +create persistent entity MyFirstModule.ProbeSub extends MyFirstModule.ProbeA (Extra: String(50)); +create persistent entity MyFirstModule.ProbeD (Name: String(50)); +create association MyFirstModule.ProbeLink from MyFirstModule.ProbeB to MyFirstModule.ProbeA; +create enumeration MyFirstModule.ProbeColor (Red 'Red', Green 'Green'); +create microflow MyFirstModule.GetProbe () returns MyFirstModule.ProbeA as $A +begin + $A = create MyFirstModule.ProbeA (Name = 'a'); + return $A; +end; +create microflow MyFirstModule.UseSub ($S: MyFirstModule.ProbeSub) returns String as $N +begin + declare $N String = $S/Name; + return $N; +end; +create microflow FeedbackModule.DropLinked ($A: MyFirstModule.ProbeA) +begin + retrieve $Linked from $A/MyFirstModule.ProbeLink; + delete $Linked; +end; +create page MyFirstModule.ProbePage (Title: 'P', Layout: Atlas_Core.Atlas_Default) { + dataview dataView1 (DataSource: microflow MyFirstModule.GetProbe) { + textbox txtName (Attribute: Name) + } +}; +`); err != nil { + t.Fatalf("setup probes: %v\n%s", err, out.String()) + } +} diff --git a/mdl/executor/catalog_ref_coverage_pedapp_test.go b/mdl/executor/catalog_ref_coverage_pedapp_test.go new file mode 100644 index 0000000000..12615c9b9d --- /dev/null +++ b/mdl/executor/catalog_ref_coverage_pedapp_test.go @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 + +package executor + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" + modelsdkbackend "github.com/mendixlabs/mxcli/mdl/backend/modelsdk" + "github.com/mendixlabs/mxcli/mdl/catalog" +) + +// ako/mxcli#1804: an incremental build rebuilds a document when a unit its +// refs name changes, so refs must cover every other document a pass reads +// for it. A read they miss is a row that goes stale without an error. The +// build traces every name its lookups read per document, and every read +// must be one a change would follow. + +// tracedFullBuild builds the catalog from scratch with every read traced and +// returns how much of it refs cover. +func tracedFullBuild(t *testing.T, exec *Executor) *catalog.DependencyCoverage { + t.Helper() + var builder *catalog.Builder + catalogBuilderHook = func(b *catalog.Builder) { + b.SetTraceDependencies(true) + builder = b + } + defer func() { catalogBuilderHook = nil }() + ctx := exec.newExecContext(context.Background()) + var out bytes.Buffer + ctx.Output = &out + ctx.catalogForce = true + if err := buildCatalog(ctx, true, false, false, 0); err != nil { + t.Fatalf("build: %v\n%s", err, out.String()) + } + defer func() { ctx.Catalog.Close(); ctx.Catalog = nil }() + cov, err := builder.DependencyCoverage() + if err != nil { + t.Fatal(err) + } + return cov +} + +func coverageReport(cov *catalog.DependencyCoverage, limit int) string { + var b strings.Builder + fmt.Fprintf(&b, "%d of %d reads of another document covered by refs (%d of the document's own unit, %d of names no unit holds)", + cov.Covered, cov.Reads, cov.Self, cov.Unowned) + bySite := map[string]int{} + for _, u := range cov.Uncovered { + bySite[u.Site]++ + } + for site, n := range bySite { + fmt.Fprintf(&b, "\n %s: %d uncovered", site, n) + } + for i, u := range cov.Uncovered { + if i == limit { + fmt.Fprintf(&b, "\n … %d more", len(cov.Uncovered)-limit) + break + } + fmt.Fprintf(&b, "\n %s read %s (held by %s's unit) through %s", u.Reader, u.Name, u.Holder, u.Site) + } + return b.String() +} + +func TestCatalog_RefsCoverEveryCrossDocumentRead(t *testing.T) { + exec, out, _ := openPedAppCopy(t) + setupIncrementalProbes(t, exec, out) + cov := tracedFullBuild(t, exec) + if cov.Reads == 0 { + t.Fatal("control: the build traced no read of another document; the tracer is not wired") + } + if len(cov.Uncovered) > 0 { + t.Errorf("refs miss reads the incremental build depends on; a change to the read document leaves the reader's rows stale.\n"+ + "Emit a refs edge from the reader to what it reads (builder_references.go), and add an incremental step for it:\n%s", + coverageReport(cov, 20)) + } + t.Log(coverageReport(cov, 0)) +} + +// TestCatalog_RefCoverageOfProject measures ref coverage on a real project: +// MXCLI_REF_COVERAGE_MPR=/path/to/app.mpr (a copy: the catalog cache is +// written beside it). It reports, and fails only on an uncovered read, so a +// project that shows a gap is a reproduction to add to the test above. +func TestCatalog_RefCoverageOfProject(t *testing.T) { + mpr := os.Getenv("MXCLI_REF_COVERAGE_MPR") + if mpr == "" { + t.Skip("set MXCLI_REF_COVERAGE_MPR to a project copy to measure it") + } + exec := New(&bytes.Buffer{}) + exec.SetBackendFactory(func() backend.FullBackend { return modelsdkbackend.New() }) + if err := exec.Execute(&ast.ConnectStmt{Path: mpr}); err != nil { + t.Fatalf("connect: %v", err) + } + t.Cleanup(func() { _ = exec.Execute(&ast.DisconnectStmt{}) }) + cov := tracedFullBuild(t, exec) + t.Logf("%s: %s", filepath.Base(mpr), coverageReport(cov, 50)) + if len(cov.Uncovered) > 0 { + t.Fail() + } +} diff --git a/mdl/executor/catalog_source_incremental_pedapp_test.go b/mdl/executor/catalog_source_incremental_pedapp_test.go index 0907b81011..7dd2943cfd 100644 --- a/mdl/executor/catalog_source_incremental_pedapp_test.go +++ b/mdl/executor/catalog_source_incremental_pedapp_test.go @@ -6,9 +6,7 @@ import ( "bytes" "context" "fmt" - "regexp" "sort" - "strconv" "strings" "sync" "testing" @@ -47,18 +45,6 @@ func sourceBuild(t *testing.T, exec *Executor, force bool) (map[string]string, s return rows, out.String() } -var reusedRe = regexp.MustCompile(`source: (\d+) reused, (\d+) described`) - -// reusedCount is how many rows the build carried over, from its progress line. -func reusedCount(out string) int { - m := reusedRe.FindStringSubmatch(out) - if m == nil { - return 0 - } - n, _ := strconv.Atoi(m[1]) - return n -} - func diffSourceRows(t *testing.T, step string, got, want map[string]string) { t.Helper() var keys []string @@ -211,7 +197,9 @@ end; t.Errorf("%s: %s was described again, want its row carried over", s.name, qn) } } - n := reusedCount(buildOut) + // A row is reused when it was not described again: carried over with + // its unit (#1804), or kept by the source plan (#1790). + n := len(got) - len(describedNow) if n > 0 { reusedAny = true } diff --git a/mdl/executor/cmd_catalog.go b/mdl/executor/cmd_catalog.go index a3c2e9bd49..80fd7981df 100644 --- a/mdl/executor/cmd_catalog.go +++ b/mdl/executor/cmd_catalog.go @@ -16,6 +16,7 @@ import ( "time" "github.com/mendixlabs/mxcli/mdl/ast" + "github.com/mendixlabs/mxcli/mdl/backend" "github.com/mendixlabs/mxcli/mdl/catalog" mdlerrors "github.com/mendixlabs/mxcli/mdl/errors" "github.com/mendixlabs/mxcli/model" @@ -520,6 +521,21 @@ func buildCatalog(ctx *ExecContext, full, isSource, communities bool, resolution // Build catalog builder := catalog.NewBuilder(cat, ctx.Backend) builder.SetFullMode(full) + // Rows of units nothing has changed under are copied from the cached + // catalog, which every save makes stale (#1804). FORCE builds everything, + // as does a cache another mxcli build or another mode wrote. + builder.SetFingerprint(sourceFingerprint()) + if !ctx.catalogForce { + builder.SetPreviousCatalog(getCachePath(ctx)) + } + // The rebuilt units are listed through a view that decodes only them. + if s, ok := ctx.Backend.(interface { + WithUnitScope(func(string) bool) backend.FullBackend + }); ok { + builder.SetScopedReader(func(keep func(string) bool) catalog.CatalogReader { + return s.WithUnitScope(keep) + }) + } if communities { builder.SetCommunitiesMode(true, resolution) } @@ -548,6 +564,9 @@ func buildCatalog(ctx *ExecContext, full, isSource, communities bool, resolution // Project-level widgets (widgets/*.mpk) are added by the catalog builder // itself by reading the filesystem. builder.SetBuiltinWidgetMetas(builtinWidgetMetas()) + if hook := catalogBuilderHook; hook != nil { + hook(builder) + } // Per-table lines are the only sign of life during a build that takes // minutes on a large app (18 on a 140MB one), so quiet mode moves them off // stdout rather than dropping them: `search -q --format names` promises one @@ -564,6 +583,9 @@ func buildCatalog(ctx *ExecContext, full, isSource, communities bool, resolution cat.Close() return mdlerrors.NewBackend("build catalog", err) } + if hook := catalogBuildHook; hook != nil { + hook(builder.Stats()) + } elapsed := time.Since(start) @@ -795,6 +817,14 @@ func sourceFingerprint() string { return fmt.Sprintf("%s|%d|%d", exe, fi.Size(), fi.ModTime().UnixNano()) } +// catalogBuildHook, when set, is told how each catalog build went. Tests use +// it to tell an incremental build from a full one. +var catalogBuildHook func(catalog.BuildStats) + +// catalogBuilderHook, when set, is handed each catalog builder before it +// builds. Tests use it to trace the reads a build makes (ref coverage). +var catalogBuilderHook func(*catalog.Builder) + // execRefreshCatalog handles REFRESH CATALOG [FULL] command (legacy signature). func execRefreshCatalog(ctx *ExecContext, full bool) error { return execRefreshCatalogStmt(ctx, &ast.RefreshCatalogStmt{Full: full, Source: false}) @@ -1101,16 +1131,17 @@ func preWarmCache(ctx *ExecContext) { } } - // Build microflow name lookup + // Build microflow name lookup. Headers, not decoded flows: a refresh that + // describes a few documents need not decode every flow first (#1804). ctx.Cache.microflowNames = make(map[model.ID]string) - mfs, _ := ctx.Backend.ListMicroflows() + mfs, _ := microflowHeaders(ctx) for _, mf := range mfs { ctx.Cache.microflowNames[mf.ID] = h.GetQualifiedName(mf.ContainerID, mf.Name) } // Build page name lookup ctx.Cache.pageNames = make(map[model.ID]string) - pgs, _ := ctx.Backend.ListPages() + pgs, _ := pageHeaders(ctx) for _, pg := range pgs { ctx.Cache.pageNames[pg.ID] = h.GetQualifiedName(pg.ContainerID, pg.Name) } diff --git a/mdl/executor/executor.go b/mdl/executor/executor.go index f16cf084dc..67c1e210d7 100644 --- a/mdl/executor/executor.go +++ b/mdl/executor/executor.go @@ -165,7 +165,7 @@ func getPageNames(ctx *ExecContext, h *ContainerHierarchy) map[model.ID]string { return ctx.Cache.pageNames } pageNames := make(map[model.ID]string) - pgs, err := ctx.Backend.ListPages() + pgs, err := pageHeaders(ctx) if err != nil { if ctx.Logger != nil { ctx.Logger.Warn("getPageNames: ListPages failed", "error", err) diff --git a/mdl/executor/helpers.go b/mdl/executor/helpers.go index bd16cc82f8..1dcca687e2 100644 --- a/mdl/executor/helpers.go +++ b/mdl/executor/helpers.go @@ -629,6 +629,25 @@ func microflowHeaders(ctx *ExecContext) ([]types.DocumentHeader, error) { return out, nil } +// pageHeaders lists every page's ID, container and name: without decoding a +// page on the modelsdk backend. +func pageHeaders(ctx *ExecContext) ([]types.DocumentHeader, error) { + if l, ok := ctx.Backend.(interface { + ListPageHeaders() ([]types.DocumentHeader, error) + }); ok { + return l.ListPageHeaders() + } + all, err := ctx.Backend.ListPages() + if err != nil { + return nil, err + } + out := make([]types.DocumentHeader, len(all)) + for i, pg := range all { + out[i] = types.DocumentHeader{ID: pg.ID, ContainerID: pg.ContainerID, Name: pg.Name, Excluded: pg.Excluded} + } + return out, nil +} + // buildMicroflowReturnTypes maps each stored microflow's qualified name to the // name of its return type ("Boolean", "String", …), or "" for a microflow that // returns nothing. diff --git a/mdl/exprcheck/hints/registry.go b/mdl/exprcheck/hints/registry.go index 00639e7654..de7f240230 100644 --- a/mdl/exprcheck/hints/registry.go +++ b/mdl/exprcheck/hints/registry.go @@ -195,6 +195,21 @@ var Registry = ®istry{byCode: map[string]Entry{ }, }, }, + "E015": { + Code: "E015", + Slug: "string-func-vs-empty", + Severity: SeverityWarning, + Trigger: "The result of a String function (trim, toLowerCase, toUpperCase, substring, replaceAll, toString) is compared with empty.", + WhyWrong: "These functions never return empty: for an empty input they return ''. Measured on the Mendix 10.18.5 runtime. So `= empty` is always false and `!= empty` always true, and the decision never takes the branch the author meant.", + HowToFix: "Compare with '' instead. trim(empty) is '', so `trim($v) != ''` also covers an empty $v — no separate `$v != empty` is needed.", + Examples: []ExampleFix{ + { + Wrong: "IF trim($Value) != empty THEN", + Right: "IF trim($Value) != '' THEN", + Note: "a blank or empty $Value took the 'has text' branch", + }, + }, + }, "E013": { Code: "E013", Slug: "bare-identifier-value", diff --git a/mdl/exprcheck/hints/registry_test.go b/mdl/exprcheck/hints/registry_test.go index 72958eac88..6d878c8fa5 100644 --- a/mdl/exprcheck/hints/registry_test.go +++ b/mdl/exprcheck/hints/registry_test.go @@ -5,7 +5,7 @@ package hints import "testing" func TestRegistry_AllCodesPresent(t *testing.T) { - want := []string{"E001", "E002", "E003", "E004", "E005", "E006", "E007", "E008", "E009", "E010", "E011", "E012"} + want := []string{"E001", "E002", "E003", "E004", "E005", "E006", "E007", "E008", "E009", "E010", "E011", "E012", "E015"} for _, c := range want { e, ok := Registry.Lookup(c) if !ok { diff --git a/mdl/exprcheck/parser.go b/mdl/exprcheck/parser.go index 5cb1228379..564ed18aa6 100644 --- a/mdl/exprcheck/parser.go +++ b/mdl/exprcheck/parser.go @@ -12,6 +12,7 @@ package exprcheck import ( + "fmt" "slices" "strings" @@ -63,6 +64,7 @@ func (p *parserImpl) Parse(src string, ctx Context) (RobustExpr, []Hint) { hs = append(hs, checkValueLiterals(expr, ctx)...) hs = append(hs, checkSlotKind(expr, ctx)...) hs = append(hs, checkBareIdentifierValue(expr, ctx)...) + hs = append(hs, checkStringFuncsComparedWithEmpty(expr, nil, ctx)...) return expr, hs } @@ -247,6 +249,160 @@ func parseCmp(s *Stream, ctx Context) (RobustExpr, []Hint) { return &BinExpr{Op: op, L: left, R: right}, hints } +// neverEmptyStringFuncs are the String built-ins whose result is never +// `empty`: given `empty`, the empty string or blanks, each returns the empty +// string or text. Measured on the +// Mendix 10.18.5 runtime with `mxcli test --local` (mendixlabs/mxcli#580), with +// the bare variable as the control that does compare equal to `empty`. Add a +// function only once it has been measured. +var neverEmptyStringFuncs = map[string]bool{ + "trim": true, + "toLowerCase": true, + "toUpperCase": true, + "substring": true, + "replaceAll": true, + "toString": true, +} + +// checkStringFuncsComparedWithEmpty emits E015 for `trim($v) != empty`: the +// result can never be `empty`, so the comparison is constant — `!=` always +// true, `=` always false. The model is valid and builds, which is why it is a +// warning; the logic is what is wrong, and the blank-check the author meant is +// trim($v) compared with the empty string literal, which also covers `empty` +// since trim(empty) is the empty string. +// +// It runs over the finished tree rather than in parseCmp because it needs the +// comparison's siblings: trim($v) != empty, and-ed with the same call compared +// with the empty string — the empty-and-blank idiom — has a dead first half, but the second decides it and the +// condition does what it says. Reporting that is noise on Studio Pro-authored +// projects, so a comparison is skipped when its own and/or chain also +// compares the same call with the empty string. chain holds those siblings. +func checkStringFuncsComparedWithEmpty(expr RobustExpr, chain []RobustExpr, ctx Context) []Hint { + switch n := expr.(type) { + case *BinExpr: + if n.Op == "AND" || n.Op == "OR" { + operands := flattenChain(n, n.Op, nil) + var hs []Hint + for _, o := range operands { + hs = append(hs, checkStringFuncsComparedWithEmpty(o, operands, ctx)...) + } + return hs + } + if n.Op == "=" || n.Op == "!=" { + if h := stringFuncVsEmptyHint(n, chain, ctx); h != nil { + return []Hint{*h} + } + } + return append(checkStringFuncsComparedWithEmpty(n.L, nil, ctx), + checkStringFuncsComparedWithEmpty(n.R, nil, ctx)...) + case *ParenExpr: + return checkStringFuncsComparedWithEmpty(n.Inner, chain, ctx) + case *UnaryExpr: + return checkStringFuncsComparedWithEmpty(n.Operand, nil, ctx) + case *IfThenElseExpr: + hs := checkStringFuncsComparedWithEmpty(n.Cond, nil, ctx) + hs = append(hs, checkStringFuncsComparedWithEmpty(n.Then, nil, ctx)...) + return append(hs, checkStringFuncsComparedWithEmpty(n.Else, nil, ctx)...) + case *CallExpr: + var hs []Hint + for _, a := range n.Args { + hs = append(hs, checkStringFuncsComparedWithEmpty(a, nil, ctx)...) + } + return hs + } + return nil +} + +// flattenChain returns the operands of a run of the same and/or operator, +// looking through parentheses: `a and (b and c)` is one chain of three. +func flattenChain(e RobustExpr, op string, out []RobustExpr) []RobustExpr { + if b, ok := unwrapParens(e).(*BinExpr); ok && b.Op == op { + return flattenChain(b.R, op, flattenChain(b.L, op, out)) + } + return append(out, e) +} + +func stringFuncVsEmptyHint(cmp *BinExpr, chain []RobustExpr, ctx Context) *Hint { + call := neverEmptyCallAgainstEmpty(cmp.L, cmp.R) + if call == nil { + call = neverEmptyCallAgainstEmpty(cmp.R, cmp.L) + } + if call == nil || chainComparesWithBlank(chain, call) { + return nil + } + always := "true" + if cmp.Op == "=" { + always = "false" + } + return &Hint{ + Code: "E015", Slug: "string-func-vs-empty", Severity: hints.SeverityWarning, + Where: hintsLocation(ctx, call.Pos()), + YouWrote: call.Name + "(...) " + cmp.Op + " empty", + Problem: call.Name + "() never returns empty — for an empty input it returns '' — " + + "so this comparison is always " + always + ".", + Fix: "Compare with '' instead: " + call.Name + "(...) " + cmp.Op + " ''", + } +} + +// chainComparesWithBlank reports whether some operand of chain compares a call +// structurally equal to call with the empty string literal. +func chainComparesWithBlank(chain []RobustExpr, call *CallExpr) bool { + key := exprKey(call) + for _, o := range chain { + b, ok := unwrapParens(o).(*BinExpr) + if !ok || (b.Op != "=" && b.Op != "!=") { + continue + } + for _, pair := range [][2]RobustExpr{{b.L, b.R}, {b.R, b.L}} { + lit, ok := unwrapParens(pair[1]).(*StringLit) + if ok && lit.Value == "" && exprKey(unwrapParens(pair[0])) == key { + return true + } + } + } + return false +} + +// exprKey renders the operand shapes a guard repeats — calls over variables, +// attribute paths and literals — position-free, so two spellings of the same +// call compare equal. Anything else keys as unique, which only ever means +// "not the same call" and so errs toward reporting. +func exprKey(e RobustExpr) string { + switch n := e.(type) { + case *CallExpr: + args := make([]string, len(n.Args)) + for i, a := range n.Args { + args[i] = exprKey(unwrapParens(a)) + } + return n.Name + "(" + strings.Join(args, ",") + ")" + case *VariableExpr: + return "$" + n.Name + case *AttributePathExpr: + if n.Predicates != nil { + return fmt.Sprintf("%p", n) + } + return "$" + n.Variable + "/" + strings.Join(n.Path, "/") + case *StringLit: + return "'" + n.Value + "'" + case *NumberLit: + return n.Value + } + return fmt.Sprintf("%p", e) +} + +// neverEmptyCallAgainstEmpty returns side as a never-empty String call when +// other is the `empty` keyword, and nil otherwise. +func neverEmptyCallAgainstEmpty(side, other RobustExpr) *CallExpr { + if _, ok := unwrapParens(other).(*EmptyExpr); !ok { + return nil + } + call, ok := unwrapParens(side).(*CallExpr) + if !ok || call.Qualified || !neverEmptyStringFuncs[call.Name] { + return nil + } + return call +} + // checkEnumComparedToString emits E001 for `$obj/EnumAttr = 'Value'`. // // This is the same defect checkStringLitVsSlot reports, found a different way. diff --git a/mdl/exprcheck/string_func_vs_empty_test.go b/mdl/exprcheck/string_func_vs_empty_test.go new file mode 100644 index 0000000000..91186068b3 --- /dev/null +++ b/mdl/exprcheck/string_func_vs_empty_test.go @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 + +// mendixlabs/mxcli#580 (comment, 2026-10-30): `if trim($Value) != empty` passed +// `mxcli check --references`, `exec` and `mx check`, and was always true — the +// blank value went down the "has text" branch and crashed later in the flow. +// +// The String built-ins below never return `empty`. Measured on the Mendix +// 10.18.5 runtime with `mxcli test --local`: for an input of `empty`, the empty +// string and blanks, each returned the empty string or text, never `empty`, +// while the control — the bare variable — did compare equal to `empty`. So +// comparing one of these results with `empty` is a constant condition. Only +// measured functions are listed; one is added when it has been measured, not +// assumed. +package exprcheck + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/exprcheck/hints" +) + +func TestStringFuncComparedWithEmpty(t *testing.T) { + ctx := Context{Microflow: "M.Probe", SlotPath: "Decision"} + cases := []struct { + src string + want bool // E015 expected + }{ + // The reported form, and its mirror and `=` variant. + {"trim($Value) != empty", true}, + {"empty != trim($Value)", true}, + {"trim($Value) = empty", true}, + // Every measured function. + {"toLowerCase($Value) = empty", true}, + {"toUpperCase($Value) = empty", true}, + {"substring($Value, 0) = empty", true}, + {"replaceAll($Value, 'x', 'y') = empty", true}, + {"toString($N) != empty", true}, + // Parenthesised still counts. + {"(trim($Value)) != empty", true}, + // Inside a larger condition. + {"$Flag and trim($Value) != empty", true}, + + // Controls: the correct forms, and an operand that CAN be empty. + {"trim($Value) != ''", false}, + {"$Value != empty", false}, + {"$Value != empty and trim($Value) != ''", false}, + // The redundant guard idiom does what it means: the `''` comparison + // decides it, so the dead `empty` half is not reported. It is in the + // Studio Pro-authored integration fixture (FeedbackModule.VAL_Feedback), + // which is how the first version of E015 was caught flagging it. + {"trim($Value) != empty and trim($Value) != ''", false}, + {"trim($Value) = empty or trim($Value) = ''", false}, + {"$Flag and trim($Value) != empty and trim($Value) != ''", false}, + // ...but only when the `''` comparison is on the SAME call. + {"trim($Value) != empty and trim($Other) != ''", true}, + {"trim($Value) != empty and toLowerCase($Value) != ''", true}, + // A guard in a different chain does not cover it. + {"(trim($Value) != empty or $Flag) and trim($Value) != ''", true}, + // An unmeasured function is left alone. + {"getKey($Value) = empty", false}, + // Ordering against empty is not this defect. + {"length($Value) > 0", false}, + } + for _, tc := range cases { + _, hs := NewParser().Parse(tc.src, ctx) + got := false + for _, h := range hs { + if h.Code == "E015" { + got = true + if h.Severity != hints.SeverityWarning { + t.Errorf("%q: E015 severity %v, want warning", tc.src, h.Severity) + } + } + } + if got != tc.want { + t.Errorf("%q: E015 reported = %v, want %v (hints: %+v)", tc.src, got, tc.want, hs) + } + } +} diff --git a/modelsdk/mpr/reader_raw.go b/modelsdk/mpr/reader_raw.go index 73e8825413..9f830b0095 100644 --- a/modelsdk/mpr/reader_raw.go +++ b/modelsdk/mpr/reader_raw.go @@ -113,7 +113,13 @@ func (r *Reader) GetUnitTypes() (map[string]int, error) { // ListRawUnitsByType returns all raw units matching the given BSON type prefix. // Returns types.RawUnit values keyed off model.ID for cross-package consumers. func (r *Reader) ListRawUnitsByType(typePrefix string) ([]*types.RawUnit, error) { - units, err := r.listUnitsByType(typePrefix) + return r.ListRawUnitsByTypeWhere(typePrefix, nil) +} + +// ListRawUnitsByTypeWhere is ListRawUnitsByType for the units whose ID keep +// accepts (nil keeps all); on v2 the others' contents are never read. +func (r *Reader) ListRawUnitsByTypeWhere(typePrefix string, keep func(id string) bool) ([]*types.RawUnit, error) { + units, err := r.listUnitsByTypeWhere(typePrefix, keep) if err != nil { return nil, err } diff --git a/modelsdk/mpr/reader_units.go b/modelsdk/mpr/reader_units.go index 25f3523a71..058d45451e 100644 --- a/modelsdk/mpr/reader_units.go +++ b/modelsdk/mpr/reader_units.go @@ -200,6 +200,43 @@ func (r *Reader) ListUnitsByTypeInContainer(typeName, containerID string) ([]Uni return result, nil } +// ListUnitsByTypeWhere is ListUnitsByType for the units whose ID keep +// accepts. On v2 the others' contents are never read: a catalog build that +// rebuilds a few units reads only those (ako/mxcli#1804). +func (r *Reader) ListUnitsByTypeWhere(typeName string, keep func(id string) bool) ([]UnitRef, error) { + units, err := r.listUnitsByTypeWhere(typeName, keep) + if err != nil { + return nil, err + } + result := make([]UnitRef, len(units)) + for i, u := range units { + result[i] = UnitRef{ID: u.ID, ContainerID: u.ContainerID, Type: u.Type, Contents: u.Contents} + } + return result, nil +} + +// listUnitsByTypeWhere is listUnitsByType for the units keep accepts (nil +// keeps all), filtered before any v2 contents are read. +func (r *Reader) listUnitsByTypeWhere(typeName string, keep func(id string) bool) ([]rawUnit, error) { + if keep == nil { + return r.listUnitsByType(typeName) + } + if r.version == MPRVersionV2 { + return r.listUnitsByTypeV2Where(typeName, "", keep) + } + units, err := r.listUnitsByTypeV1(typeName) + if err != nil { + return nil, err + } + kept := units[:0] + for _, u := range units { + if keep(u.ID) { + kept = append(kept, u) + } + } + return kept, nil +} + // listUnitsByType returns all units of exactly the given storage type. An empty // typeName returns every unit. // @@ -243,6 +280,12 @@ func (r *Reader) listUnitsByTypeV2(typeName string) ([]rawUnit, error) { // listUnitsByTypeV2In is listUnitsByTypeV2 restricted to one container ("" for // any), filtered before any contents are read. func (r *Reader) listUnitsByTypeV2In(typeName, containerID string) ([]rawUnit, error) { + return r.listUnitsByTypeV2Where(typeName, containerID, nil) +} + +// listUnitsByTypeV2Where is listUnitsByTypeV2In for the units keep accepts +// (nil keeps all). +func (r *Reader) listUnitsByTypeV2Where(typeName, containerID string, keep func(id string) bool) ([]rawUnit, error) { if !r.unitCacheValid { if err := r.buildUnitCache(); err != nil { return nil, err @@ -255,6 +298,9 @@ func (r *Reader) listUnitsByTypeV2In(typeName, containerID string) ([]rawUnit, e if containerID != "" && cu.ContainerID != containerID { continue } + if keep != nil && !keep(cu.ID) { + continue + } if typeName == "" || cu.Type == typeName { contents, err := r.readMprContents(cu.ID) if err != nil { diff --git a/modelsdk/mprread/decode_count.go b/modelsdk/mprread/decode_count.go new file mode 100644 index 0000000000..de85267182 --- /dev/null +++ b/modelsdk/mprread/decode_count.go @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 + +package mprread + +import "sync/atomic" + +// unitsDecoded counts the units decoded into typed elements, process-wide. +// Decoding is most of what a document listing costs, so a test can hold a +// pass to a budget by counting it (ako/mxcli#1804: an incremental catalog +// build decodes the units it rebuilds, not the project). +var unitsDecoded atomic.Int64 + +// UnitsDecoded returns how many units have been decoded so far. +func UnitsDecoded() int64 { + return unitsDecoded.Load() +} diff --git a/modelsdk/mprread/generic.go b/modelsdk/mprread/generic.go index fa643e0843..7612d77f5a 100644 --- a/modelsdk/mprread/generic.go +++ b/modelsdk/mprread/generic.go @@ -76,6 +76,17 @@ func ListUnitsWithContainerWhere[T element.Element](r *mmpr.Reader, keep func(bs return decodeUnits[T](refs, typeName, keep) } +// ListUnitsWithContainerIn is ListUnitsWithContainer for the units whose ID +// keepID accepts, and reads no other unit's contents on v2 (ako/mxcli#1804). +func ListUnitsWithContainerIn[T element.Element](r *mmpr.Reader, keepID func(id string) bool) ([]Unit[T], error) { + typeName := bsonTypeName[T]() + refs, err := r.ListUnitsByTypeWhere(typeName, keepID) + if err != nil { + return nil, err + } + return decodeUnits[T](refs, typeName, nil) +} + // decodeUnits decodes the refs of typeName that keep accepts (nil keeps all). func decodeUnits[T element.Element](refs []mmpr.UnitRef, typeName string, keep func(bson.Raw) bool) ([]Unit[T], error) { dec := codec.NewDecoder(codec.DefaultRegistry) @@ -87,6 +98,7 @@ func decodeUnits[T element.Element](refs []mmpr.UnitRef, typeName string, keep f if keep != nil && !keep(bson.Raw(ref.Contents)) { continue } + unitsDecoded.Add(1) elem, err := dec.Decode(bson.Raw(ref.Contents)) if err != nil { return nil, fmt.Errorf("decode unit %s: %w", ref.ID, err) diff --git a/modelsdk/mprread/reader_documents.go b/modelsdk/mprread/reader_documents.go index 7462aada2a..7496eec7b4 100644 --- a/modelsdk/mprread/reader_documents.go +++ b/modelsdk/mprread/reader_documents.go @@ -57,6 +57,7 @@ func decodeOne[T element.Element](r *mmpr.Reader, unitID string) (T, error) { return zero, fmt.Errorf("read unit %s: %w", unitID, err) } dec := codec.NewDecoder(codec.DefaultRegistry) + unitsDecoded.Add(1) elem, err := dec.Decode(bson.Raw(raw)) if err != nil { return zero, fmt.Errorf("decode unit %s: %w", unitID, err) @@ -703,6 +704,7 @@ func ListModules(r *mmpr.Reader) ([]Unit[*genProj.Module], error) { if ref.Type != typeName { continue } + unitsDecoded.Add(1) elem, err := dec.Decode(bson.Raw(ref.Contents)) if err != nil { return nil, fmt.Errorf("decode module %s: %w", ref.ID, err)