diff --git a/.github/actions/conformance/action.yml b/.github/actions/conformance/action.yml index 60270d3..395b34a 100644 --- a/.github/actions/conformance/action.yml +++ b/.github/actions/conformance/action.yml @@ -1,13 +1,13 @@ -# Checks a codec against fspm-TF's conformance files, in a codec's own CI, once the codec has run: +# Checks a codec against encoding3d's conformance files, in a codec's own CI, once the codec has run: # -# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/fspm-TF/.github/actions/conformance@main) +# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/encoding3d/.github/actions/conformance@main) # with: # dumps: out/dumps # what the codec read: one JSON dump per conformance GLB, x.json for x.expected.json # written: out/written # what the codec wrote: the conformance scenes, as .glb files (optional) # -# Reading passes when every dump equals its expected values; writing, when every written file passes fspm-tf validate +# Reading passes when every dump equals its expected values; writing, when every written file passes encoding3d validate # and Khronos' glTF-Validator with no error. -name: fspm-TF conformance +name: encoding3d conformance description: Compare a codec's reading of the conformance files with their expected values, and validate what it wrote inputs: dumps: @@ -17,8 +17,8 @@ inputs: description: the folder of the .glb files the codec wrote for the conformance scenes (empty to skip) required: false default: "" - fspm-tf: - description: the path of an fspm-TF checkout (its conformance folder and Python package) + encoding3d: + description: the path of an encoding3d checkout (its conformance folder and Python package) required: false default: "." gltf-validator: @@ -31,16 +31,16 @@ runs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf + - name: Install openalea.encoding3d shell: bash - run: python -m pip install numpy jsonschema "${{ inputs.fspm-tf }}" + run: python -m pip install numpy jsonschema "${{ inputs.encoding3d }}" - name: Reading shell: bash - run: fspm-tf compare-all "${{ inputs.dumps }}" "${{ inputs.fspm-tf }}/conformance" - - name: Writing, fspm-TF validator + run: encoding3d compare-all "${{ inputs.dumps }}" "${{ inputs.encoding3d }}/conformance" + - name: Writing, encoding3d validator if: inputs.written != '' shell: bash - run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 fspm-tf validate + run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 encoding3d validate - name: Writing, Khronos' glTF-Validator if: inputs.written != '' && runner.os == 'Linux' shell: bash diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index dae2e39..50ec812 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,4 +1,4 @@ -# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid fspm-TF (our validator) +# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid encoding3d (our validator) name: Conformance on: @@ -17,16 +17,16 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests - run: python -m pip install numpy jsonschema pytest vtk . + - name: Install openalea.encoding3d (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests + run: python -m pip install numpy jsonschema pytest pyzmq vtk . - name: Tests, the ParaView reader's conversion included run: pytest bindings/python/test -v - name: The committed files are the generated ones run: | python conformance/generate.py git diff --exit-code -- conformance - - name: fspm-TF validator - run: fspm-tf validate conformance/*.glb conformance/growing_sequence/scene.fspm.json + - name: encoding3d validator + run: encoding3d validate conformance/*.glb conformance/growing_sequence/scene.fspm.json - name: Khronos glTF-Validator run: | curl -sSL -o validator.tar.xz "https://github.com/KhronosGroup/glTF-Validator/releases/download/${GLTF_VALIDATOR}/gltf_validator-${GLTF_VALIDATOR}-linux64.tar.xz" @@ -42,10 +42,14 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf and the documentation's tools - run: python -m pip install numpy jsonschema sphinx pydata-sphinx-theme myst-parser sphinx-favicon sphinx-copybutton . + - name: Install openalea.encoding3d and the documentation's tools + run: | + sudo apt-get install -y pandoc + python -m pip install numpy jsonschema pytest sphinx ".[doc]" + - name: The notebooks run + run: python -m pytest --nbmake doc/notebooks tutorials/01_encoding3d_alone - name: Build the documentation, warnings as errors - run: python -m sphinx -b html docs docs/_build -W --keep-going + run: python -m sphinx -b html doc doc/_build -W --keep-going # The reference codec through the action other codecs use: its dumps of the conformance files, and the files it writes python-codec-conformance: diff --git a/.github/workflows/cpp-codec.yml b/.github/workflows/cpp-codec.yml new file mode 100644 index 0000000..7023473 --- /dev/null +++ b/.github/workflows/cpp-codec.yml @@ -0,0 +1,51 @@ +# The C++ codec (bindings/cpp): its tests on Linux, macOS and Windows; the conformance check other codecs run (its +# dumps of the conformance files compared with their expected values, the files it writes validated); the coupling +# conformance sessions. No platform: CPlantBox's adapter (bindings/cpp/cplantbox), a proposal for CPlantBox, is tested +# by hand (bindings/cpp/cplantbox/README.md). OpenAlea's conda CI builds the Python package only. +name: C++ codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - name: Configure (nlohmann_json fetched) + run: cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build build --config Release + - name: Tests + run: ctest --test-dir build -C Release --output-on-failure + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dump and write the conformance scenes with the C++ codec + run: | + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target encoding3d_conformance_run + ./build/encoding3d_conformance_run conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + # the coupling conformance sessions (specification §9, §11): the C++ participant, over ZeroMQ (libzmq) and a folder + - name: The coupling conformance sessions + run: | + sudo apt-get install -y -q libzmq3-dev + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target encoding3d_coupling_conformance + python -m pip install numpy jsonschema pyzmq . + encoding3d couple-conformance conformance/coupling --participant "$PWD/build/encoding3d_coupling_conformance" + diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml new file mode 100644 index 0000000..f9e661e --- /dev/null +++ b/.github/workflows/java-codec.yml @@ -0,0 +1,53 @@ +# The Java codec (bindings/java): its tests on Linux, macOS and Windows; the conformance check other codecs run; the +# coupling conformance sessions. No platform: GroIMP's adapter and plugin (bindings/java/groimp, groimp-plugin), +# proposals for GroIMP, are tested by hand (their README.md). OpenAlea's conda CI builds the Python package only. +name: Java codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Tests, the conformance files included + run: mvn -B -f bindings/java/pom.xml verify + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Dump and write the conformance scenes with the Java codec + run: | + mvn -B -q -f bindings/java/pom.xml package -DskipTests dependency:copy-dependencies -DincludeScope=runtime + java -cp "bindings/java/target/encoding3d-0.1.0.jar:bindings/java/target/dependency/*" org.openalea.encoding3d.ConformanceRun conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + # the coupling conformance sessions (specification §9, §11): the Java participant, over ZeroMQ (JeroMQ) and a folder + - name: The coupling conformance sessions + run: | + python -m pip install pyzmq + encoding3d couple-conformance conformance/coupling --participant \ + "java -cp $PWD/bindings/java/target/encoding3d-0.1.0.jar:$PWD/bindings/java/target/dependency/* org.openalea.encoding3d.coupling.ConformanceParticipant" + diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml new file mode 100644 index 0000000..30fe852 --- /dev/null +++ b/.github/workflows/julia-codec.yml @@ -0,0 +1,86 @@ +# The Julia codec (bindings/julia/encoding3d): its tests, on the oldest supported Julia and the latest release, and the +# conformance check other codecs run (its dumps of the conformance files compared with their expected values, the files +# it writes validated by encoding3d validate and Khronos' glTF-Validator); its coupling participant. No platform: the +# package extensions for VirtualPlantLab (PlantGraphs, PlantGeomTurtle) and MTG.jl, proposals for those platforms, are +# tested by hand in their own projects (test/plantgraphs, test/multiscaletreegraph; bindings/julia/README.md). OpenAlea's +# conda CI builds the Python package only. +name: Julia codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: Julia ${{ matrix.julia-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + julia-version: ["1.10", "1"] + os: [ubuntu-latest] + include: + - julia-version: "1" + os: windows-latest + - julia-version: "1" + os: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.julia-version }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + with: + project: bindings/julia/encoding3d + - uses: julia-actions/julia-runtest@v1 + with: + project: bindings/julia/encoding3d + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - name: Dump and write the conformance scenes with the Julia codec + run: | + julia --project=bindings/julia/encoding3d -e 'using Pkg; Pkg.instantiate()' + julia --project=bindings/julia/encoding3d bindings/julia/conformance_run.jl conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + + # encoding3d.jl as a participant of coupling sessions run by the Python coordinator (specification §11), over ZeroMQ + # and over a folder + coupling: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: The Python coordinator + run: pip install -e . pytest pyzmq jsonschema + - name: A Julia project with encoding3d and ZMQ + run: | + mkdir -p coupling-env + julia --project=coupling-env -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.add("ZMQ")' + - name: The coupling tests, with the Julia participant + env: + ENCODING3D_JULIA: julia --project=coupling-env + run: pytest -v bindings/python/test/test_coupling.py + - name: The coupling conformance sessions, the Julia participant + run: | + julia --project=coupling-env -e 'using Pkg; Pkg.add("JSON")' + encoding3d couple-conformance conformance/coupling --participant \ + "julia --project=$PWD/coupling-env $PWD/bindings/julia/coupling_conformance.jl" diff --git a/.gitignore b/.gitignore index b92e6f0..64f4752 100644 --- a/.gitignore +++ b/.gitignore @@ -116,5 +116,8 @@ examples/**/outputs/ *.report.json # the documentation build, and its pages generated from spec/vocabulary -docs/_build/ -docs/_generated/ +doc/_build/ +doc/_generated/ + +# Julia: environments resolved locally (the codec is a library: no committed manifest) +bindings/julia/**/Manifest*.toml diff --git a/.readthedocs.yml b/.readthedocs.yml index d27053d..a160bb4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,4 +9,4 @@ conda: environment: conda/environment.yml sphinx: - configuration: docs/conf.py + configuration: doc/conf.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a918dc8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,99 @@ +# Changelog + +## Unreleased: frame views (specification §8.1, draft for 0.3) + +- Views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): named configurations + of layers, each a mesh's, segments' or grid's entities selected by `where` conditions, coloured by a variable (or a + solid colour), with a colour map, range, log scale, opacity, visibility, representation, and tubes for segments. +- `openalea.encoding3d.views`: reading, writing and checking views (the schema, and every name against a frame), the + selection, and the display options ParaView's plugin draws and saves back (`block_displays`, + `displays_from_blocks`, `layer_from_display`). +- The ParaView reader's **Time Unit** (`d` by default, `h`, `min`, `s`): the times ParaView shows, converted from the + frames' own unit; the files are unchanged. +- The ParaView plugin is one file, `paraview/plugin_encoding3d.py` (was `encoding3d_reader.py`), with a how-to, + `paraview/README.md`; no macros (`apply_encoding3d_views.py`, `save_encoding3d_views.py` and + `show_encoding3d_selection.py` removed, their work in the plugin). +- The ParaView reader: "Views" and "Configuration" (a drop-down), one block per layer holding only its entities, a + segments layer with a `tube` as tubes (one closed strip per segment); `frame_to_vtk(frame, views=, configuration=)`, + unchanged without views. The plugin draws each layer as its block when ParaView shows the reader (Apply), with its + own colour variable, colour map, range, log scale or solid colour, opacity and visibility (the representation's + block properties, editable in the MultiBlock Inspector), drawn again when the Configuration or the views file + changes; the reader's "Save Views" button (and "Save As") writes the display back into `views.json`. +- Selection by scale in ParaView: the plugin's filter "encoding3d Select at scale" (Filters → encoding3d) takes ParaView's + selection and grows the picked cells to their entity at its **Scale** (a drop-down of the frames' scales) with + everything it contains, in every layer of the reader, hidden ones included, following the picked entities through + the time steps, drawn as the layers; its buttons Isolate (the other items hidden), Show In Panel (a view of its + own) and Put Back (one step back), one selection over another. Each cell carries the ids of the entities + containing it (`.id`), each block its data structure and scales. + +## Unreleased: specification 0.3 (draft), coupling platforms + +- Lazy reading (`read(path, lazy=True)`): the file mapped in memory, its JSON parsed, each column and mesh decoded on + first access (`LazyColumns`, `LazyList`), `frame.close()` or a `with` block; used by the ParaView reader and by + replays (only the columns links use). +- The folder transport deletes each message once read, unless the session keeps them (`keepMessages`, + `--keep-messages`); every participant (Python, Julia, C++, Java) applies it, and the conformance sessions check + that the folder is left empty. + +- Specification 0.3: the frames say 0.3 (0.1 and 0.2 files stay valid); "Changes from 0.2"; the conformance files + regenerated (their expected values unchanged). +- Links between two data structures, through a mapping a participant provides (declared, sent in its updates): + `sum` shares each source's amount among its targets by weight, `mean` gives each target its sources' weighted mean; + replayed sequences provide their mappings; a replay declares every entity set and variable of its frames. +- Coupling conformance sessions (`conformance/coupling/`, written by its `generate.py`), `encoding3d couple-conformance`, + and a participant's program in each codec; partial frames (`update`) in every codec's dump. +- The C++ participant (`encoding3d/coupling.hpp`; ZeroMQ with libzmq) and the Java one (`org.openalea.encoding3d.coupling`; + ZeroMQ with JeroMQ, optional); partial frames in both codecs. +- CPlantBox's and GroIMP's logs (the adapter's tests, the plugin's headless run) replayed in coupling sessions. +- The coordinator's lookups by id vectorized, and the Python codec's enum and text columns encoded without per-row + loops: a step of 100,000 segments takes 99 ms instead of 230 ms. + +- Coupling sessions: platforms' models run together, each in its own process, through encoding3d. Partial frames + (`FSPM_scene.update`: the sender, removed entities); the messages (`spec/schema/coupling.schema.json`): a + participant's declaration (its entity sets and rights, `owns`, `refines`, `follows`; its outputs and inputs), links + (factors, units, a change of scale), turns and updates; stages; one shared structure with id ranges and removals + down the scales; two transports, ZeroMQ and a folder of messages; a recorded sequence as a participant (replay, + held or interpolated linearly). +- The reference coordinator and participant (`openalea.encoding3d.coupling`, `encoding3d couple`), unit conversion factors + (`ucum.conversion_factor`); encoding3d.jl's participant (`Participant`, `turns`, `send_update`; ZeroMQ through the + `encoding3dZMQExt` extension), tested against the coordinator over both transports. +- The Python codec refuses enum columns (labels) with missing values instead of writing an unnamed enum value. + +## Unreleased: specification 0.2 (draft) + +- Specification 0.2, additive (0.1 files stay valid), from the bindings' points (`spec/specification.md`, "Changes + from 0.1"): missing values in integer, text and boolean columns; an entity set's `scale` and ranks as base scale + positions; reading by content; position columns in their declared unit; a data structure's `labels`; the extras + key; `edge_type` `unknown`; the `sibling_order` column; parents in another complex; bearer relations' labels and names; + dates (`format`); sparse ids; the conformance files' status and the sources' precedence. +- Vocabulary 0.2: `Seed`, `mtg_index`, the new enumerations. +- The reference codec, the validator, the ParaView reader and the conformance files (a new scene, `missing_values`). +- The Julia, C++ and Java codecs on 0.2, and their adapters using it instead of their extras: MTG.jl (missing + attributes, dates, children's order, `mtg_index`), PlantGraphs (`unknown` edges, children's order), CPlantBox (a + `mixed` Organ set, segment ends in cm), GroIMP (missing fields, scales, labels); extras keys in + lower case (`virtualplantlab`, `cplantbox`, `groimp`). + +## Specification 0.1 (draft) + +- The specification (`spec/specification.md`), its JSON Schemas and the first vocabulary (41 standard names, the + labels with their Plant Ontology terms, the enumerations, a column of names per platform). +- `openalea.encoding3d`, the Python reference codec (`bindings/python/`): reading and writing frames and sequences, the + validator, display copies, the comparison with expected values, the `encoding3d` command line. +- The conformance files (`conformance/`), their expected values, and the GitHub action checking a codec against them. +- The ParaView reader plugin (`paraview/`). +- encoding3d.jl, the Julia codec (`bindings/julia/encoding3d/`): conformant reading and writing, sequences, and a + MultiScaleTreeGraph.jl package extension (`from_mtg`, `to_mtg`); its CI, `.github/workflows/julia-codec.yml`. +- The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter + (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph + with segment ends); its CI, `.github/workflows/cpp-codec.yml`. +- The Java codec (`bindings/java/`): conformant reading and writing, sequences; GroIMP's adapter (`bindings/java/groimp/`: + scales from refinement edges, topology, edge bits as relations, managed fields; graphs back into the model's node + classes); its CI, `.github/workflows/java-codec.yml`. +- GroIMP's plugin (`bindings/java/groimp-plugin/`): the 3D view exported as an encoding3d frame from GroIMP 3.0 (File > + Export, CLI `export3d`), its geometry meshed by GroIMP's glTF plugin, each triangle with its node; tested headless in + GroIMP 3.0.0 in CI. +- encoding3d.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry + (PlantGeomTurtle: `add_geometry!`), as package extensions. +- encoding3d.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `encoding3d_id`), and parents + in another complex kept (`encoding3d_parent`). +- The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..da16f1f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,8 @@ +# Code of conduct + +encoding3d is developed by several plant-modelling teams together. Everyone taking part (in issues, pull requests, +reviews, meetings) is expected to follow the +[Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/): +be welcoming and respectful, give and accept constructive criticism, and focus on what is best for the community. + +Unacceptable behaviour can be reported to the maintainers listed in `pyproject.toml`; reports are kept confidential. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..12b699c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing + +encoding3d is a format shared by several FSPM platforms. Contributions are welcome from each of them: to the +specification, to the vocabulary, to a codec, to the tools. + +## What goes where + +- **`spec/`**: the agreement between the platforms (the specification, its JSON Schemas, the vocabulary). A change + there is proposed as a pull request, with the conformance scene showing it (`conformance/generate.py` and the + files it writes), the validator's check and the reference codec's update. See + [Changing the format](doc/governance.md). +- **The vocabulary** (`spec/vocabulary/`): a new standard name comes with its definition, canonical unit and kind, + and is agreed by at least two platforms. Each platform fills its own column of names. +- **`bindings//`**: one codec per language, each with its own build, tests and release, maintained by its + team. A codec passes the conformance check (`conformance/README.md`) in its CI. +- **Adapters** (a platform's model to and from the tables) stay in the platforms' own repositories. + +## Developing the Python package + +```bash +mamba env create -f conda/environment.yml # the package, editable, with the tests' and the docs' tools +mamba activate encoding3d +pytest # the package's tests (bindings/python/test) +pytest --nbmake doc/notebooks # the documentation's notebooks run +python -m sphinx -b html doc doc/_build -W # the documentation, warnings as errors +python conformance/generate.py # after a change to the conformance scenes +``` + +Commit with explicit messages, one subject per commit, and open the pull request on the development branch. diff --git a/README.md b/README.md index 11e0f41..330afca 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# FSPM-TF - Functional Structural Plant Model Transmission Format - based on GLTF 2.0 +# encoding3d - functional-structural plant model scenes, based on glTF 2.0 An exchange format for whole functional-structural plant model (FSPM) scenes, across platforms and languages (openalea.metafspm, CPlantBox, GroIMP, VirtualPlantLab). It holds: @@ -8,16 +8,16 @@ An exchange format for whole functional-structural plant model (FSPM) scenes, ac - their geometry; - the environment's grids. -Each time step is one glTF file, and the variables share one vocabulary. It also aims to serve as an in-memory -interface for coupling components across platforms. +Each time step is one glTF file, and the variables share one vocabulary. It also runs models of several platforms +together, step by step (coupling sessions: a coordinator and each platform's participant, over ZeroMQ or a folder). -Status: specification 0.1, draft. +Status: specification 0.3, draft (0.1 and 0.2 files stay valid): 0.3 adds coupling platforms (§11). -- [The documentation](docs/index.md): for users (viewing, checking, Python) and for platform developers +- [The documentation](doc/index.md): for users (viewing, checking, Python) and for platform developers (implementing a codec, conformance, changing the format), built with Sphinx (`.readthedocs.yml`; locally: - `python -m sphinx -b html docs docs/_build`) + `python -m sphinx -b html doc doc/_build`) - [The specification](spec/specification.md) -- [The design note](docs/design_note.md): the review of existing formats and the decisions taken +- [The design note](doc/design_note.md): the review of existing formats and the decisions taken - [Implementations](implementations.md) ## The repository @@ -28,73 +28,85 @@ change there, released as a new version. Everything else implements, checks or d | Folder | Content | Discussed with the platforms | |---|---|---| | `spec/specification.md` | the specification: data model, glTF layout, sequences, conformance | **yes** | -| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | +| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension, of a sequence's index and views, and of a coupling session's messages | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | -| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it | no | -| `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | -| `docs/design_note.md` | how the decisions were reached | no | +| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `encoding3d compare-all` and the action `.github/actions/conformance`); `coupling/`: the coupling sessions a participant runs (`encoding3d couple-conformance`) | follows `spec/` | +| `bindings//` | each language's codec and coupling participant, with its own build, tests and release; `bindings/python/`: `openalea.encoding3d`, the reference (codec with lazy reading, validator, display copies, views, the coupling's coordinator, command line), which reads `spec/` and ships it; `bindings/julia/`: encoding3d.jl, the Julia codec, with a ZeroMQ extension and the MultiScaleTreeGraph.jl and VirtualPlantLab extensions (proposals); `bindings/cpp/`: the C++ codec, with CPlantBox's adapter (a proposal); `bindings/java/`: the Java codec, with GroIMP's adapter and plugin (proposals) | no | +| `paraview/` | the ParaView plugin (`plugin_encoding3d.py`), its how-to, and the script checking it in a `pvpython` | no | +| `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | ## The Python package ```bash -git clone https://github.com/openalea/fspm-TF.git -cd fspm-TF +git clone https://github.com/openalea/encoding3d.git +cd encoding3d mamba env create -f conda/environment.yml # numpy and jsonschema from conda-forge, then pip install -e . -mamba activate fspm_tf +mamba activate encoding3d pytest ``` ```python -from openalea import fspm_tf +from openalea import encoding3d -frame = fspm_tf.read("scene.glb") # data structures, entity sets as numpy columns, meshes -fspm_tf.write("copy.glb", frame) +frame = encoding3d.read("scene.glb") # data structures, entity sets as numpy columns, meshes +encoding3d.write("copy.glb", frame) ``` ```bash -fspm-tf validate scene.glb # or a sequence: fspm-tf validate run/scene.fspm.json -fspm-tf info scene.glb -fspm-tf display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView +encoding3d validate scene.glb # or a sequence: encoding3d validate run/scene.fspm.json +encoding3d info scene.glb +encoding3d display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView ``` ## Viewing in ParaView -`paraview/fspm_tf_reader.py` is a ParaView reader plugin (ParaView ≥ 5.10): +`paraview/plugin_encoding3d.py` is encoding3d's ParaView plugin (ParaView ≥ 5.11), the only file to load: the +*encoding3d reader* and the *encoding3d Select at scale* filter, no macros. `paraview/README.md` is the how-to. -1. In ParaView, open Tools → Manage Plugins → Load New…, and choose the file. Tick "Auto Load" to keep it loaded. -2. File → Open a `.glb` frame or a sequence's `scene.fspm.json`, and pick "fspm-TF reader". +1. In ParaView, open Tools → Manage Plugins..., click Load New ..., and choose the file. Tick "Auto Load" to keep it + loaded. +2. File → Open... a `.glb` frame or a sequence's `scene.fspm.json`, pick "encoding3d reader", and Apply. What it shows: - Each data structure is a block: its meshes, its segments as lines (for a Tube filter on `radius`) and its grid. +- With a `views.json` next to the frames, one block per layer of its configuration instead (a drop-down), drawn at + Apply as the layer says: its colour variable and colour map, range, opacity, visibility, tubes for roots, each + layer editable on its own in View → MultiBlock Inspector, and saved back with the reader's "Save Views" button. - Every variable of every scale is a data array named `.`, at full precision, with no display copies needed. A sequence's frames are the time steps. - Text columns such as labels become integer codes, `._code`; the names behind the codes are in the block's field data. +- Filters → encoding3d → encoding3d Select at scale grows picked cells to their plant, axis, organ ... in every layer; + its buttons isolate the selection, show it in a panel, and put it back. -The plugin finds `openalea.fspm_tf` in ParaView's Python if it is installed there, otherwise in this clone's `bindings/python/src/`. -It needs nothing beyond what ParaView ships (numpy). In `pvpython` scripts the reader is -`fspmTFreader(FileName=...)`, after `LoadPlugin("paraview/fspm_tf_reader.py", ns=globals())`. Checked with -ParaView 6.0.1: `pvpython paraview/check_in_paraview.py conformance` (also run by the tests when -`FSPM_TF_PVPYTHON` names a pvpython). +The plugin finds `openalea.encoding3d` in ParaView's Python if it is installed there, otherwise in this clone's +`bindings/python/src/`. It needs nothing beyond what ParaView ships (numpy). In `pvpython` scripts the reader is +`encoding3dreader(FileName=...)`, after `LoadPlugin("paraview/plugin_encoding3d.py", ns=globals())`. Checked with +ParaView 6.2.0: `pvpython paraview/check_in_paraview.py conformance` (also run by the tests when +`ENCODING3D_PVPYTHON` names a pvpython). ## For another platform -Two pieces of code make a platform read and write fspm-TF: +Two pieces of code make a platform read and write encoding3d: - **A codec** for its language: GLB files, property tables, sequences, the vocabulary. It depends on `spec/` only, so it lives here, in `bindings//` (`cpp/`, `java/`, `julia/`...), with its build files, its tests and its own release, maintained by the platform's team. A change to `spec/` then updates every codec in the same pull request, and the CI shows which languages follow. -- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.fspm_tf` for +- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.encoding3d` for metafspm's DataStructures). It depends on the platform, so it lives in the platform's repository. +**encoding3d depends on no platform.** Its requirements and its CI install none (not even metafspm). The adapters for +CPlantBox (`bindings/cpp/cplantbox/`), GroIMP (`bindings/java/groimp/`, `bindings/java/groimp-plugin/`), MTG.jl and +VirtualPlantLab (`bindings/julia/encoding3d/ext/`) are kept here only as **first proposals for those platforms' +teams**: built and tested by hand (each one's README), they move into the platforms once adopted. + Each codec runs the same conformance check in its own tests. It reads the conformance GLBs and compares what it -reads with their `*.expected.json`, and it runs `fspm-tf validate` on what it writes. The steps are in the +reads with their `*.expected.json`, and it runs `encoding3d validate` on what it writes. The steps are in the [specification, §9](spec/specification.md#9-conformance). Each platform has a column of `spec/vocabulary/standard_names.csv`, listing its own names for each standard name -(`;`-separated). The writers read their mapping from it (`openalea.fspm_tf.vocabulary.platform_names("metafspm")`), so +(`;`-separated). The writers read their mapping from it (`openalea.encoding3d.vocabulary.platform_names("metafspm")`), so the vocabulary changes without changing the platforms' code. diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt new file mode 100644 index 0000000..fc6ac56 --- /dev/null +++ b/bindings/cpp/CMakeLists.txt @@ -0,0 +1,96 @@ +# encoding3d, the C++ codec of encoding3d: a library (encoding3d::encoding3d), its tests, the conformance tool, and, when +# CPlantBox is given (ENCODING3D_CPLANTBOX_SOURCE_DIR and ENCODING3D_CPLANTBOX_LIBRARY), CPlantBox's adapter. +# +# cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +# +# Licence: GPL-3.0 (LICENSE), as CPlantBox; the rest of encoding3d is CeCILL-C. +cmake_minimum_required(VERSION 3.16) +project(encoding3d VERSION 0.1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +option(ENCODING3D_BUILD_TESTS "Build the tests and the conformance tool" ${PROJECT_IS_TOP_LEVEL}) +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder (its src/ holds structural/), for the adapter") +set(ENCODING3D_CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox), for the adapter") + +# The one dependency, header-only: nlohmann_json, found installed or fetched +find_package(nlohmann_json 3.11 QUIET) +if(NOT nlohmann_json_FOUND) + include(FetchContent) + FetchContent_Declare(nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz + URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa) + set(JSON_Install ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(nlohmann_json) +endif() + +add_library(encoding3d + src/model.cpp + src/glb.cpp + src/codec.cpp + src/sequence.cpp + src/dump.cpp + src/coupling.cpp) +add_library(encoding3d::encoding3d ALIAS encoding3d) +target_include_directories(encoding3d PUBLIC + $ + $) +target_link_libraries(encoding3d PUBLIC nlohmann_json::nlohmann_json) + +# The coupling's ZeroMQ transport, with libzmq when found (its folder transport needs nothing) +option(ENCODING3D_WITH_ZMQ "Build the coupling's ZeroMQ transport (libzmq)" ON) +if(ENCODING3D_WITH_ZMQ) + find_path(ZMQ_INCLUDE_DIR zmq.h) + find_library(ZMQ_LIBRARY NAMES zmq libzmq) + if(ZMQ_INCLUDE_DIR AND ZMQ_LIBRARY) + target_include_directories(encoding3d PRIVATE ${ZMQ_INCLUDE_DIR}) + target_link_libraries(encoding3d PRIVATE ${ZMQ_LIBRARY}) + target_compile_definitions(encoding3d PRIVATE ENCODING3D_WITH_ZMQ) + message(STATUS "encoding3d: the coupling's ZeroMQ transport, with ${ZMQ_LIBRARY}") + else() + message(STATUS "encoding3d: libzmq not found, the coupling's folder transport only") + endif() +endif() +target_compile_features(encoding3d PUBLIC cxx_std_17) +set_target_properties(encoding3d PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(MSVC) + target_compile_options(encoding3d PRIVATE /W4 /bigobj) +else() + target_compile_options(encoding3d PRIVATE -Wall -Wextra -Wpedantic) +endif() + +# Installed as a CMake package: find_package(encoding3d) then target_link_libraries(... encoding3d::encoding3d) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +install(TARGETS encoding3d EXPORT encoding3dTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/encoding3d) +install(EXPORT encoding3dTargets NAMESPACE encoding3d:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +configure_package_config_file(cmake/encoding3dConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake + COMPATIBILITY SameMinorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) + +if(ENCODING3D_BUILD_TESTS) + enable_testing() + set(ENCODING3D_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") + add_executable(encoding3d_conformance_run tools/conformance_run.cpp) + target_link_libraries(encoding3d_conformance_run PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_coupling_conformance tools/coupling_conformance.cpp) + target_link_libraries(encoding3d_coupling_conformance PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_tests test/test_encoding3d.cpp) + target_link_libraries(encoding3d_tests PRIVATE encoding3d::encoding3d) + add_test(NAME encoding3d_tests COMMAND encoding3d_tests "${ENCODING3D_CONFORMANCE}") +endif() + +# CPlantBox's adapter: built when CPlantBox's sources and library are given +if(ENCODING3D_CPLANTBOX_SOURCE_DIR AND ENCODING3D_CPLANTBOX_LIBRARY) + add_subdirectory(cplantbox) +endif() diff --git a/bindings/cpp/LICENSE b/bindings/cpp/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/bindings/cpp/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md new file mode 100644 index 0000000..d1ada2c --- /dev/null +++ b/bindings/cpp/README.md @@ -0,0 +1,24 @@ +# encoding3d (C++) + +The C++ codec of encoding3d (C++17, CMake): reading and writing frames and sequences (spec 0.3). In `cplantbox/` is +CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, +and turns a graph into a static CPlantBox root system. + +- `include/encoding3d/`: the data model (`model.hpp`) and the codec (`encoding3d.hpp`). +- `src/`: the GLB container, reading and writing, sequences, the conformance dump. +- `tools/conformance_run.cpp`: the codec's dumps of the conformance files and its written scenes, for + `.github/actions/conformance`. +- `test/`: the tests, the conformance files included. +- `cplantbox/`: the adapter (`encoding3d::cplantbox`), built when `ENCODING3D_CPLANTBOX_SOURCE_DIR` and + `ENCODING3D_CPLANTBOX_LIBRARY` are given. + +```bash +cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +``` + +`encoding3d/coupling.hpp` is a participant of coupling sessions (its ZeroMQ transport built when libzmq is found). + +Its one dependency is nlohmann_json (header-only, MIT), found installed or fetched. It installs as a CMake package: +`find_package(encoding3d)`, then `target_link_libraries(... encoding3d::encoding3d)`. See [the documentation](../../doc/cpp.md). + +Licence: GPL-3.0 (`LICENSE`), as CPlantBox; the rest of encoding3d is CeCILL-C. diff --git a/bindings/cpp/cmake/encoding3dConfig.cmake.in b/bindings/cpp/cmake/encoding3dConfig.cmake.in new file mode 100644 index 0000000..fd62ca3 --- /dev/null +++ b/bindings/cpp/cmake/encoding3dConfig.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(nlohmann_json 3.11) +include("${CMAKE_CURRENT_LIST_DIR}/encoding3dTargets.cmake") +check_required_components(encoding3d) diff --git a/bindings/cpp/cplantbox/CMakeLists.txt b/bindings/cpp/cplantbox/CMakeLists.txt new file mode 100644 index 0000000..4271763 --- /dev/null +++ b/bindings/cpp/cplantbox/CMakeLists.txt @@ -0,0 +1,28 @@ +# CPlantBox's adapter (encoding3d::cplantbox), on CPlantBox's built library and its sources' headers: +# +# cmake -S bindings/cpp -B build -DENCODING3D_CPLANTBOX_SOURCE_DIR=/path/to/CPlantBox \ +# -DENCODING3D_CPLANTBOX_LIBRARY=/path/to/build/src/libCPlantBox.a +# +# Status: a first proposal for CPlantBox's team, not a part of encoding3d (whose requirements and CI do not depend on it), +# built and tested by hand (README.md here); it moves into CPlantBox once its team adopts it. +set(CPLANTBOX_SOURCE "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/src") +add_library(CPlantBox::CPlantBox UNKNOWN IMPORTED) +set_target_properties(CPlantBox::CPlantBox PROPERTIES + IMPORTED_LOCATION "${ENCODING3D_CPLANTBOX_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CPLANTBOX_SOURCE}/structural;${CPLANTBOX_SOURCE}/functional;${CPLANTBOX_SOURCE}/visualisation;${CPLANTBOX_SOURCE}/external/tinyxml2;${CPLANTBOX_SOURCE}/external/eigen") + +add_library(encoding3d_cplantbox src/cplantbox.cpp) +add_library(encoding3d::cplantbox ALIAS encoding3d_cplantbox) +target_include_directories(encoding3d_cplantbox PUBLIC $) +target_link_libraries(encoding3d_cplantbox PUBLIC encoding3d::encoding3d CPlantBox::CPlantBox) +if(MSVC) + target_compile_definitions(encoding3d_cplantbox PRIVATE _USE_MATH_DEFINES NOMINMAX) +endif() + +if(ENCODING3D_BUILD_TESTS) + add_executable(encoding3d_cplantbox_tests test/test_cplantbox.cpp) + target_link_libraries(encoding3d_cplantbox_tests PRIVATE encoding3d::cplantbox) + add_test(NAME encoding3d_cplantbox_tests + COMMAND encoding3d_cplantbox_tests "${ENCODING3D_CONFORMANCE}" "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/modelparameter/structural" + "${CMAKE_CURRENT_BINARY_DIR}/cplantbox_out") +endif() diff --git a/bindings/cpp/cplantbox/README.md b/bindings/cpp/cplantbox/README.md new file mode 100644 index 0000000..dde6e68 --- /dev/null +++ b/bindings/cpp/cplantbox/README.md @@ -0,0 +1,45 @@ +# CPlantBox's adapter: a proposal for CPlantBox + +**Status: a first proposal, for CPlantBox's team to take over.** It lives here so that the C++ binding could be built +and tested before CPlantBox's team is contacted. Once they adopt it, it moves into CPlantBox, with its tests and CI. +encoding3d itself does not depend on it: the C++ codec builds without it, and encoding3d's CI does not run it. + +What it does (`encoding3d/cplantbox.hpp`, namespace `encoding3d::cplantbox`): + +- `to_data_structure`, `to_frame`: a CPlantBox organism (plant, organs, segments, their variables) as an encoding3d + graph data structure, its ids stable while the plant grows; +- `add_geometry`: the plant's geometry (CPlantBox's PlantVisualiser) as a mesh; +- `soil_grid`, `segments_in_soil`, `add_cell_values`, `add_segment_values`, `add_node_values`: the soil grid, which + cell each segment is in, and a solver's or CPlantBox's hydraulic values; +- `to_root_system`: any platform's graph back as a static CPlantBox root system. + +## Building and testing it by hand + +It needs CPlantBox's sources and its built library. The commit it was last tested with: +`17e1fab617e72fa5e2773dfee723083bdf515053`. + +```bash +# CPlantBox's library (no Python bindings, no PiafMunch) +git clone https://github.com/Plant-Root-Soil-Interactions-Modelling/CPlantBox.git cplantbox +git -C cplantbox checkout 17e1fab617e72fa5e2773dfee723083bdf515053 +cmake -S cplantbox -B cplantbox-build -DCMAKE_BUILD_TYPE=Release -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF \ + -DCPB_BUILD_SHARED=OFF -DCPB_NATIVE_CPU_TUNING=OFF +cmake --build cplantbox-build --target CPlantBox -j 4 + +# the codec and the adapter, from encoding3d's root +cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release -DENCODING3D_CPLANTBOX_SOURCE_DIR="$PWD/cplantbox" \ + -DENCODING3D_CPLANTBOX_LIBRARY="$PWD/cplantbox-build/src/libCPlantBox.a" +cmake --build build -j 4 +ctest --test-dir build --output-on-failure + +# what it wrote is valid encoding3d; its growing log replayed in a coupling session +encoding3d validate build/cplantbox/cplantbox_out/*.glb +ENCODING3D_CPLANTBOX_LOG=build/cplantbox/cplantbox_out/anagallis_log/scene.fspm.json \ + pytest bindings/python/test/test_coupling.py -k platform_log +``` + +The tests (`test/test_cplantbox.cpp`) export a simulated root system and a plant with its geometry, soil grid and +mapping, read them back, rebuild a root system from another platform's frame, and write a growing log (a frame a day +for six days). + +Licence: GPL-3.0, as CPlantBox. diff --git a/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp new file mode 100644 index 0000000..5c61dbb --- /dev/null +++ b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp @@ -0,0 +1,106 @@ +// CPlantBox's adapter: an organism (a Plant, a RootSystem, a MappedPlant) as an encoding3d graph, its leaves' and +// stems' geometry as a mesh, its soil grid and segment-to-cell mapping; and back, a graph (CPlantBox's, or another +// platform's) as a static CPlantBox root system. +// +// Status: a first proposal for CPlantBox's team to take over, not a part of encoding3d: encoding3d's requirements and CI do not depend on it; +// it is built and tested by hand (bindings/cpp/cplantbox/README.md), and moves into CPlantBox once its team adopts it. +// +// | encoding3d | CPlantBox | +// |----------------------------------|---------------------------------------------------------------------------| +// | Plant (rank 1), one row, id 0 | the organism | +// | Organ (rank 5, scale "mixed"), | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | +// | | plant, parent its parent organ (branch) | +// | SubOrgan (rank 6), id | its segments: complex its organ, parent the segment ending at its | +// | 2^32 + distal node id | proximal node (successor along an organ, branch at a lateral's base) | +// | positions (proximal_end_*, ...) | node coordinates, in cm (declared; readers convert) | +// | other variables | CPlantBox's names and units (cm, d), declared with their standard names | +// | extras["cplantbox"] | the id offsets, the organism's parameters (its XML) | +// +// CPlantBox has two scales below the plant: organs and segments. A frame with more scales (e.g. metafspm's Plant, +// Axis, SubOrgan) is read through the entity set holding the segment ends (proximal_end_*, distal_end_*) and the +// coarser entity set its complex points to, whatever their names and ranks (see to_root_system). +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/encoding3d.hpp" + +namespace CPlantBox { +class Organism; +class MappedPlant; +class MappedSegments; +class Plant; +} // namespace CPlantBox + +namespace encoding3d::cplantbox { + +/// The name of CPlantBox's entry in a data structure's extras, and of its column in the vocabulary. +inline constexpr const char *PLATFORM = "cplantbox"; +/// Organ ids are CPlantBox's organ ids plus this offset (the plant is 0). +inline constexpr std::int64_t ORGAN_ID_OFFSET = 1; +/// Segment ids are their distal node's id plus this offset: stable while the plant grows, distinct from organ ids. +inline constexpr std::int64_t SEGMENT_ID_OFFSET = std::int64_t(1) << 32; + +struct Options { + std::string id = "plant"; ///< the data structure's id + std::optional model; ///< e.g. the parameter file's name + bool parameters_in_extras = true; ///< the organism's parameters (XML) in extras, to rebuild it +}; + +/// The organism as a graph data structure: its plant, organs and segments, with their variables. +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options = {}); + +/// The organism at its simulation time (days) as a frame of one data structure. +Frame to_frame(const CPlantBox::Organism &organism, const Options &options = {}); + +/// Adds the plant's geometry (CPlantBox's PlantVisualiser: leaf blades and stems, triangles) to its data structure, as +/// a mesh whose vertices carry their segment, organ and plant rows. resolution: PlantVisualiser's geometry +/// resolution (points around a stem), leaf_resolution: points along a leaf. +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, + int resolution = 8, int leaf_resolution = 20); + +/// The rectangular soil grid of a MappedSegments (set by setRectangularGrid), as a regular_grid data structure of +/// `cells` (cell ids: C-order index of (x, y, z), z pointing up, metres). +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id = "soil", + const std::string &cells = "Voxel"); + +/// The segments' soil cells (MappedSegments::seg2cell) as a mapping from the plant's SubOrgan to the soil's cells, +/// weight 1. The grid must be set without cutting segments (cut = false), so that segments keep CPlantBox's numbering. +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name = "roots_in_soil"); + +/// Values to write with their declaration: e.g. a soil solver's water potential per cell, or CPlantBox's hydraulic +/// results per segment or per node (PlantHydraulicModel), declared with their units and standard names. +struct Values { + std::string name; ///< the column's name + std::vector values; ///< one per cell, segment or node, in CPlantBox's order + Variable variable; ///< its unit, standard name, kind, role +}; + +/// Adds values per soil cell (in CPlantBox's cell order: x fastest, then y, then z) to the soil grid written by +/// soil_grid, as columns of its cells (reordered to encoding3d's C order of x, y, z). +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values); + +/// Adds values per segment of the MappedSegments (e.g. PlantHydraulicModel's radial fluxes) to the plant's SubOrgan +/// set, each in the row of its segment. The segments must keep CPlantBox's numbering (a grid set without cutting). +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + +/// Adds values per node of the MappedSegments (e.g. PlantHydraulicModel's xylem potentials) to the plant's SubOrgan +/// set: each segment takes its distal node's value. +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + +/// A graph as a static CPlantBox root system (StaticRoot organs on a seed, as CPlantBox's StaticPlant builds from +/// RSML): its segments' ends as the organs' nodes, a lateral at its parent's node, each organ's radius the mean of its +/// segments'. The segments: the entity set holding the segment ends (proximal_end_* and distal_end_*, by standard +/// name or by CPlantBox's names); the organs: the entity set their complex points to. Units are converted from those +/// declared (m, cm, mm). Organs that are not roots (CPlantBox's organ_type, when written by CPlantBox) are skipped: +/// CPlantBox has static roots only. +std::shared_ptr to_root_system(const DataStructure &data_structure); + +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp new file mode 100644 index 0000000..53f8296 --- /dev/null +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -0,0 +1,541 @@ +#include "encoding3d/cplantbox.hpp" + +#include +#include +#include +#include +#include +#include + +#include "MappedOrganism.h" +#include "Organ.h" +#include "Organism.h" +#include "Plant.h" +#include "PlantVisualiser.h" +#include "Root.h" +#include "Seed.h" +#include "organparameter.h" +#include "rootparameter.h" +#include "tinyxml2.h" + +namespace encoding3d::cplantbox { + +namespace { + +constexpr double METRES_PER_CENTIMETRE = 0.01; + +Variable declared(const std::string &unit, const std::string &kind, const std::string &description, + std::optional standard_name = std::nullopt, const std::string &role = "state") { + Variable variable; + variable.unit = unit; + variable.kind = kind; + variable.description = description; + variable.standard_name = std::move(standard_name); + variable.role = role; + return variable; +} + +/// The vocabulary's label of a CPlantBox organ type: a root or a stem is a whole axis, a leaf an organ. +std::string organ_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_seed: return "Seed"; + case CPlantBox::Organism::ot_root: return "RootAxis"; + case CPlantBox::Organism::ot_stem: return "ShootAxis"; + case CPlantBox::Organism::ot_leaf: return "Leaf"; + default: return "Organ"; + } +} + +std::string segment_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_root: return "RootSegment"; + case CPlantBox::Organism::ot_stem: return "StemElement"; + case CPlantBox::Organism::ot_leaf: return "LeafElement"; + default: return "SubOrgan"; + } +} + +/// The organism's organs, seeds included, parents before their laterals. +std::vector> all_organs(const CPlantBox::Organism &organism) { + std::vector> organs; + std::set seen; + std::function &)> visit = [&](const std::shared_ptr &organ) { + if (!organ || seen.count(organ.get())) return; + seen.insert(organ.get()); + organs.push_back(organ); + for (int child = 0; child < organ->getNumberOfChildren(); ++child) visit(organ->getChild(child)); + }; + for (const auto &organ : organism.getOrgans(-1, true)) { // from each organ's root: its seed, or itself + auto base = organ; + while (base->getParent()) base = base->getParent(); + visit(base); + } + return organs; +} + +/// The organism's parameters as CPlantBox's XML (as Organism::writeParameters writes them, skipping the empty entries +/// a MappedPlant can leave in its parameter map, on which writeParameters fails). +std::string parameters_xml(const CPlantBox::Organism &organism) { + tinyxml2::XMLDocument document; + tinyxml2::XMLElement *parameters = document.NewElement("plant"); + for (int organ_type = 0; organ_type <= CPlantBox::Organism::ot_leaf; ++organ_type) + for (const auto ¶meter : organism.getOrganRandomParameter(organ_type)) + if (parameter && (organ_type == CPlantBox::Organism::ot_seed || parameter->subType > 0)) + parameters->InsertEndChild(parameter->writeXML(document, false)); + document.InsertEndChild(parameters); + tinyxml2::XMLPrinter printer; + document.Print(&printer); + return printer.CStr(); +} + +} // namespace + +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options) { + const auto organs = all_organs(organism); + + // Plant: the organism + EntitySet plant("Plant", 1); + plant.scale = "Plant"; + plant.set("id", std::vector{0}); + + // Organ: one row per organ + EntitySet organ_set("Organ", 5); + organ_set.scale = "mixed"; // whole root and stem axes, leaves, the seed + std::vector organ_ids, complexes, parents; + std::vector labels, edge_types; + std::vector organ_types, sub_types, orders, parent_node_indices, node_counts; + std::vector ages, lengths, radii, creation_times; + std::vector alive, active; + for (const auto &organ : organs) { + const auto parent = organ->getParent(); + organ_ids.push_back(ORGAN_ID_OFFSET + organ->getId()); + complexes.push_back(0); + parents.push_back(parent ? ORGAN_ID_OFFSET + parent->getId() : NO_ID); + edge_types.push_back(parent ? "branch" : "none"); + labels.push_back(organ_label(organ->organType())); + organ_types.push_back(organ->organType()); + sub_types.push_back(static_cast(organ->getParameter("subType"))); + orders.push_back(static_cast(organ->getParameter("order"))); + parent_node_indices.push_back(organ->getParentNI()); + node_counts.push_back(organ->getNumberOfNodes()); + ages.push_back(organ->getAge()); + lengths.push_back(organ->getLength(true)); + radii.push_back(organ->getParameter("radius")); + creation_times.push_back(organ->getNumberOfNodes() > 0 ? organ->getNodeCT(0) : std::nan("")); + alive.push_back(organ->isAlive()); + active.push_back(organ->isActive()); + } + organ_set.set("id", organ_ids).set("label", Column(labels)).set("complex", complexes).set("parent", parents) + .set("edge_type", Column(edge_types)).set("organType", organ_types).set("subType", sub_types) + .set("order", orders).set("age", ages).set("length", lengths).set("radius", radii) + .set("creationTime", creation_times).set("parentNI", parent_node_indices).set("numberOfNodes", node_counts) + .set("alive", alive).set("active", active); + organ_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type: 1 seed, 2 root, 3 stem, 4 leaf", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type: the organ's parameter set", {}, "setting")) + .declare("order", declared("1", "descriptor", "branching order (0: a base organ)", "branching_order")) + .declare("age", declared("d", "intensive", "the organ's age", "age")) + .declare("length", declared("cm", "extensive", "the organ's realized length", "length")) + .declare("radius", declared("cm", "intensive", "the organ's radius", "radius", "parameter")) + .declare("creationTime", declared("d", "descriptor", "the creation time of the organ's first node")) + .declare("parentNI", declared("1", "descriptor", "the node index, in its parent organ, where it emerges")) + .declare("numberOfNodes", declared("1", "descriptor", "the number of its nodes")) + .declare("alive", declared("1", "descriptor", "whether it is alive")) + .declare("active", declared("1", "descriptor", "whether it still grows")); + + // SubOrgan: one row per segment, its id its distal node's + std::map segment_ending_at; // node id -> segment id + for (const auto &organ : organs) + for (int node = 1; node < organ->getNumberOfNodes(); ++node) + segment_ending_at[organ->getNodeId(node)] = SEGMENT_ID_OFFSET + organ->getNodeId(node); + EntitySet segment_set("SubOrgan", 6); + segment_set.scale = "SubOrgan"; + std::vector segment_ids, segment_complexes, segment_parents; + std::vector segment_labels, segment_edge_types; + std::vector segment_organ_types, segment_sub_types; + std::vector x1, y1, z1, x2, y2, z2, segment_radii, segment_lengths, segment_creation_times; + for (std::size_t index = 0; index < organs.size(); ++index) { + const auto &organ = organs[index]; + const double radius = radii[index]; + for (int node = 1; node < organ->getNumberOfNodes(); ++node) { + const auto proximal = organ->getNode(node - 1), distal = organ->getNode(node); + segment_ids.push_back(SEGMENT_ID_OFFSET + organ->getNodeId(node)); + segment_complexes.push_back(organ_ids[index]); + const auto parent = segment_ending_at.find(organ->getNodeId(node - 1)); + segment_parents.push_back(parent == segment_ending_at.end() ? NO_ID : parent->second); + segment_edge_types.push_back(parent == segment_ending_at.end() ? "none" : node == 1 ? "branch" : "successor"); + segment_labels.push_back(segment_label(organ->organType())); + segment_organ_types.push_back(organ->organType()); + segment_sub_types.push_back(sub_types[index]); + x1.push_back(proximal.x); + y1.push_back(proximal.y); + z1.push_back(proximal.z); + x2.push_back(distal.x); + y2.push_back(distal.y); + z2.push_back(distal.z); + segment_radii.push_back(radius); + segment_lengths.push_back(distal.minus(proximal).length()); + segment_creation_times.push_back(organ->getNodeCT(node)); + } + } + segment_set.set("id", segment_ids).set("label", Column(segment_labels)).set("complex", segment_complexes) + .set("parent", segment_parents).set("edge_type", Column(segment_edge_types)) + .set("organType", segment_organ_types).set("subType", segment_sub_types) + .set("x1", x1).set("y1", y1).set("z1", z1).set("x2", x2).set("y2", y2).set("z2", z2) + .set("radius", segment_radii).set("length", segment_lengths).set("creationTime", segment_creation_times); + segment_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type of its organ", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type of its organ", {}, "setting")) + .declare("x1", declared("cm", "descriptor", "proximal node, x", "proximal_end_x")) + .declare("y1", declared("cm", "descriptor", "proximal node, y", "proximal_end_y")) + .declare("z1", declared("cm", "descriptor", "proximal node, z", "proximal_end_z")) + .declare("x2", declared("cm", "descriptor", "distal node, x", "distal_end_x")) + .declare("y2", declared("cm", "descriptor", "distal node, y", "distal_end_y")) + .declare("z2", declared("cm", "descriptor", "distal node, z", "distal_end_z")) + .declare("radius", declared("cm", "intensive", "the segment's radius (its organ's)", "radius", "parameter")) + .declare("length", declared("cm", "extensive", "the segment's length", "length", "derived")) + .declare("creationTime", declared("d", "descriptor", "the creation time of its distal node")); + + DataStructure data_structure(options.id, "graph"); + data_structure.platform = "CPlantBox"; + data_structure.model = options.model; + data_structure.entity_sets = {plant, organ_set, segment_set}; + json extras = {{"organ_id_offset", ORGAN_ID_OFFSET}, {"segment_id_offset", SEGMENT_ID_OFFSET}, + {"length_unit", "cm"}, {"time_unit", "d"}}; + if (options.parameters_in_extras) extras["parameters"] = parameters_xml(organism); + data_structure.extras = {{PLATFORM, extras}}; + return data_structure; +} + +Frame to_frame(const CPlantBox::Organism &organism, const Options &options) { + Frame frame; + frame.data_structures.push_back(to_data_structure(organism, options)); + frame.time = organism.getSimTime(); + frame.time_unit = "d"; + frame.writer = {{"platform", "CPlantBox"}, {"adapter", "encoding3d::cplantbox"}}; + frame.vocabulary_version = "0.1"; + return frame; +} + +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, int resolution, + int leaf_resolution) { + CPlantBox::PlantVisualiser visualiser(plant); + visualiser.SetGeometryResolution(resolution); + visualiser.SetLeafResolution(leaf_resolution); + visualiser.ComputeGeometry(); + const auto points = visualiser.GetGeometry(); // x, y, z per vertex, cm + const auto indices = visualiser.GetGeometryIndices(); + const auto node_ids = visualiser.GetGeometryNodeIds(); // the node each vertex belongs to + const auto &segments = data_structure.entity_set("SubOrgan"); + const auto &organs = data_structure.entity_set("Organ"); + std::map segment_row, organ_row; + const auto &segment_ids = segments.column("id").as(); + const auto &segment_complexes = segments.column("complex").as(); + for (std::size_t row = 0; row < segment_ids.size(); ++row) segment_row[segment_ids[row]] = std::int64_t(row); + const auto &organ_ids = organs.column("id").as(); + for (std::size_t row = 0; row < organ_ids.size(); ++row) organ_row[organ_ids[row]] = std::int64_t(row); + Mesh mesh; + mesh.name = "plant"; + mesh.positions.reserve(points.size()); + for (const double value : points) mesh.positions.push_back(value * METRES_PER_CENTIMETRE); + std::vector segment_rows, organ_rows, plant_rows; + for (std::size_t vertex = 0; vertex < mesh.vertex_count(); ++vertex) { + const auto node = vertex < node_ids.size() ? node_ids[vertex] : -1; + const auto found = segment_row.find(SEGMENT_ID_OFFSET + node); + if (found == segment_row.end()) { // a base node: no segment ends there + segment_rows.push_back(NO_ID); + organ_rows.push_back(NO_ID); + } else { + segment_rows.push_back(found->second); + organ_rows.push_back(organ_row.at(segment_complexes[static_cast(found->second)])); + } + plant_rows.push_back(0); + } + mesh.feature_rows = {{"SubOrgan", segment_rows}, {"Organ", organ_rows}, {"Plant", plant_rows}}; + mesh.indices = std::vector(indices.begin(), indices.end()); + data_structure.meshes.push_back(std::move(mesh)); +} + +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id, const std::string &cells) { + const auto &minimum = segments.minBound, &maximum = segments.maxBound, &resolution = segments.resolution; + const std::vector shape = {std::int64_t(resolution.x), std::int64_t(resolution.y), + std::int64_t(resolution.z)}; + if (shape[0] <= 0 || shape[1] <= 0 || shape[2] <= 0) throw Error("no rectangular grid set (setRectangularGrid)"); + Grid grid; + grid.shape = shape; + grid.origin = {minimum.x * METRES_PER_CENTIMETRE, minimum.y * METRES_PER_CENTIMETRE, minimum.z * METRES_PER_CENTIMETRE}; + grid.spacing = {(maximum.x - minimum.x) / resolution.x * METRES_PER_CENTIMETRE, + (maximum.y - minimum.y) / resolution.y * METRES_PER_CENTIMETRE, + (maximum.z - minimum.z) / resolution.z * METRES_PER_CENTIMETRE}; + grid.periodic = {false, false, false}; + grid.z_up = true; + grid.cells = cells; + EntitySet voxels(cells); + std::vector ids(static_cast(shape[0] * shape[1] * shape[2])); + std::iota(ids.begin(), ids.end(), 0); + voxels.set("id", ids); + DataStructure soil(id, "regular_grid"); + soil.platform = "CPlantBox"; + soil.grid = grid; + soil.entity_sets.push_back(voxels); + return soil; +} + +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name) { + const auto &resolution = segments.resolution; + const auto nx = std::int64_t(resolution.x), ny = std::int64_t(resolution.y), nz = std::int64_t(resolution.z); + const auto &segment_set = plant.entity_set("SubOrgan"); + const auto &ids = segment_set.column("id").as(); + const std::set known(ids.begin(), ids.end()); + std::vector sources, targets; + for (const auto &[segment, cell] : segments.seg2cell) { + if (cell < 0) continue; // above the soil + const auto id = SEGMENT_ID_OFFSET + segments.segments.at(static_cast(segment)).y; + if (!known.count(id)) + throw Error("segment " + std::to_string(segment) + " of the mapped segments is not in '" + plant.id + + "': set the grid without cutting segments"); + // CPlantBox's cell index has x fastest, then y, then z from the bottom: encoding3d's is C order of (x, y, z) + const std::int64_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + sources.push_back(id); + targets.push_back((i * ny + j) * nz + k); + } + Mapping mapping(name, {plant.id, "SubOrgan"}, {soil.id, soil.grid ? soil.grid->cells : "Voxel"}, "containment"); + mapping.set("source", sources).set("target", targets).set("weight", std::vector(sources.size(), 1.0)); + return mapping; +} + +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values) { + if (!soil.grid) throw Error("'" + soil.id + "' has no grid: write it with soil_grid"); + const auto &resolution = segments.resolution; + const auto nx = std::size_t(resolution.x), ny = std::size_t(resolution.y), nz = std::size_t(resolution.z); + auto &cells = soil.entity_set(soil.grid->cells); + for (const auto &item : values) { + if (item.values.size() != nx * ny * nz) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, the grid has " + + std::to_string(nx * ny * nz) + " cells"); + std::vector column(item.values.size()); + for (std::size_t cell = 0; cell < item.values.size(); ++cell) { // CPlantBox: x fastest; encoding3d: C order + const std::size_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + column[(i * ny + j) * nz + k] = item.values[cell]; + } + cells.set(item.name, column).declare(item.name, item.variable); + } +} + +namespace { + +/// The SubOrgan row of each segment of the MappedSegments (by its distal node's id), or throws. +std::vector segment_rows(const DataStructure &plant, const CPlantBox::MappedSegments &segments) { + const auto &set = plant.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = set.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + std::vector rows; + for (std::size_t segment = 0; segment < segments.segments.size(); ++segment) { + const auto found = row_of.find(SEGMENT_ID_OFFSET + segments.segments[segment].y); + if (found == row_of.end()) + throw Error("segment " + std::to_string(segment) + " is not in '" + plant.id + + "': set the grid without cutting segments"); + rows.push_back(found->second); + } + return rows; +} + +void add_per_segment(DataStructure &plant, const std::vector &rows, const Values &item, + const std::vector &per_segment) { + auto &set = plant.entity_set("SubOrgan"); + std::vector column(set.row_count(), std::nan("")); // segments outside the mapped ones: NaN + for (std::size_t segment = 0; segment < rows.size(); ++segment) column[rows[segment]] = per_segment[segment]; + set.set(item.name, column).declare(item.name, item.variable); +} + +} // namespace + +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != rows.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(rows.size()) + " segments"); + add_per_segment(plant, rows, item, item.values); + } +} + +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != segments.nodes.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(segments.nodes.size()) + " nodes"); + std::vector per_segment; // each segment: its distal node's value + for (const auto &segment : segments.segments) per_segment.push_back(item.values.at(std::size_t(segment.y))); + add_per_segment(plant, rows, item, per_segment); + } +} + +// ── A graph as a static root system ────────────────────────────────────── + +namespace { + +/// The factor converting a length unit to centimetres. +double to_centimetres(const std::string &unit) { + if (unit == "m") return 100.; + if (unit == "cm") return 1.; + if (unit == "mm") return 0.1; + if (unit == "um") return 1e-4; + throw Error("length unit '" + unit + "' not converted to cm (m, cm, mm, um)"); +} + +/// The column of `set` holding the standard name (or, failing that, named `fallback`); nullptr if none. +const std::pair *find_variable(const EntitySet &set, const std::string &standard_name, + const std::string &fallback) { + for (const auto &item : set.columns) { + auto declared = set.variables.find(item.first); + if (declared != set.variables.end() && declared->second.standard_name == standard_name) return &item; + } + for (const auto &item : set.columns) + if (item.first == fallback) return &item; + return nullptr; +} + +std::vector as_doubles(const Column &column) { + return std::visit([](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) + return std::vector(values.begin(), values.end()); + else + throw Error("a numeric column expected"); + return std::vector{}; + }, column.values); +} + +std::vector centimetres(const EntitySet &set, const std::pair &item) { + auto values = as_doubles(item.second); + auto declared = set.variables.find(item.first); + const double factor = to_centimetres(declared == set.variables.end() ? "m" : declared->second.unit); + for (auto &value : values) value *= factor; + return values; +} + +} // namespace + +std::shared_ptr to_root_system(const DataStructure &data_structure) { + // The segments: the entity set with the segment ends; the organs: the one their complex points to + const EntitySet *segments = nullptr; + const char *ends[6][2] = {{"proximal_end_x", "x1"}, {"proximal_end_y", "y1"}, {"proximal_end_z", "z1"}, + {"distal_end_x", "x2"}, {"distal_end_y", "y2"}, {"distal_end_z", "z2"}}; + for (const auto &set : data_structure.entity_sets) { + bool all = set.has_column("id") && set.has_column("complex"); + for (const auto &end : ends) all = all && find_variable(set, end[0], end[1]); + if (all && (!segments || (set.rank && segments->rank && *set.rank > *segments->rank))) segments = &set; + } + if (!segments) throw Error("'" + data_structure.id + "': no entity set with segment ends and complexes"); + std::vector> coordinates; + for (const auto &end : ends) coordinates.push_back(centimetres(*segments, *find_variable(*segments, end[0], end[1]))); + const auto &segment_ids = segments->column("id").as(); + const auto &segment_complexes = segments->column("complex").as(); + std::vector segment_parents(segment_ids.size(), NO_ID); + if (segments->has_column("parent")) segment_parents = segments->column("parent").as(); + std::vector segment_radii(segment_ids.size(), 0.05); + if (auto radius = find_variable(*segments, "radius", "radius")) segment_radii = centimetres(*segments, *radius); + + const EntitySet *organs = nullptr; + const std::set complex_ids(segment_complexes.begin(), segment_complexes.end()); + for (const auto &set : data_structure.entity_sets) { + if (&set == segments || !set.has_column("id")) continue; + const auto &ids = set.column("id").as(); + if (std::all_of(complex_ids.begin(), complex_ids.end(), + [&](std::int64_t id) { return std::find(ids.begin(), ids.end(), id) != ids.end(); })) { + organs = &set; + break; + } + } + std::map organ_types; // CPlantBox's own frames: roots only + if (organs && organs->has_column("organType")) { + const auto &ids = organs->column("id").as(); + const auto types = as_doubles(organs->column("organType")); + for (std::size_t row = 0; row < ids.size(); ++row) organ_types[ids[row]] = int(types[row]); + } + + // Each organ's segments, in order along it (from the one whose parent is outside the organ) + std::map row_of; + for (std::size_t row = 0; row < segment_ids.size(); ++row) row_of[segment_ids[row]] = row; + std::map> rows_of_organ; + std::map next_in_organ; + std::vector organ_order; // organs, in the order of their first segment + for (std::size_t row = 0; row < segment_ids.size(); ++row) { + const auto organ = segment_complexes[row]; + if (organ_types.count(organ) && organ_types[organ] != CPlantBox::Organism::ot_root) continue; + if (!rows_of_organ.count(organ)) organ_order.push_back(organ); + rows_of_organ[organ].push_back(row); + const auto parent = row_of.find(segment_parents[row]); + if (parent != row_of.end() && segment_complexes[parent->second] == organ) next_in_organ[parent->second] = row; + } + + auto plant = std::make_shared(); + plant->initializeReader(); + auto seed = std::make_shared(plant); + plant->addOrgan(seed); + std::map> organ_of; + std::map, int>> node_of_segment; // row -> organ, node + // Parents first: an organ's first segment's parent belongs to an organ already built + std::vector pending = organ_order, deferred; + std::size_t progress = 1; + while (!pending.empty() && progress) { + progress = 0; + deferred.clear(); + for (const auto organ : pending) { + const auto &rows = rows_of_organ[organ]; + std::size_t first = rows.front(); + for (const auto row : rows) { + const auto parent = row_of.find(segment_parents[row]); + if (parent == row_of.end() || segment_complexes[parent->second] != organ) { + first = row; + break; + } + } + const auto parent_segment = row_of.find(segment_parents[first]); + std::shared_ptr parent_organ = seed; + int parent_node = 0; + if (parent_segment != row_of.end()) { + const auto found = node_of_segment.find(parent_segment->second); + if (found == node_of_segment.end()) { // its parent organ is not built yet + deferred.push_back(organ); + continue; + } + parent_organ = found->second.first; + parent_node = found->second.second; + } + std::vector chain = {first}; + while (next_in_organ.count(chain.back())) chain.push_back(next_in_organ[chain.back()]); + double length = 0., radius = 0.; + for (const auto row : chain) { + length += std::sqrt(std::pow(coordinates[3][row] - coordinates[0][row], 2) + + std::pow(coordinates[4][row] - coordinates[1][row], 2) + + std::pow(coordinates[5][row] - coordinates[2][row], 2)); + radius += segment_radii[row] / double(chain.size()); + } + auto parameter = std::make_shared(0, 0., length, std::vector{}, + 1e6, radius, 0., 1e6, false); + auto root = std::make_shared(plant->getOrganIndex(), parameter, length, parent_node); + root->setOrganism(plant); + if (parent_segment != row_of.end()) + root->addNode(parent_organ->getNode(parent_node), parent_organ->getNodeId(parent_node), 0.); + else + root->addNode(CPlantBox::Vector3d(coordinates[0][first], coordinates[1][first], coordinates[2][first]), 0.); + for (const auto row : chain) { + root->addNode(CPlantBox::Vector3d(coordinates[3][row], coordinates[4][row], coordinates[5][row]), 0.); + node_of_segment[row] = {root, root->getNumberOfNodes() - 1}; + } + parent_organ->addChild(root); + organ_of[organ] = root; + ++progress; + } + pending = deferred; + } + if (!pending.empty()) throw Error("'" + data_structure.id + "': organs whose parent segments are not found"); + return plant; +} + +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp new file mode 100644 index 0000000..a999492 --- /dev/null +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -0,0 +1,283 @@ +// CPlantBox's adapter: a simulated plant exported (organs, segments, variables, geometry, soil and mapping), written +// and read back; a static root system rebuilt from its own frame and from another platform's (the conformance root +// system, three scales). +// +// encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT +#include +#include +#include +#include +#include +#include +#include + +#include "encoding3d/cplantbox.hpp" + +#include "MappedOrganism.h" +#include "Plant.h" + +namespace filesystem = std::filesystem; +using namespace encoding3d; + +namespace { + +int failures = 0, checks = 0; + +void check(bool condition, const std::string &what); + +/// Checks that two JSON values are equal, printing their first differences when they are not. +void check_same(const json &actual, const json &expected, const std::string &what) { + const auto differences = compare(actual, expected); + for (std::size_t item = 0; item < std::min(differences.size(), 3); ++item) + std::cerr << " " << differences[item] << "\n"; + check(differences.empty(), what); +} + +void check(bool condition, const std::string &what) { + ++checks; + if (!condition) { + ++failures; + std::cerr << "FAILED: " << what << "\n"; + } +} + +std::shared_ptr grown_plant(const std::string ¶meters, double days) { + auto plant = std::make_shared(1); + plant->readParameters(parameters, "plant", true, false); + plant->initialize(false); + plant->simulate(days, false); + return plant; +} + +/// The column of an entity set as doubles. +std::vector values(const EntitySet &set, const std::string &name) { + return std::visit([](const auto &column) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) return std::vector(column.begin(), column.end()); + return std::vector{}; + }, set.column(name).values); +} + +/// The topology is valid: ids unique, complexes and parents found, edge types consistent. +void check_topology(const DataStructure &data_structure, const std::string &what) { + std::set all; + std::size_t total = 0; + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + all.insert(ids.begin(), ids.end()); + total += ids.size(); + } + check(all.size() == total, what + ": ids unique across entity sets"); + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + const std::set own(ids.begin(), ids.end()); + if (set.has_column("parent")) { + const auto &parents = set.column("parent").as(); + const auto &edges = set.column("edge_type").as(); + bool good = true; + for (std::size_t row = 0; row < parents.size(); ++row) + good = good && (parents[row] == NO_ID ? edges[row] == "none" : own.count(parents[row]) && edges[row] != "none"); + check(good, what + ": parents of " + set.name + " in their set, with their edge types"); + } + if (set.has_column("complex")) { + bool good = true; + for (const auto complex : set.column("complex").as()) good = good && (complex == NO_ID || all.count(complex)); + check(good, what + ": complexes of " + set.name + " found"); + } + } +} + +void test_root_system(const filesystem::path ¶meters, const filesystem::path &out) { + const auto plant = grown_plant((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), 30.); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + const Frame frame = cplantbox::to_frame(*plant, options); + const auto &data_structure = frame.data_structures[0]; + const auto &segments = data_structure.entity_set("SubOrgan"); + check(segments.row_count() == std::size_t(plant->getNumberOfSegments()), "one row per CPlantBox segment"); + check(data_structure.entity_set("Organ").row_count() >= plant->getOrgans().size(), "every organ (and the seed)"); + check_topology(data_structure, "root system"); + double total = 0.; + for (const auto length : values(segments, "length")) total += length; + double organs_total = 0.; + for (const auto length : plant->getParameter("length")) organs_total += length; + check(std::abs(total - organs_total) < 1e-9 * organs_total, "segment lengths sum to the organs' lengths"); + check(frame.time == 30., "the frame's time: the simulation time, days"); + + write_frame((out / "anagallis.glb").string(), frame); // written, read back the same + const auto read_back = read_frame((out / "anagallis.glb").string()); + check_same(frame_to_json(read_back), frame_to_json(frame), "the root system read back as written"); + + // Rebuilt as a static root system (CPlantBox has static roots only: the stem is left out), and exported again: + // the same root segments, their ends bit for bit + const auto rebuilt = cplantbox::to_root_system(read_back.data_structures[0]); + const auto organ_types = values(segments, "organType"); + const auto root_segments = std::count(organ_types.begin(), organ_types.end(), double(CPlantBox::Organism::ot_root)); + check(rebuilt->getNumberOfSegments() == root_segments, + "the static root system has every root segment (" + std::to_string(rebuilt->getNumberOfSegments()) + " of " + + std::to_string(root_segments) + ")"); + const auto again = cplantbox::to_data_structure(*rebuilt); + const auto &rebuilt_segments = again.entity_set("SubOrgan"); + double root_total = 0., rebuilt_total = 0.; + const auto lengths = values(segments, "length"); + for (std::size_t row = 0; row < lengths.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) root_total += lengths[row]; + for (const auto length : values(rebuilt_segments, "length")) rebuilt_total += length; + check(std::abs(rebuilt_total - root_total) < 1e-9 * root_total, "the static root system has the same root length"); + std::multiset ends, rebuilt_ends; // in cm both ways: bit for bit + for (const char *name : {"x2", "y2", "z2"}) { + const auto coordinates = values(segments, name); + for (std::size_t row = 0; row < coordinates.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) ends.insert(coordinates[row]); + for (const auto value : values(rebuilt_segments, name)) rebuilt_ends.insert(value); + } + check(ends == rebuilt_ends, "the static root system's segment ends are the same"); + check(data_structure.entity_set("Organ").scale == std::optional("mixed"), "the Organ set is mixed"); + check(segments.variables.at("x1").unit == "cm", "segment ends in CPlantBox's unit, cm"); +} + +void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "fspm2023.xml").string(), "plant", true, false); + plant->initialize(false); + plant->simulate(28., false); + auto data_structure = cplantbox::to_data_structure(*plant); + cplantbox::add_geometry(data_structure, plant); + check(data_structure.meshes.size() == 1 && data_structure.meshes[0].vertex_count() > 0, "a mesh of the shoot"); + const auto &rows = data_structure.meshes[0].feature_rows[0].second; + std::size_t with_segment = 0; + for (const auto row : rows) with_segment += row != NO_ID; + check(with_segment > rows.size() / 2, "most vertices carry their segment"); + check_topology(data_structure, "shoot and roots"); + + plant->setRectangularGrid(CPlantBox::Vector3d(-10, -10, -40), CPlantBox::Vector3d(10, 10, 0), + CPlantBox::Vector3d(4, 4, 8), false); + auto soil = cplantbox::soil_grid(*plant); + const auto mapping = cplantbox::segments_in_soil(*plant, data_structure, soil); + check(soil.entity_set("Voxel").row_count() == 4 * 4 * 8, "the soil grid's cells"); + check(mapping.row_count() > 0, "segments mapped to soil cells"); + bool inside = true; // each mapped segment's midpoint is in its cell + const auto &segments = data_structure.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = segments.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + const auto x1 = values(segments, "x1"), y1 = values(segments, "y1"), z1 = values(segments, "z1"); + const auto x2 = values(segments, "x2"), y2 = values(segments, "y2"), z2 = values(segments, "z2"); + const auto &grid = *soil.grid; + const auto &sources = mapping.column("source").as(); + const auto &targets = mapping.column("target").as(); + for (std::size_t item = 0; item < sources.size(); ++item) { + const auto row = row_of.at(sources[item]); + const double middle[3] = {(x1[row] + x2[row]) / 200, (y1[row] + y2[row]) / 200, // cm -> m + (z1[row] + z2[row]) / 200}; + std::int64_t index[3]; + for (int axis = 0; axis < 3; ++axis) + index[axis] = std::int64_t(std::floor((middle[axis] - grid.origin[axis]) / grid.spacing[axis])); + inside = inside && targets[item] == (index[0] * grid.shape[1] + index[1]) * grid.shape[2] + index[2]; + } + check(inside, "each mapped segment's midpoint lies in its cell (C order of x, y, z)"); + + // A soil solver's values per cell and CPlantBox's hydraulics per node and segment, each where it belongs + std::vector soil_potential, xylem_potential, radial_flux; + for (std::size_t cell = 0; cell < 4 * 4 * 8; ++cell) soil_potential.push_back(-100. - double(cell)); + for (const auto &node : plant->nodes) xylem_potential.push_back(node.z); // the node's own z + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) radial_flux.push_back(double(segment)); + Variable potential; + potential.unit = "cm[H2O]"; + potential.standard_name = "soil_water_potential"; + cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", soil_potential, potential}}); + potential.standard_name = "water_potential"; + Variable flux; + flux.unit = "cm3/d"; + flux.standard_name = "root_water_uptake"; + cplantbox::add_node_values(data_structure, *plant, {{"xylem_potential", xylem_potential, potential}}); + cplantbox::add_segment_values(data_structure, *plant, {{"radial_flux", radial_flux, flux}}); + const auto cell_values = values(soil.entity_set("Voxel"), "soil_water_potential"); + bool cells_right = true; + for (std::size_t item = 0; item < sources.size(); ++item) { // each mapped segment's cell: its own value + std::size_t segment = 0; + while (cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y != sources[item]) ++segment; + cells_right = cells_right && cell_values[std::size_t(targets[item])] == soil_potential[std::size_t(plant->seg2cell.at(int(segment)))]; + } + check(cells_right, "soil values reordered to the cells' C order"); + const auto xylem = values(data_structure.entity_set("SubOrgan"), "xylem_potential"); + const auto distal_z = values(data_structure.entity_set("SubOrgan"), "z2"); + bool nodes_right = true; + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) { + const auto row = row_of.at(cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y); + nodes_right = nodes_right && xylem[row] == distal_z[row]; + } + check(nodes_right, "each segment takes its distal node's value"); + + Frame frame; + frame.time = plant->getSimTime(); + frame.data_structures = {data_structure, soil}; + frame.mappings = {mapping}; + write_frame((out / "fspm2023.glb").string(), frame); + const auto read_back = read_frame((out / "fspm2023.glb").string()); + check_same(frame_to_json(read_back), frame_to_json(frame), "the plant, soil and mapping read back"); +} + +void test_another_platform(const filesystem::path &conformance) { + // The conformance root system: Plant, Axis and SubOrgan scales (metafspm's layout), segments in metres + const auto frame = read_frame((conformance / "root_system.glb").string()); + const auto &data_structure = frame.data_structures[0]; + const auto rebuilt = cplantbox::to_root_system(data_structure); + const auto &segments = data_structure.entity_set("SubOrgan"); + check(rebuilt->getNumberOfSegments() == int(segments.row_count()), "another platform's segments, one each"); + check(int(rebuilt->getOrgans().size()) == int(data_structure.entity_set("Axis").row_count()), + "another platform's axes as CPlantBox organs"); + double total = 0., rebuilt_total = 0.; + for (const auto length : values(segments, "length")) total += length * 100.; // m -> cm + for (const auto length : rebuilt->getParameter("length")) rebuilt_total += length; + check(std::abs(total - rebuilt_total) < 1e-6 * total, "another platform's root length, in cm"); +} + +} // namespace + +/// CPlantBox's log of a growing root system: a frame a day for six days (out/anagallis_log/), the sequence a coupling +/// session replays (bindings/python/test/test_coupling.py, ENCODING3D_CPLANTBOX_LOG). +void test_growing_log(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), "plant", true, false); + plant->initialize(false); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + options.parameters_in_extras = false; + std::vector frames; + std::size_t previous = 0; + bool growing = true; + for (int day = 0; day <= 6; ++day) { + if (day > 0) plant->simulate(1., false); + Frame frame = cplantbox::to_frame(*plant, options); + frame.step = day; + const auto segments = frame.data_structures[0].entity_set("SubOrgan").row_count(); + growing = growing && (day == 0 || segments > previous); + previous = segments; + frames.push_back(std::move(frame)); + } + check(growing, "the log: the root system grows every day"); + const auto index = read_index(write_sequence((out / "anagallis_log").string(), frames)); + check(index.frames.size() == 7, "the log: seven frames and their index"); +} + +int main(int argument_count, char **arguments) { + if (argument_count != 4) { + std::cerr << "usage: encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; + return 2; + } + const filesystem::path conformance = arguments[1], parameters = arguments[2], out = arguments[3]; + filesystem::create_directories(out); + for (auto test : {std::function([&] { test_root_system(parameters, out); }), + std::function([&] { test_plant_with_geometry_and_soil(parameters, out); }), + std::function([&] { test_another_platform(conformance); }), + std::function([&] { test_growing_log(parameters, out); })}) { + try { + test(); + } catch (const std::exception &error) { + check(false, std::string("exception: ") + error.what()); + } + } + std::cout << checks - failures << " of " << checks << " checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/bindings/cpp/include/encoding3d/coupling.hpp b/bindings/cpp/include/encoding3d/coupling.hpp new file mode 100644 index 0000000..8962546 --- /dev/null +++ b/bindings/cpp/include/encoding3d/coupling.hpp @@ -0,0 +1,93 @@ +// A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and +// requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is encoding3d's +// (`encoding3d couple`). +// +// encoding3d::coupling::Participant participant(declaration); // a JSON declaration; endpoint from the environment +// while (auto turn = participant.next_turn()) { // until the coordinator stops the session +// ... turn->frame: its entity sets, their entities, its inputs; the model from turn->time to turn->until +// participant.send_update(frame); // its outputs and topology changes +// } +// +// Transports: a folder ("files:path") always; ZeroMQ ("tcp://…", "ipc://…") when the library is built with libzmq +// (ENCODING3D_WITH_ZMQ). +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d::coupling { + +/// The version of the coupling protocol this library speaks. +inline constexpr const char *PROTOCOL_VERSION = "0.3"; +/// The environment variables a participant launched by the coordinator reads: the session's endpoint, its name. +inline constexpr const char *ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT"; +inline constexpr const char *PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT"; + +/// A message: its JSON header and, for turns and updates, a partial frame's GLB bytes. +struct Message { + json header; + std::optional> payload; +}; + +/// A participant's channel to the coordinator. +class Channel { + public: + virtual ~Channel() = default; + virtual void send(const json &header, const std::vector *payload = nullptr) = 0; + /// The next message; throws Error after `timeout` seconds without one (none: wait). + virtual Message receive(std::optional timeout) = 0; + /// As the session says (its welcome's keepMessages): keep the messages read, or delete them (the folder transport). + virtual void keep_messages(bool) {} +}; + +/// A participant's channel to `endpoint` ("files:path", or a ZeroMQ endpoint). +std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant); + +/// Whether this library speaks ZeroMQ (built with libzmq). +bool has_zeromq(); + +/// A turn: step (0: the initial state, phase "init"), the time it starts from and the time it reaches, and its frame, +/// the participant's inputs (frame.update->removed: the entities removed since its last turn). +struct Turn { + std::int64_t step = 0; + std::string phase; + double time = 0.0; + double until = 0.0; + Frame frame; +}; + +/// A participant in a session: joining it sends the declaration (and the links it proposes) and waits for the welcome. +/// The ids of the entities it creates are in [id_offset, id_offset + id_count). +class Participant { + public: + /// `declaration`: its JSON (spec/schema/coupling.schema.json); its participant name, if empty, and `endpoint`, if + /// empty, from the environment. + explicit Participant(json declaration, std::string endpoint = "", json links = json::array(), + std::optional timeout = std::nullopt); + /// The next turn, or none when the coordinator stopped the session (`stopped`: its reason). + std::optional next_turn(); + /// Sends the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the + /// entities it adds or changes); in frame.update->removed, the entities it removed. + void send_update(Frame frame); + /// Tells the coordinator this participant cannot go on (the session stops). + void fail(const std::string &message); + + std::string name; + std::int64_t id_offset = 0; + std::int64_t id_count = 0; + json session; + json couplings; + std::optional stopped; + + private: + std::unique_ptr channel_; + std::optional timeout_; + std::int64_t step_ = -1; +}; + +} // namespace encoding3d::coupling diff --git a/bindings/cpp/include/encoding3d/encoding3d.hpp b/bindings/cpp/include/encoding3d/encoding3d.hpp new file mode 100644 index 0000000..cddef80 --- /dev/null +++ b/bindings/cpp/include/encoding3d/encoding3d.hpp @@ -0,0 +1,77 @@ +// Reading and writing encoding3d files in C++: whole functional-structural plant model scenes (multiscale plant graphs, +// their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 +// binary (GLB) file per time step. The specification is spec/specification.md in the encoding3d repository. +// +// read_frame(path) -> Frame write_frame(path, frame) +// from_bytes(data) -> Frame to_bytes(frame) -> bytes +// read_index(path), read_sequence(path) write_sequence(directory, frames) +// frame_to_json(frame) the layout of the conformance files' expected values +#pragma once + +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d { + +// ── The GLB container ───────────────────────────────────────────────────── + +/// The bytes of a GLB file from its JSON document and its binary buffer (both padded to 8 bytes). +std::vector pack(const json &document, const std::vector &binary); +/// The JSON document (keys in their order) and the binary buffer of a GLB file's bytes. +std::pair> unpack(const std::vector &data); + +// ── Frames ──────────────────────────────────────────────────────────────── + +/// The GLB file of a frame. +std::vector to_bytes(const Frame &frame); +/// The frame of a GLB file's bytes. +Frame from_bytes(const std::vector &data); +/// Writes a frame to the GLB file `path`. +void write_frame(const std::string &path, const Frame &frame); +/// The frame of the GLB file `path`. +Frame read_frame(const std::string &path); + +// ── Sequences ───────────────────────────────────────────────────────────── + +/// The name of a sequence's index file. +inline constexpr const char *INDEX_FILE = "scene.fspm.json"; + +/// One frame of a sequence's index. +struct IndexEntry { + double time = 0.0; + std::int64_t step = 0; + std::string file; + std::string path; ///< the file's absolute path (when read) +}; + +/// A sequence's index: its frames, in time order. +struct Index { + std::string version = SPECIFICATION_VERSION; + std::string time_unit = "d"; + std::vector frames; +}; + +/// Writes the index of a sequence. +void write_index(const std::string &path, const Index &index); +/// The index of a sequence, each frame with the absolute path of its file. +Index read_index(const std::string &path); +/// Writes frames to `directory`, one GLB each (name_000000.glb, ...), and their index; the index's path. +std::string write_sequence(const std::string &directory, const std::vector &frames, + const std::string &name = "frame"); +/// The frames of the sequence whose index is `path`. +std::vector read_sequence(const std::string &path); + +// ── The conformance files' layout ───────────────────────────────────────── + +/// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): +/// every table's columns as lists, NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. +json frame_to_json(const Frame &frame); +/// The differences between two JSON values, as `encoding3d compare` finds them: objects by key, arrays in order, +/// numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). +std::vector compare(const json &actual, const json &expected, const std::string &path = "$"); + +} // namespace encoding3d diff --git a/bindings/cpp/include/encoding3d/model.hpp b/bindings/cpp/include/encoding3d/model.hpp new file mode 100644 index 0000000..28813eb --- /dev/null +++ b/bindings/cpp/include/encoding3d/model.hpp @@ -0,0 +1,262 @@ +// The encoding3d data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +// +// Frame one time step of a scene +// DataStructure one plant population's graph, or one environment model's grid +// EntitySet the entities of one scale (or a grid's cells): a table of columns +// Relation an edge table between entity sets (cell adjacency, a branch's bearer, ...) +// Table the data structure's scalars: a table of one row +// Grid a regular or adaptive grid's layout +// Mesh triangles, each vertex carrying the row of its entity in one or several entity sets +// Mapping source entity <-> target entity weights between two data structures +// +// A column holds one value per row (numbers, booleans, text), or a fixed-size vector per row (`width` values per +// row, row after row). Missing values: NaN in float columns; elsewhere the column's `missing` mask (one flag per row, +// empty when none is missing). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace encoding3d { + +using json = nlohmann::ordered_json; + +/// The version of the specification this library reads and writes. +inline constexpr const char *SPECIFICATION_VERSION = "0.3"; +/// The id of no entity: the complex or parent of an entity that has none, the feature row of a vertex with none. +inline constexpr std::int64_t NO_ID = -1; +/// The values of edge_type: the root of its scale, MTG '<', MTG '+'. +inline const std::vector EDGE_TYPES = {"none", "successor", "branch", "unknown"}; +/// The vocabulary's base scales: an entity set's scale is one of them, or "mixed". +inline const std::vector BASE_SCALES = {"Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", + "SubOrgan", "Layer", "Cell", "Compartment", "Connection"}; + +/// What the specification forbids, or a file that is not encoding3d. +struct Error : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/// The values of a column: one vector per element type. Booleans are std::vector; text, labels and edge types +/// std::vector. +using ColumnValues = std::variant, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector>; + +/// A column: its values, `width` per row (1: one value per row; >= 2: a fixed-size vector per row), and which rows are +/// missing (`missing`, one flag per row, empty when none is; NaN in a float column). +struct Column { + ColumnValues values; + std::size_t width = 1; + std::vector missing; + + Column() : values(std::vector{}) {} + template Column(std::vector column_values, std::size_t width = 1) + : values(std::move(column_values)), width(width) {} + Column(std::initializer_list text) : values(std::vector(text.begin(), text.end())) {} + + /// The number of values (rows × width). + std::size_t size() const; + /// The number of rows. + std::size_t rows() const { return width == 0 ? 0 : size() / width; } + /// The values as a vector of T (throws Error when the column holds another type). + template const std::vector &as() const { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template std::vector &as() { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template bool holds() const { return std::holds_alternative>(values); } + /// Whether row `row` is missing (by the mask; a float's NaN is read as a value). + bool is_missing(std::size_t row) const { return !missing.empty() && missing[row]; } + bool has_missing() const; + /// Marks rows as missing (one flag per row); returns the column, for chaining. + Column &with_missing(std::vector rows) { + missing = std::move(rows); + return *this; + } + /// The column's type name in the conformance files' expected values: numpy's (float64, int64, bool...), or + /// "string". + std::string dtype() const; +}; + +/// The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind +/// (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance. +struct Variable { + std::string unit = "1"; + std::optional standard_name; + std::optional kind; + std::string role = "state"; + std::optional description; + std::optional minimum; + std::optional maximum; + std::vector references; + std::vector doi; + std::optional format; ///< a text column of dates or times: "date", "date-time" or "time" + + bool operator==(const Variable &other) const; +}; + +enum class TableKind { scalars, entity_set, relation, mapping }; + +/// Columns of equal length, in their order, and the declarations of its variables. +struct Table { + std::string name; + std::vector> columns; + std::map variables; + TableKind table_kind = TableKind::scalars; + + Table() = default; + explicit Table(std::string name, TableKind kind = TableKind::scalars) : name(std::move(name)), table_kind(kind) {} + + /// The number of rows (0 without columns). + std::size_t row_count() const; + bool has_column(const std::string &column_name) const; + const Column &column(const std::string &column_name) const; + Column &column(const std::string &column_name); + /// Sets the column `column_name`, replacing it or appending it; returns the table, for chaining. + Table &set(const std::string &column_name, Column values); + /// Declares the variable `column_name`. + Table &declare(const std::string &column_name, Variable variable); + /// The columns whose meaning the specification fixes, for this kind of table. + const std::vector &reserved() const; + bool is_reserved(const std::string &column_name) const; + /// The names of the variable columns (the columns but the reserved ones). + std::vector variable_names() const; + /// Throws Error when columns have different numbers of rows. + void check_lengths() const; +}; + +/// The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (no rank). +/// Reserved columns: id (int64, stable across frames, never reused), label (text), complex (the id of the containing +/// entity at a coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), +/// edge_type ("none", "successor", "branch"); an adaptive grid's cells: level and index. +struct EntitySet : Table { + std::optional rank; + std::optional scale; ///< a base scale (BASE_SCALES), or "mixed" + + EntitySet() : Table("", TableKind::entity_set) {} + explicit EntitySet(std::string name, std::optional rank = std::nullopt) + : Table(std::move(name), TableKind::entity_set), rank(rank) {} + /// The ids of its entities: its id column, or a grid's row numbers (from 0) when it has none. + std::vector ids() const; +}; + +/// An edge table: source and target entity ids (in the entity sets source_set and target_set), an optional id per +/// edge, an optional relation label per edge, and edge variables. +struct Relation : Table { + std::string source_set; + std::string target_set; + + Relation() : Table("", TableKind::relation) {} + Relation(std::string name, std::string source_set, std::string target_set) + : Table(std::move(name), TableKind::relation), source_set(std::move(source_set)), + target_set(std::move(target_set)) {} +}; + +/// Weights between the entities of two data structures: source and target ids, weight. source and target: (data +/// structure id, entity set name). +struct Mapping : Table { + std::pair source; + std::pair target; + std::optional method; + + Mapping() : Table("", TableKind::mapping) {} + Mapping(std::string name, std::pair source, std::pair target, + std::optional method = std::nullopt) + : Table(std::move(name), TableKind::mapping), source(std::move(source)), target(std::move(target)), + method(std::move(method)) {} +}; + +/// A grid's layout. Regular: shape cells of size spacing from origin, along axes, periodic per axis, z pointing up or +/// down (z_up). Adaptive: the same for its base level, refined up to max_level, its leaf cells (an entity set with +/// level and index columns) explicit. cells: the name of the cells' entity set. +struct Grid { + std::string kind = "regular"; + std::vector shape; + std::vector origin; + std::vector spacing; + std::vector axes = {"x", "y", "z"}; + std::vector periodic; + bool z_up = true; + std::int64_t max_level = 0; + std::string cells = "Cell"; +}; + +/// Triangles: positions (3 per vertex, in the data structure's frame), three consecutive vertices per triangle, or +/// indices (3 per triangle, from 0) into them; feature_rows: per entity set, the row (from 0) of each vertex's +/// entity in that set, NO_ID for none; display: per variable, its float copy per vertex, from the entity set +/// display_sources[variable]. +struct Mesh { + std::string name; + std::vector positions; + std::vector>> feature_rows; + std::optional> indices; + std::vector>> display; + std::map display_sources; + + std::size_t vertex_count() const { return positions.size() / 3; } +}; + +/// One plant population's graph, or one environment model's grid. origin: its frame in the scene, a 4×4 matrix row +/// by row. extras: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); +/// other readers keep them without interpreting them. +struct DataStructure { + std::string id; + std::string kind = "graph"; + std::vector entity_sets; + std::vector relations; + std::optional scalars; + std::optional grid; + std::vector meshes; + std::array origin = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + std::optional platform; + std::optional model; + json extras = json::object(); + std::vector> labels; ///< the platform's labels -> the vocabulary's + + DataStructure() = default; + explicit DataStructure(std::string id, std::string kind = "graph") : id(std::move(id)), kind(std::move(kind)) {} + const EntitySet &entity_set(const std::string &name) const; + EntitySet &entity_set(const std::string &name); +}; + +/// What makes a frame partial: a coupling session's message (specification §11). participant: its sender +/// ("coordinator" for a turn's inputs); its tables hold some rows and columns only. removed: (data structure id, +/// entity set name) -> the ids of the entities removed. +struct Update { + std::string participant; + std::map, std::vector> removed; +}; + +/// A scene at one time step (time in time_unit, step), or a parameter set. update: set for a partial frame, a coupling +/// session's message. unknown: the content of a read file this library does not interpret (other root extensions, +/// the root's extras), written back as it was. +struct Frame { + std::vector data_structures; + std::vector mappings; + double time = 0.0; + std::string time_unit = "d"; + std::int64_t step = 0; + json writer = json::object(); + std::optional vocabulary_version; + std::optional update; + json unknown = json::object(); + + const DataStructure &data_structure(const std::string &id) const; + DataStructure &data_structure(const std::string &id); +}; + +} // namespace encoding3d diff --git a/bindings/cpp/src/bytes.hpp b/bindings/cpp/src/bytes.hpp new file mode 100644 index 0000000..c0bead8 --- /dev/null +++ b/bindings/cpp/src/bytes.hpp @@ -0,0 +1,56 @@ +// Little-endian values in byte buffers, whatever the machine's byte order. +#pragma once + +#include +#include +#include +#include +#include + +#include "encoding3d/model.hpp" + +namespace encoding3d::detail { + +inline bool machine_is_little_endian() { + const std::uint16_t probe = 1; + std::uint8_t first; + std::memcpy(&first, &probe, 1); + return first == 1; +} + +/// Appends `count` values of type T, little-endian. +template void append_values(std::vector &bytes, const T *values, std::size_t count) { + const std::size_t start = bytes.size(); + bytes.resize(start + count * sizeof(T)); + if (count == 0) return; + std::memcpy(bytes.data() + start, values, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (std::size_t item = 0; item < count; ++item) + std::reverse(bytes.begin() + start + item * sizeof(T), bytes.begin() + start + (item + 1) * sizeof(T)); +} + +/// The `count` values of type T stored little-endian in `bytes` (of `available` bytes) from byte `offset`. +template +std::vector read_values(const std::uint8_t *bytes, std::size_t available, std::size_t count, + std::size_t offset = 0) { + if (offset + count * sizeof(T) > available) + throw Error("a buffer view holds " + std::to_string(available) + " bytes, " + + std::to_string(offset + count * sizeof(T)) + " needed"); + std::vector values(count); + if (count == 0) return values; + std::memcpy(values.data(), bytes + offset, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (auto &value : values) { + auto raw = reinterpret_cast(&value); + std::reverse(raw, raw + sizeof(T)); + } + return values; +} + +inline std::uint32_t read_word(const std::vector &data, std::size_t offset) { + return read_values(data.data(), data.size(), 1, offset)[0]; +} + +inline std::size_t padding(std::size_t length) { return (8 - length % 8) % 8; } + +} // namespace encoding3d::detail diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp new file mode 100644 index 0000000..9170adf --- /dev/null +++ b/bindings/cpp/src/codec.cpp @@ -0,0 +1,905 @@ +// Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), +// meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data +// structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, +// standard names, kinds and roles. +#include "encoding3d/encoding3d.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bytes.hpp" + +namespace encoding3d { + +namespace { + +const std::string EXTENSION = "FSPM_scene"; +const std::string METADATA = "EXT_structural_metadata"; +const std::string FEATURES = "EXT_mesh_features"; +const std::string EDGE_TYPE_ENUM = "FSPM_edge_type"; +const std::vector ENUM_COLUMNS = {"label", "edge_type", "relation"}; +const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index", + "sibling_order"}; +constexpr std::uint8_t BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing +constexpr int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; +constexpr int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; +constexpr std::int64_t MAX_FLOAT_FEATURE = std::int64_t(1) << 24; // feature rows are float vertex attributes +// The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about +// x, column-major). Readers of the tables use the scene's frame, not this node. +const std::vector Z_UP_TO_Y_UP = {1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1}; + +bool contains(const std::vector &names, const std::string &name) { + return std::find(names.begin(), names.end(), name) != names.end(); +} + +void check_identifier(const std::string &name, const std::string &what) { + static const std::regex identifier("^[a-zA-Z_][a-zA-Z0-9_]*$"); + if (!std::regex_match(name, identifier)) + throw Error(what + " '" + name + "' must be an identifier (letters, digits, '_'; not starting with a digit)"); +} + +template struct Component; +template <> struct Component { static constexpr const char *name = "INT8"; }; +template <> struct Component { static constexpr const char *name = "UINT8"; }; +template <> struct Component { static constexpr const char *name = "INT16"; }; +template <> struct Component { static constexpr const char *name = "UINT16"; }; +template <> struct Component { static constexpr const char *name = "INT32"; }; +template <> struct Component { static constexpr const char *name = "UINT32"; }; +template <> struct Component { static constexpr const char *name = "INT64"; }; +template <> struct Component { static constexpr const char *name = "UINT64"; }; +template <> struct Component { static constexpr const char *name = "FLOAT32"; }; +template <> struct Component { static constexpr const char *name = "FLOAT64"; }; + +template constexpr bool is_number_v = std::is_arithmetic_v && !std::is_same_v; + +// EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; +inf +// and -inf as the largest finite value and the next one after the most negative, declared in the property's extras +// (FSPM_infinity). +template T no_data_float() { return std::numeric_limits::lowest(); } +template T positive_stand_in() { return std::numeric_limits::max(); } +template T negative_stand_in() { + return std::nextafter(std::numeric_limits::lowest(), T(0)); +} + +/// A value of T the column does not hold: its minimum (signed) or maximum (unsigned) first, then inwards. +template T free_integer(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + const T first = std::is_signed_v ? std::numeric_limits::min() : std::numeric_limits::max(); + const T second = std::is_signed_v ? std::numeric_limits::max() : std::numeric_limits::min(); + if (!present.count(first)) return first; + if (!present.count(second)) return second; + T candidate = first; + while (present.count(candidate)) candidate = std::is_signed_v ? T(candidate + 1) : T(candidate - 1); + return candidate; +} + +/// A string the column does not hold: the empty string first, then NUL characters. +std::string free_string(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + std::string candidate; + while (present.count(candidate)) candidate.push_back('\0'); + return candidate; +} + +json get(const json &object, const std::string &key, json fallback = nullptr) { + if (object.is_object()) { + auto found = object.find(key); + if (found != object.end()) return *found; + } + return fallback; +} + +/// Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. +struct BufferBuilder { + std::vector data; + json views = json::array(); + + std::int64_t add_bytes(const std::vector &bytes, int target = 0) { + data.resize(data.size() + detail::padding(data.size()), 0); + json view = {{"buffer", 0}, {"byteOffset", data.size()}, {"byteLength", bytes.size()}}; + if (target != 0) view["target"] = target; + data.insert(data.end(), bytes.begin(), bytes.end()); + views.push_back(view); + return static_cast(views.size()) - 1; + } + template std::int64_t add(const std::vector &values, int target = 0) { + std::vector bytes; + detail::append_values(bytes, values.data(), values.size()); + return add_bytes(bytes, target); + } +}; + +json declaration_json(const Variable &variable) { + json declaration = {{"unit", variable.unit}, {"role", variable.role}}; + if (variable.standard_name) declaration["standardName"] = *variable.standard_name; + if (variable.kind) declaration["kind"] = *variable.kind; + if (variable.minimum) declaration["minimum"] = *variable.minimum; + if (variable.maximum) declaration["maximum"] = *variable.maximum; + if (!variable.references.empty()) declaration["references"] = variable.references; + if (!variable.doi.empty()) declaration["doi"] = variable.doi; + if (variable.format) declaration["format"] = *variable.format; + return declaration; +} + +std::pair> enum_codes(const std::string &name, const std::vector &values, + const std::string &class_id, json &schema, + std::string &value_type) { + if (name == "edge_type") { + std::vector codes; + for (const auto &value : values) { + auto found = std::find(EDGE_TYPES.begin(), EDGE_TYPES.end(), value); + if (found == EDGE_TYPES.end()) + throw Error("edge_type of '" + class_id + "': unknown value '" + value + + "' (allowed: none, successor, branch)"); + codes.push_back(static_cast(found - EDGE_TYPES.begin())); + } + value_type = "UINT8"; + return {EDGE_TYPE_ENUM, codes}; + } + std::set sorted(values.begin(), values.end()); + std::vector names(sorted.begin(), sorted.end()); + const std::string enum_name = class_id + "__" + name; + value_type = names.size() <= 256 ? "UINT8" : "UINT16"; + json enum_values = json::array(); + std::map code_of; + for (std::size_t code = 0; code < names.size(); ++code) { + enum_values.push_back({{"name", names[code]}, {"value", code}}); + code_of[names[code]] = static_cast(code); + } + schema["enums"][enum_name] = {{"valueType", value_type}, {"values", enum_values}}; + std::vector bytes; + for (const auto &value : values) { + const std::uint16_t code = code_of[value]; + if (value_type == "UINT8") { + const auto narrow = static_cast(code); + detail::append_values(bytes, &narrow, 1); + } else { + detail::append_values(bytes, &code, 1); + } + } + return {enum_name, bytes}; +} + +/// The class properties and property table values of a table's columns. +std::pair encode_table(const Table &table, const std::string &class_id, BufferBuilder &buffers, + json &schema) { + table.check_lengths(); + json properties = json::object(), stored = json::object(); + for (const auto &[name, column] : table.columns) { + check_identifier(name, "column of '" + table.name + "'"); + json property = json::object(); + auto declared = table.variables.find(name); + if (declared != table.variables.end() && declared->second.description) + property["description"] = *declared->second.description; + const bool reserved = table.is_reserved(name); + const std::string where = "column '" + name + "' of '" + table.name + "'"; + if (reserved && contains(ENUM_COLUMNS, name)) { + if (!column.holds()) throw Error(where + ": labels are text"); + std::string value_type; + auto [enum_name, codes] = enum_codes(name, column.as(), class_id, schema, value_type); + property["type"] = "ENUM"; + property["enumType"] = enum_name; + stored[name] = {{"values", buffers.add_bytes(codes)}}; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": text vectors are not supported"); + property["type"] = "STRING"; + std::vector strings = column.as(); + if (column.has_missing()) { // missing text: noData, a string the column lacks + const std::string no_data = free_string(strings, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < strings.size(); ++row) + if (column.missing[row]) strings[row] = no_data; + } + std::vector text; + std::vector offsets = {0}; + for (const auto &value : strings) { + text.insert(text.end(), value.begin(), value.end()); + offsets.push_back(text.size()); + } + json item = {{"values", buffers.add_bytes(text)}}; + if (offsets.back() < (std::uint64_t(1) << 32)) { + item["stringOffsets"] = buffers.add(std::vector(offsets.begin(), offsets.end())); + item["stringOffsetType"] = "UINT32"; + } else { + item["stringOffsets"] = buffers.add(offsets); + item["stringOffsetType"] = "UINT64"; + } + stored[name] = item; + } else if (column.holds() && column.has_missing()) { // booleans with missing values: UINT8 0, 1, 255 + property["type"] = "SCALAR"; + property["componentType"] = "UINT8"; + property["noData"] = BOOLEAN_NO_DATA; + property["extras"] = {{"FSPM_boolean", true}}; + const auto &values = column.as(); + std::vector codes(values.size()); + for (std::size_t row = 0; row < values.size(); ++row) + codes[row] = column.missing[row] ? BOOLEAN_NO_DATA : std::uint8_t(values[row]); + stored[name] = {{"values", buffers.add(codes)}}; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": boolean vectors are not supported"); + property["type"] = "BOOLEAN"; + const auto &values = column.as(); + std::vector packed((values.size() + 7) / 8, 0); + for (std::size_t row = 0; row < values.size(); ++row) + if (values[row]) packed[row / 8] |= static_cast(1u << (row % 8)); // bit order: little + stored[name] = {{"values", buffers.add_bytes(packed)}}; + } else { + std::visit([&](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (is_number_v) { + if constexpr (std::is_integral_v && !std::is_same_v) { + if (reserved && contains(ID_COLUMNS, name)) { // ids are INT64, whatever the platform's + property["type"] = "SCALAR"; + property["componentType"] = "INT64"; + if (column.width >= 2) { + property["array"] = true; + property["count"] = column.width; + } + if (name == "complex" || name == "parent") property["noData"] = NO_ID; + stored[name] = {{"values", buffers.add(std::vector(values.begin(), values.end()))}}; + return; + } + } + property["type"] = "SCALAR"; + property["componentType"] = Component::name; + if (column.width != 1) { + if (column.width < 2) throw Error(where + ": vectors have at least 2 components"); + property["array"] = true; + property["count"] = column.width; + } + std::vector data = values; + if constexpr (std::is_floating_point_v) { + for (std::size_t row = 0; row < column.missing.size(); ++row) // a missing float: NaN + if (column.missing[row]) data[row] = std::numeric_limits::quiet_NaN(); + const bool infinite = std::any_of(data.begin(), data.end(), [](T value) { return std::isinf(value); }); + if (infinite) { + const T positive = positive_stand_in(), negative = negative_stand_in(); + if (std::any_of(data.begin(), data.end(), + [&](T value) { return value == positive || value == negative; })) + throw Error(where + " holds both infinities and the values standing for them"); + property["extras"] = {{"FSPM_infinity", {{"positive", static_cast(positive)}, + {"negative", static_cast(negative)}}}}; + for (auto &value : data) + if (std::isinf(value)) value = value > 0 ? positive : negative; + } + if (std::any_of(data.begin(), data.end(), [](T value) { return std::isnan(value); })) { + const T no_data = no_data_float(); + if (std::any_of(data.begin(), data.end(), [&](T value) { return value == no_data; })) + throw Error(where + " holds both NaN and the value standing for NaN"); + property["noData"] = static_cast(no_data); + for (auto &value : data) + if (std::isnan(value)) value = no_data; + } + } else if (reserved && (name == "complex" || name == "parent")) { + property["noData"] = NO_ID; + } else if (column.has_missing()) { // missing integers: noData, a value the column lacks + const T no_data = free_integer(data, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < data.size(); ++row) + if (column.missing[row]) data[row] = no_data; + } + stored[name] = {{"values", buffers.add(data)}}; + } + }, column.values); + } + if (name == "id" && table.table_kind == TableKind::entity_set) property["required"] = true; + properties[name] = property; + } + return {properties, stored}; +} + +json grid_json(const Grid &grid) { + json entry = {{"kind", grid.kind}, {"shape", grid.shape}, {"origin", grid.origin}, {"spacing", grid.spacing}, + {"axes", grid.axes}, {"periodic", grid.periodic}, {"zUp", grid.z_up}, {"cells", grid.cells}}; + if (grid.kind == "adaptive") entry["maxLevel"] = grid.max_level; + return entry; +} + +std::string display_attribute(const std::string &name) { + std::string attribute = "_" + name; + std::transform(attribute.begin(), attribute.end(), attribute.begin(), + [](unsigned char character) { return static_cast(std::toupper(character)); }); + return attribute; +} + +json encode_mesh(const Mesh &mesh, const DataStructure &data_structure, const json &entry, BufferBuilder &buffers, + json &accessors) { + const std::size_t vertices = mesh.vertex_count(); + if (mesh.positions.size() != vertices * 3) throw Error("mesh '" + mesh.name + "': 3 coordinates per vertex"); + auto accessor = [&](auto values, const char *type, int component_type, int target, std::size_t count, + bool bounds = false) { + json item = {{"bufferView", buffers.add(values, target)}, {"componentType", component_type}, + {"count", count}, {"type", type}}; + if (bounds && count > 0) { + std::vector minimum(3, std::numeric_limits::infinity()), maximum(3, -minimum[0]); + for (std::size_t vertex = 0; vertex < count; ++vertex) + for (int axis = 0; axis < 3; ++axis) { + minimum[axis] = std::min(minimum[axis], static_cast(values[vertex * 3 + axis])); + maximum[axis] = std::max(maximum[axis], static_cast(values[vertex * 3 + axis])); + } + item["min"] = minimum; + item["max"] = maximum; + } + accessors.push_back(item); + return static_cast(accessors.size()) - 1; + }; + std::vector positions(mesh.positions.begin(), mesh.positions.end()); + json attributes = {{"POSITION", accessor(positions, "VEC3", FLOAT, ARRAY_BUFFER, vertices, true)}}; + std::map tables; + for (const auto &set_entry : entry["entitySets"]) + tables[set_entry["name"].get()] = set_entry["propertyTable"].get(); + json feature_ids = json::array(); + for (std::size_t set_index = 0; set_index < mesh.feature_rows.size(); ++set_index) { + const auto &[set_name, rows] = mesh.feature_rows[set_index]; + if (!tables.count(set_name)) + throw Error("mesh '" + mesh.name + "': no entity set '" + set_name + "' in '" + data_structure.id + "'"); + if (rows.size() != vertices) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' are not one per vertex"); + const auto count = static_cast(data_structure.entity_set(set_name).row_count()); + if (count >= MAX_FLOAT_FEATURE) + throw Error("mesh '" + mesh.name + "': too many entities in '" + set_name + "', feature IDs stop at 2^24"); + std::vector stored; + std::set present; + bool has_null = false; + for (const auto row : rows) { + if (row < NO_ID || row >= count) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' outside its rows"); + has_null = has_null || row == NO_ID; + if (row != NO_ID) present.insert(row); + stored.push_back(static_cast(row == NO_ID ? count : row)); // null: one past the last row + } + attributes["_FEATURE_ID_" + std::to_string(set_index)] = accessor(stored, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + json feature = {{"featureCount", present.size()}, {"attribute", set_index}, + {"propertyTable", tables[set_name]}, {"label", set_name}}; + if (has_null) feature["nullFeatureId"] = count; + feature_ids.push_back(feature); + } + for (const auto &[name, values] : mesh.display) { + if (values.size() != vertices) + throw Error("mesh '" + mesh.name + "': display copy '" + name + "' is not one value per vertex"); + attributes[display_attribute(name)] = accessor(values, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + } + json primitive = {{"attributes", attributes}, {"mode", 4}}; + if (mesh.indices) { + std::vector indices(mesh.indices->begin(), mesh.indices->end()); + primitive["indices"] = accessor(indices, "SCALAR", UNSIGNED_INT, ELEMENT_ARRAY_BUFFER, indices.size()); + } + if (!feature_ids.empty()) primitive["extensions"] = {{FEATURES, {{"featureIds", feature_ids}}}}; + return {{"name", mesh.name}, {"primitives", json::array({primitive})}}; +} + +bool is_identity(const std::array &origin) { + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) + if (origin[row * 4 + column] != (row == column ? 1.0 : 0.0)) return false; + return true; +} + +std::vector column_major(const std::array &origin) { + std::vector values(16); + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) values[column * 4 + row] = origin[row * 4 + column]; + return values; +} + +} // namespace + +// ── Writing ────────────────────────────────────────────────────────────── + +std::vector to_bytes(const Frame &frame) { + BufferBuilder buffers; + json edge_values = json::array(); + for (std::size_t value = 0; value < EDGE_TYPES.size(); ++value) + edge_values.push_back({{"name", EDGE_TYPES[value]}, {"value", value}}); + json schema = {{"id", "fspm_scene"}, {"classes", json::object()}, + {"enums", {{EDGE_TYPE_ENUM, {{"valueType", "UINT8"}, {"values", edge_values}}}}}}; + json tables = json::array(), variables = json::object(); + json document = {{"asset", {{"version", "2.0"}, {"generator", std::string("encoding3d (C++) ") + SPECIFICATION_VERSION}}}, + {"scene", 0}, {"scenes", json::array({{{"nodes", json::array({0})}}})}, + {"nodes", json::array({{{"name", "fspm_scene"}, {"matrix", Z_UP_TO_Y_UP}, + {"children", json::array()}}})}}; + json accessors = json::array(), meshes = json::array(); + + auto add_table = [&](const Table &table, const std::string &class_id) { + check_identifier(class_id, "class"); + auto [properties, stored] = encode_table(table, class_id, buffers, schema); + schema["classes"][class_id] = {{"name", table.name}, {"properties", properties}}; + tables.push_back({{"name", table.name}, {"class", class_id}, {"count", table.row_count()}, + {"properties", stored}}); + json declared = json::object(); + for (const auto &name : table.variable_names()) { + auto found = table.variables.find(name); + if (found != table.variables.end()) declared[name] = declaration_json(found->second); + } + if (!declared.empty()) variables[class_id] = declared; + return static_cast(tables.size()) - 1; + }; + + json data_structures = json::array(); + for (const auto &data_structure : frame.data_structures) { + check_identifier(data_structure.id, "data structure id"); + json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}}; + if (data_structure.platform) entry["platform"] = *data_structure.platform; + if (data_structure.model) entry["model"] = *data_structure.model; + entry["origin"] = column_major(data_structure.origin); // column-major, as glTF matrices + json sets = json::array(); + for (const auto &set : data_structure.entity_sets) { + json item = {{"name", set.name}, {"rank", nullptr}}; + if (set.rank) item["rank"] = *set.rank; + if (set.scale) item["scale"] = *set.scale; + item["propertyTable"] = add_table(set, data_structure.id + "__" + set.name); + sets.push_back(item); + } + entry["entitySets"] = sets; + if (!data_structure.relations.empty()) { + json relations = json::array(); + for (const auto &relation : data_structure.relations) + relations.push_back({{"name", relation.name}, {"sourceSet", relation.source_set}, + {"targetSet", relation.target_set}, + {"propertyTable", add_table(relation, data_structure.id + "__" + relation.name)}}); + entry["relations"] = relations; + } + if (data_structure.scalars) entry["scalars"] = add_table(*data_structure.scalars, data_structure.id + "__scalars"); + if (data_structure.grid) entry["grid"] = grid_json(*data_structure.grid); + if (!data_structure.labels.empty()) { + json labels = json::object(); + for (const auto &[label, standard] : data_structure.labels) labels[label] = standard; + entry["labels"] = labels; + } + if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + json node = {{"name", data_structure.id}}; + if (!is_identity(data_structure.origin)) node["matrix"] = entry["origin"]; + document["nodes"].push_back(node); + const auto node_index = document["nodes"].size() - 1; + document["nodes"][0]["children"].push_back(node_index); + entry["node"] = node_index; + if (!data_structure.meshes.empty()) { + document["nodes"][node_index]["children"] = json::array(); + entry["meshes"] = json::array(); + for (const auto &mesh : data_structure.meshes) { + meshes.push_back(encode_mesh(mesh, data_structure, entry, buffers, accessors)); + document["nodes"].push_back({{"name", mesh.name}, {"mesh", meshes.size() - 1}}); + document["nodes"][node_index]["children"].push_back(document["nodes"].size() - 1); + json mesh_entry = {{"name", mesh.name}, {"mesh", meshes.size() - 1}}; + if (!mesh.display.empty()) { + json display = json::array(); + for (const auto &item : mesh.display) { + auto source = mesh.display_sources.find(item.first); + display.push_back({{"attribute", display_attribute(item.first)}, {"variable", item.first}, + {"entitySet", source == mesh.display_sources.end() ? json(nullptr) + : json(source->second)}}); + } + mesh_entry["display"] = display; + } + entry["meshes"].push_back(mesh_entry); + } + } + data_structures.push_back(entry); + } + + json mappings = json::array(); + for (const auto &mapping : frame.mappings) { + check_identifier(mapping.name, "mapping name"); + json entry = {{"name", mapping.name}, + {"source", {{"dataStructure", mapping.source.first}, {"entitySet", mapping.source.second}}}, + {"target", {{"dataStructure", mapping.target.first}, {"entitySet", mapping.target.second}}}}; + if (mapping.method) entry["method"] = *mapping.method; + entry["propertyTable"] = add_table(mapping, "mapping__" + mapping.name); + mappings.push_back(entry); + } + + json update = nullptr; + if (frame.update) { + check_identifier(frame.update->participant, "participant"); + update = {{"participant", frame.update->participant}}; + if (!frame.update->removed.empty()) { + json removed = json::array(); + for (const auto &[key, ids] : frame.update->removed) { + Table table(key.second + "__removed"); + table.set("id", Column(ids)); + removed.push_back({{"dataStructure", key.first}, {"entitySet", key.second}, + {"propertyTable", add_table(table, key.first + "__" + key.second + "__removed")}}); + } + update["removed"] = removed; + } + } + + json scene = {{"version", SPECIFICATION_VERSION}, + {"frame", {{"time", frame.time}, {"timeUnit", frame.time_unit}, {"step", frame.step}}}}; + if (frame.vocabulary_version) scene["vocabulary"] = *frame.vocabulary_version; + if (!frame.writer.empty()) scene["writer"] = frame.writer; + if (!update.is_null()) scene["update"] = update; + scene["dataStructures"] = data_structures; + if (!mappings.empty()) scene["mappings"] = mappings; + if (!variables.empty()) scene["variables"] = variables; + + const json other_extensions = get(frame.unknown, "extensions", json::object()); + json extensions = {{METADATA, {{"schema", schema}, {"propertyTables", tables}}}, {EXTENSION, scene}}; + json used = json::array({METADATA, EXTENSION}); + if (!meshes.empty()) used.push_back(FEATURES); + for (const auto &item : other_extensions.items()) { + extensions[item.key()] = item.value(); + used.push_back(item.key()); + } + document["extensionsUsed"] = used; + document["extensions"] = extensions; + if (frame.unknown.is_object() && frame.unknown.contains("extras")) document["extras"] = frame.unknown["extras"]; + if (!meshes.empty()) { + document["meshes"] = meshes; + document["accessors"] = accessors; + } + if (!buffers.views.empty()) { + // the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + buffers.data.resize(buffers.data.size() + detail::padding(buffers.data.size()), 0); + document["buffers"] = json::array({{{"byteLength", buffers.data.size()}}}); + document["bufferViews"] = buffers.views; + } + return pack(document, buffers.data); +} + +void write_frame(const std::string &path, const Frame &frame) { + const auto bytes = to_bytes(frame); + std::ofstream file(path, std::ios::binary); + if (!file) throw Error("cannot write '" + path + "'"); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} + +// ── Reading ────────────────────────────────────────────────────────────── + +namespace { + +struct ViewBytes { + const std::uint8_t *data; + std::size_t size; +}; + +ViewBytes view_bytes(const json &document, const std::vector &binary, std::int64_t index) { + const json &view = document.at("bufferViews").at(static_cast(index)); + if (get(view, "buffer", 0).get() != 0) + throw Error("encoding3d files keep every buffer view in the GLB binary chunk"); + const auto start = get(view, "byteOffset", 0).get(); + const auto length = view.at("byteLength").get(); + if (start + length > binary.size()) throw Error("a buffer view runs past the binary chunk"); + return {binary.data() + start, length}; +} + +template std::vector read_view(const ViewBytes &bytes, std::size_t count, std::size_t offset = 0) { + return detail::read_values(bytes.data, bytes.size, count, offset); +} + +template +Column decode_scalar(const ViewBytes &raw, const json &property, std::size_t count, std::size_t width) { + std::vector values = read_view(raw, count * width); + if constexpr (std::is_floating_point_v) { + if (property.contains("noData")) { + const T no_data = static_cast(property["noData"].get()); + for (auto &value : values) + if (value == no_data) value = std::numeric_limits::quiet_NaN(); + } + const json infinity = get(get(property, "extras", json::object()), "FSPM_infinity"); + if (!infinity.is_null()) { + const T positive = static_cast(infinity["positive"].get()); + const T negative = static_cast(infinity["negative"].get()); + for (auto &value : values) { + if (value == positive) value = std::numeric_limits::infinity(); + else if (value == negative) value = -std::numeric_limits::infinity(); + } + } + } + return Column(std::move(values), width); +} + +Column decode_scalar_of(const std::string &component, const ViewBytes &raw, const json &property, std::size_t count, + std::size_t width) { + if (component == "INT8") return decode_scalar(raw, property, count, width); + if (component == "UINT8") return decode_scalar(raw, property, count, width); + if (component == "INT16") return decode_scalar(raw, property, count, width); + if (component == "UINT16") return decode_scalar(raw, property, count, width); + if (component == "INT32") return decode_scalar(raw, property, count, width); + if (component == "UINT32") return decode_scalar(raw, property, count, width); + if (component == "INT64") return decode_scalar(raw, property, count, width); + if (component == "UINT64") return decode_scalar(raw, property, count, width); + if (component == "FLOAT32") return decode_scalar(raw, property, count, width); + if (component == "FLOAT64") return decode_scalar(raw, property, count, width); + throw Error("component type " + component + " is not part of encoding3d 0.1"); +} + +std::vector read_unsigned(const std::string &type, const ViewBytes &raw, std::size_t count) { + std::vector values; + if (type == "UINT8") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT16") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT32") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT64") values = read_view(raw, count); + else throw Error("offsets or enum values of type " + type + " are not part of encoding3d 0.1"); + return values; +} + +void decode_table(const json &document, const std::vector &binary, const json &description, + const json &schema, const json &variables, Table &table) { + const std::string class_id = description.at("class").get(); + table.name = get(description, "name", class_id).get(); + const json &properties = schema.at("classes").at(class_id).at("properties"); + const auto count = description.at("count").get(); + for (const auto &item : description.at("properties").items()) { + const std::string &name = item.key(); + const json &stored = item.value(); + const json &property = properties.at(name); + const auto raw = view_bytes(document, binary, stored.at("values").get()); + const std::string type = property.at("type").get(); + Column column; + if (type == "ENUM") { + const json &enumeration = schema.at("enums").at(property.at("enumType").get()); + const auto codes = read_unsigned(get(enumeration, "valueType", "UINT16").get(), raw, count); + std::map name_of; + for (const auto &value : enumeration.at("values")) + name_of[value.at("value").get()] = value.at("name").get(); + std::vector labels; + for (auto code : codes) labels.push_back(name_of.at(code)); + column = Column(std::move(labels)); + } else if (type == "STRING") { + const auto offsets = + read_unsigned(get(stored, "stringOffsetType", "UINT32").get(), + view_bytes(document, binary, stored.at("stringOffsets").get()), count + 1); + std::vector text; + for (std::size_t row = 0; row < count; ++row) { + if (offsets[row + 1] > raw.size || offsets[row] > offsets[row + 1]) + throw Error("'" + class_id + "." + name + "': string offsets outside its values"); + text.emplace_back(reinterpret_cast(raw.data + offsets[row]), offsets[row + 1] - offsets[row]); + } + std::vector missing; + if (property.contains("noData")) { // missing text + const std::string no_data = property["noData"].get(); + missing.assign(count, false); + for (std::size_t row = 0; row < count; ++row) + if (text[row] == no_data) { + missing[row] = true; + text[row].clear(); + } + if (std::none_of(missing.begin(), missing.end(), [](bool flag) { return flag; })) missing.clear(); + } + column = Column(std::move(text)); + column.missing = std::move(missing); + } else if (type == "BOOLEAN") { + if (raw.size * 8 < count) throw Error("'" + class_id + "." + name + "': too few bits"); + std::vector values(count); + for (std::size_t row = 0; row < count; ++row) values[row] = (raw.data[row / 8] >> (row % 8)) & 1u; + column = Column(std::move(values)); + } else if (type == "SCALAR") { + const bool vector = get(property, "array", false).get(); + if (vector && !property.contains("count")) + throw Error("'" + class_id + "." + name + "': variable-length arrays are not part of encoding3d 0.1"); + const std::size_t width = vector ? property["count"].get() : 1; + const std::string component = property.at("componentType").get(); + column = decode_scalar_of(component, raw, property, count, width); + if (get(get(property, "extras", json::object()), "FSPM_boolean", false).get()) { + const auto codes = column.as(); // booleans with missing values + const auto no_data = get(property, "noData", BOOLEAN_NO_DATA).get(); + std::vector values(count), missing(count); + for (std::size_t row = 0; row < count; ++row) { + missing[row] = codes[row] == no_data; + values[row] = !missing[row] && codes[row] == 1; + } + column = Column(std::move(values)); + if (std::any_of(missing.begin(), missing.end(), [](bool flag) { return flag; })) column.missing = missing; + } else if (width == 1 && component.rfind("FLOAT", 0) != 0 && property.contains("noData") && + name != "complex" && name != "parent") { // missing integers + const double no_data = property["noData"].get(); + std::vector missing(count); + bool any = false; + for (std::size_t row = 0; row < count; ++row) { + const double value = std::visit([&](const auto &values) -> double { + using V = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v) return static_cast(values[row]); + else return 0.0; + }, column.values); + missing[row] = value == no_data; + any = any || missing[row]; + } + if (any) column.missing = missing; + } + } else { + throw Error("'" + class_id + "." + name + "': property type " + type + " is not part of encoding3d 0.1"); + } + table.columns.emplace_back(name, std::move(column)); + const json declared = get(get(variables, class_id, json::object()), name); + if (!declared.is_null() || property.contains("description")) { + Variable variable; + variable.unit = get(declared, "unit", "1").get(); + variable.role = get(declared, "role", "state").get(); + if (!get(declared, "standardName").is_null()) variable.standard_name = declared["standardName"].get(); + if (!get(declared, "kind").is_null()) variable.kind = declared["kind"].get(); + if (property.contains("description")) variable.description = property["description"].get(); + if (!get(declared, "minimum").is_null()) variable.minimum = declared["minimum"].get(); + if (!get(declared, "maximum").is_null()) variable.maximum = declared["maximum"].get(); + if (!get(declared, "format").is_null()) variable.format = declared["format"].get(); + for (const auto &reference : get(declared, "references", json::array())) + variable.references.push_back(reference.get()); + for (const auto &doi : get(declared, "doi", json::array())) variable.doi.push_back(doi.get()); + table.variables[name] = variable; + } + } +} + +template +std::vector accessor_values(const json &document, const std::vector &binary, std::int64_t index, + std::size_t &width) { + const json &accessor = document.at("accessors").at(static_cast(index)); + const std::string type = accessor.at("type").get(); + width = type == "SCALAR" ? 1 : type == "VEC2" ? 2 : type == "VEC3" ? 3 : type == "VEC4" ? 4 : 0; + if (width == 0) throw Error("accessor type " + type + " is not used by encoding3d"); + const auto view_index = accessor.at("bufferView").get(); + const auto raw = view_bytes(document, binary, view_index); + const json &view = document["bufferViews"][static_cast(view_index)]; + const auto count = accessor.at("count").get(); + const auto start = get(accessor, "byteOffset", 0).get(); + const auto component = accessor.at("componentType").get(); + auto read = [&](auto tag) { + using C = decltype(tag); + const std::size_t item_size = sizeof(C) * width; + const auto stride = get(view, "byteStride", item_size).template get(); + std::vector values; + values.reserve(count * width); + for (std::size_t item = 0; item < count; ++item) + for (auto value : read_view(raw, width, start + item * stride)) values.push_back(static_cast(value)); + return values; + }; + if (component == FLOAT) return read(float{}); + if (component == UNSIGNED_INT) return read(std::uint32_t{}); + if (component == UNSIGNED_SHORT) return read(std::uint16_t{}); + if (component == UNSIGNED_BYTE) return read(std::uint8_t{}); + throw Error("accessor component type " + std::to_string(component) + " is not used by encoding3d"); +} + +Mesh decode_mesh(const json &document, const std::vector &binary, const json &mesh_entry, + const std::map &tables) { + const json &primitive = document.at("meshes").at(mesh_entry.at("mesh").get()).at("primitives").at(0); + const json &attributes = primitive.at("attributes"); + Mesh mesh; + mesh.name = mesh_entry.at("name").get(); + std::size_t width = 0; + const auto positions = accessor_values(document, binary, attributes.at("POSITION").get(), width); + mesh.positions.assign(positions.begin(), positions.end()); + const json features = get(get(get(primitive, "extensions", json::object()), FEATURES, json::object()), "featureIds", + json::array()); + for (const auto &feature : features) { + const std::string attribute = "_FEATURE_ID_" + std::to_string(feature.at("attribute").get()); + auto rows = accessor_values(document, binary, attributes.at(attribute).get(), width); + if (feature.contains("nullFeatureId")) { + const auto null_row = feature["nullFeatureId"].get(); + for (auto &row : rows) + if (row == null_row) row = NO_ID; + } + mesh.feature_rows.emplace_back(tables.at(feature.at("propertyTable").get()), std::move(rows)); + } + for (const auto &item : get(mesh_entry, "display", json::array())) { + const std::string variable = item.at("variable").get(); + mesh.display.emplace_back(variable, accessor_values( + document, binary, attributes.at(item.at("attribute").get()).get(), width)); + if (!get(item, "entitySet").is_null()) mesh.display_sources[variable] = item["entitySet"].get(); + } + if (primitive.contains("indices")) + mesh.indices = accessor_values(document, binary, primitive["indices"].get(), width); + return mesh; +} + +} // namespace + +Frame from_bytes(const std::vector &data) { + auto [document, binary] = unpack(data); + const json extensions = get(document, "extensions", json::object()); + if (!extensions.contains(EXTENSION)) throw Error("not an encoding3d file: no " + EXTENSION + " extension"); + const json &scene = extensions[EXTENSION]; + const json metadata = get(extensions, METADATA, {{"schema", json::object()}, {"propertyTables", json::array()}}); + const json variables = get(scene, "variables", json::object()); + auto table = [&](std::int64_t index, Table &target) { + decode_table(document, binary, metadata.at("propertyTables").at(static_cast(index)), + metadata.at("schema"), variables, target); + }; + + Frame frame; + const json frame_entry = get(scene, "frame", json::object()); + frame.time = get(frame_entry, "time", 0.0).get(); + frame.time_unit = get(frame_entry, "timeUnit", "d").get(); + frame.step = get(frame_entry, "step", 0).get(); + frame.writer = get(scene, "writer", json::object()); + if (!get(scene, "vocabulary").is_null()) frame.vocabulary_version = scene["vocabulary"].get(); + for (const auto &entry : get(scene, "dataStructures", json::array())) { + DataStructure data_structure(entry.at("id").get(), get(entry, "kind", "graph").get()); + if (!get(entry, "platform").is_null()) data_structure.platform = entry["platform"].get(); + if (!get(entry, "model").is_null()) data_structure.model = entry["model"].get(); + data_structure.extras = get(entry, "extras", json::object()); + const json labels = get(entry, "labels", json::object()); + for (const auto &item : labels.items()) + data_structure.labels.emplace_back(item.key(), item.value().get()); + if (entry.contains("origin")) { + const auto values = entry["origin"].get>(); + if (values.size() != 16) throw Error("data structure '" + data_structure.id + "': an origin of 16 numbers"); + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) + data_structure.origin[row * 4 + column] = values[column * 4 + row]; + } + std::map tables; + for (const auto &item : get(entry, "entitySets", json::array())) { + EntitySet set; + if (!get(item, "rank").is_null()) set.rank = item["rank"].get(); + if (!get(item, "scale").is_null()) set.scale = item["scale"].get(); + table(item.at("propertyTable").get(), set); + tables[item["propertyTable"].get()] = item.at("name").get(); + data_structure.entity_sets.push_back(std::move(set)); + } + for (const auto &item : get(entry, "relations", json::array())) { + Relation relation("", item.at("sourceSet").get(), item.at("targetSet").get()); + table(item.at("propertyTable").get(), relation); + data_structure.relations.push_back(std::move(relation)); + } + if (entry.contains("scalars")) { + Table scalars; + table(entry["scalars"].get(), scalars); + data_structure.scalars = std::move(scalars); + } + if (entry.contains("grid")) { + const json &item = entry["grid"]; + Grid grid; + grid.kind = item.at("kind").get(); + grid.shape = item.at("shape").get>(); + grid.origin = item.at("origin").get>(); + grid.spacing = item.at("spacing").get>(); + grid.axes = item.at("axes").get>(); + grid.periodic = item.at("periodic").get>(); + grid.z_up = get(item, "zUp", true).get(); + grid.max_level = get(item, "maxLevel", 0).get(); + grid.cells = item.at("cells").get(); + data_structure.grid = std::move(grid); + } + for (const auto &mesh_entry : get(entry, "meshes", json::array())) + data_structure.meshes.push_back(decode_mesh(document, binary, mesh_entry, tables)); + frame.data_structures.push_back(std::move(data_structure)); + } + for (const auto &entry : get(scene, "mappings", json::array())) { + Mapping mapping("", {entry.at("source").at("dataStructure").get(), + entry.at("source").at("entitySet").get()}, + {entry.at("target").at("dataStructure").get(), + entry.at("target").at("entitySet").get()}); + if (!get(entry, "method").is_null()) mapping.method = entry["method"].get(); + table(entry.at("propertyTable").get(), mapping); + frame.mappings.push_back(std::move(mapping)); + } + if (scene.contains("update")) { + Update update; + update.participant = scene["update"].at("participant").get(); + for (const auto &item : get(scene["update"], "removed", json::array())) { + Table removed; + table(item.at("propertyTable").get(), removed); + update.removed[{item.at("dataStructure").get(), item.at("entitySet").get()}] = + removed.column("id").as(); + } + frame.update = std::move(update); + } + json other_extensions = json::object(); + for (const auto &item : extensions.items()) + if (item.key() != METADATA && item.key() != EXTENSION && item.key() != FEATURES) + other_extensions[item.key()] = item.value(); + if (!other_extensions.empty()) frame.unknown["extensions"] = other_extensions; + if (document.contains("extras")) frame.unknown["extras"] = document["extras"]; + return frame; +} + +Frame read_frame(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error("cannot read '" + path + "'"); + std::vector data((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + return from_bytes(data); +} + +} // namespace encoding3d diff --git a/bindings/cpp/src/coupling.cpp b/bindings/cpp/src/coupling.cpp new file mode 100644 index 0000000..66d6ba8 --- /dev/null +++ b/bindings/cpp/src/coupling.cpp @@ -0,0 +1,223 @@ +// A participant's side of a coupling session: the folder transport, the participant, and the ZeroMQ transport when +// built with libzmq (ENCODING3D_WITH_ZMQ). +#include "encoding3d/coupling.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "encoding3d/encoding3d.hpp" + +#ifdef ENCODING3D_WITH_ZMQ +#include +#endif + +namespace encoding3d::coupling { + +namespace { + +namespace filesystem = std::filesystem; + +std::vector read_bytes(const filesystem::path &path) { + std::ifstream file(path, std::ios::binary); + return std::vector((std::istreambuf_iterator(file)), std::istreambuf_iterator()); +} + +void write_bytes(const filesystem::path &path, const void *data, std::size_t size) { + std::ofstream file(path, std::ios::binary); + file.write(static_cast(data), static_cast(size)); +} + +std::string numbered(std::int64_t number) { + std::ostringstream text; + text << std::setw(8) << std::setfill('0') << number; + return text.str(); +} + +/// A participant's folder queues: //to_coordinator/ and to_participant/. Message n is n.json +/// (its header) and n.glb (its payload), the payload written first and the header renamed into place last. +class FolderChannel : public Channel { + public: + FolderChannel(const std::string &folder, const std::string &participant) + : incoming_(filesystem::path(folder) / participant / "to_participant"), + outgoing_(filesystem::path(folder) / participant / "to_coordinator") { + filesystem::create_directories(incoming_); + filesystem::create_directories(outgoing_); + } + + void send(const json &header, const std::vector *payload) override { + const auto stem = outgoing_ / numbered(next_write_); + json written = header; + if (payload) { + write_bytes(stem.string() + ".glb", payload->data(), payload->size()); + written["payload"] = numbered(next_write_) + ".glb"; + } + const std::string text = written.dump(); + write_bytes(stem.string() + ".json.part", text.data(), text.size()); + filesystem::rename(stem.string() + ".json.part", stem.string() + ".json"); + ++next_write_; + } + + Message receive(std::optional timeout) override { + const auto stem = incoming_ / numbered(next_read_); + const auto start = std::chrono::steady_clock::now(); + while (!filesystem::exists(stem.string() + ".json")) { + if (timeout && std::chrono::duration(std::chrono::steady_clock::now() - start).count() > *timeout) + throw Error("no message in " + std::to_string(*timeout) + " s"); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + const auto text = read_bytes(stem.string() + ".json"); + Message message{json::parse(text.begin(), text.end()), std::nullopt}; + std::vector files{stem.string() + ".json"}; + if (message.header.contains("payload")) { + files.push_back(incoming_ / message.header["payload"].get()); + message.payload = read_bytes(files.back()); + message.header.erase("payload"); + } + ++next_read_; + if (keep_) read_files_.insert(read_files_.end(), files.begin(), files.end()); + else for (const auto &path : files) filesystem::remove(path); + return message; + } + + void keep_messages(bool keep) override { + if (keep) return; + keep_ = false; + for (const auto &path : read_files_) filesystem::remove(path); + read_files_.clear(); + } + + private: + filesystem::path incoming_, outgoing_; + std::int64_t next_read_ = 0, next_write_ = 0; + bool keep_ = true; // false: a message's files deleted once read + std::vector read_files_; // those of the messages read and kept +}; + +#ifdef ENCODING3D_WITH_ZMQ +/// A DEALER socket named after the participant, connected to the coordinator's ROUTER socket; a message is one or two +/// frames, its header then its payload. +class ZmqChannel : public Channel { + public: + ZmqChannel(const std::string &endpoint, const std::string &participant) { + context_ = zmq_ctx_new(); + socket_ = zmq_socket(context_, ZMQ_DEALER); + zmq_setsockopt(socket_, ZMQ_ROUTING_ID, participant.data(), participant.size()); + const int linger = 1000; + zmq_setsockopt(socket_, ZMQ_LINGER, &linger, sizeof linger); + if (zmq_connect(socket_, endpoint.c_str()) != 0) + throw Error("cannot connect to " + endpoint + ": " + zmq_strerror(zmq_errno())); + } + ~ZmqChannel() override { + zmq_close(socket_); + zmq_ctx_term(context_); + } + + void send(const json &header, const std::vector *payload) override { + const std::string text = header.dump(); + zmq_send(socket_, text.data(), text.size(), payload ? ZMQ_SNDMORE : 0); + if (payload) zmq_send(socket_, payload->data(), payload->size(), 0); + } + + Message receive(std::optional timeout) override { + zmq_pollitem_t item{socket_, 0, ZMQ_POLLIN, 0}; + if (zmq_poll(&item, 1, timeout ? static_cast(*timeout * 1000) : -1) <= 0) + throw Error("no message in " + std::to_string(timeout.value_or(0.0)) + " s"); + std::vector> parts; + int more = 1; + while (more) { + zmq_msg_t part; + zmq_msg_init(&part); + zmq_msg_recv(&part, socket_, 0); + const auto *data = static_cast(zmq_msg_data(&part)); + parts.emplace_back(data, data + zmq_msg_size(&part)); + more = zmq_msg_more(&part); + zmq_msg_close(&part); + } + Message message{json::parse(parts[0].begin(), parts[0].end()), std::nullopt}; + if (parts.size() > 1) message.payload = std::move(parts[1]); + return message; + } + + private: + void *context_ = nullptr; + void *socket_ = nullptr; +}; +#endif + +std::string environment(const char *name) { + const char *value = std::getenv(name); + return value ? value : ""; +} + +} // namespace + +bool has_zeromq() { +#ifdef ENCODING3D_WITH_ZMQ + return true; +#else + return false; +#endif +} + +std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant) { + if (endpoint.rfind("files:", 0) == 0) return std::make_unique(endpoint.substr(6), participant); +#ifdef ENCODING3D_WITH_ZMQ + return std::make_unique(endpoint, participant); +#else + throw Error("the endpoint " + endpoint + " needs ZeroMQ: build encoding3d with libzmq, or use a folder (files:path)"); +#endif +} + +Participant::Participant(json declaration, std::string endpoint, json links, std::optional timeout) + : timeout_(timeout) { + if (endpoint.empty()) endpoint = environment(ENDPOINT_VARIABLE); + if (endpoint.empty()) throw Error(std::string("no endpoint: give one, or set ") + ENDPOINT_VARIABLE); + name = declaration.value("participant", std::string()); + if (name.empty()) { + name = environment(PARTICIPANT_VARIABLE); + declaration["participant"] = name; + } + channel_ = participant_channel(endpoint, name); + channel_->send({{"type", "hello"}, {"protocol", PROTOCOL_VERSION}, {"declaration", declaration}, + {"links", links}}); + const auto welcome = channel_->receive(timeout_); + const auto type = welcome.header.at("type").get(); + if (type == "stop") + throw Error("the coordinator refused " + name + ": " + welcome.header.value("reason", std::string())); + if (type != "welcome") throw Error("expected a welcome, got " + type); + id_offset = welcome.header.at("idOffset").get(); + id_count = welcome.header.at("idCount").get(); + session = welcome.header.at("session"); + channel_->keep_messages(session.value("keepMessages", true)); + couplings = welcome.header.value("couplings", json::array()); +} + +std::optional Participant::next_turn() { + auto message = channel_->receive(timeout_); + const auto type = message.header.at("type").get(); + if (type == "stop") { + stopped = message.header.value("reason", std::string()); + return std::nullopt; + } + if (type != "turn") throw Error("expected a turn, got " + type); + step_ = message.header.at("step").get(); + return Turn{step_, message.header.at("phase").get(), message.header.at("time").get(), + message.header.at("until").get(), from_bytes(message.payload.value())}; +} + +void Participant::send_update(Frame frame) { + if (!frame.update) frame.update = Update{}; + frame.update->participant = name; + const auto bytes = to_bytes(frame); + channel_->send({{"type", "update"}, {"step", step_}}, &bytes); +} + +void Participant::fail(const std::string &message) { channel_->send({{"type", "error"}, {"message", message}}); } + +} // namespace encoding3d::coupling diff --git a/bindings/cpp/src/dump.cpp b/bindings/cpp/src/dump.cpp new file mode 100644 index 0000000..436a5c3 --- /dev/null +++ b/bindings/cpp/src/dump.cpp @@ -0,0 +1,183 @@ +// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md), and +// the comparison `encoding3d compare` makes. +#include "encoding3d/encoding3d.hpp" + +#include +#include + +namespace encoding3d { + +namespace { + +template json value_json(T value) { + if constexpr (std::is_floating_point_v) { + if (std::isnan(value)) return nullptr; + if (std::isinf(value)) return value > 0 ? "Infinity" : "-Infinity"; + return static_cast(value); + } else { + return value; + } +} + +json column_json(const Column &column) { + return std::visit([&](const auto &values) { + json items = json::array(); + if (column.width == 1) { + for (std::size_t item = 0; item < values.size(); ++item) + items.push_back(column.is_missing(item) ? json(nullptr) : value_json(values[item])); // missing: null + return items; + } + for (std::size_t row = 0; row < column.rows(); ++row) { + json vector = json::array(); + for (std::size_t component = 0; component < column.width; ++component) + vector.push_back(value_json(values[row * column.width + component])); + items.push_back(vector); + } + return items; + }, column.values); +} + +json table_json(const Table &table) { + json columns = json::object(); + for (const auto &[name, column] : table.columns) + columns[name] = {{"dtype", column.dtype()}, {"values", column_json(column)}}; + json entry = {{"name", table.name}, {"count", table.row_count()}, {"columns", columns}}; + json declarations = json::object(); + for (const auto &[name, variable] : table.variables) { + json declared = {{"unit", variable.unit}, {"role", variable.role}}; + if (variable.standard_name) declared["standard_name"] = *variable.standard_name; + if (variable.kind) declared["kind"] = *variable.kind; + if (variable.description) declared["description"] = *variable.description; + if (variable.minimum) declared["minimum"] = *variable.minimum; + if (variable.maximum) declared["maximum"] = *variable.maximum; + if (!variable.references.empty()) declared["references"] = variable.references; + if (!variable.doi.empty()) declared["doi"] = variable.doi; + if (variable.format) declared["format"] = *variable.format; + declarations[name] = declared; + } + if (!declarations.empty()) entry["variables"] = declarations; + return entry; +} + +bool is_number(const json &value) { return value.is_number(); } + +} // namespace + +json frame_to_json(const Frame &frame) { + json data_structures = json::array(); + for (const auto &data_structure : frame.data_structures) { + json origin = json::array(); + for (int row = 0; row < 4; ++row) + origin.push_back({data_structure.origin[row * 4], data_structure.origin[row * 4 + 1], + data_structure.origin[row * 4 + 2], data_structure.origin[row * 4 + 3]}); + json sets = json::array(); + for (const auto &set : data_structure.entity_sets) { + json item = table_json(set); + item["rank"] = set.rank ? json(*set.rank) : json(nullptr); + if (set.scale) item["scale"] = *set.scale; + sets.push_back(item); + } + json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}, {"origin", origin}, + {"entity_sets", sets}}; + if (data_structure.platform) entry["platform"] = *data_structure.platform; + if (data_structure.model) entry["model"] = *data_structure.model; + if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + if (!data_structure.labels.empty()) { + json labels = json::object(); + for (const auto &[label, standard] : data_structure.labels) labels[label] = standard; + entry["labels"] = labels; + } + if (!data_structure.relations.empty()) { + json relations = json::array(); + for (const auto &relation : data_structure.relations) { + json item = table_json(relation); + item["source_set"] = relation.source_set; + item["target_set"] = relation.target_set; + relations.push_back(item); + } + entry["relations"] = relations; + } + if (data_structure.scalars) entry["scalars"] = table_json(*data_structure.scalars); + if (data_structure.grid) { + const Grid &grid = *data_structure.grid; + entry["grid"] = {{"kind", grid.kind}, {"shape", grid.shape}, {"origin", grid.origin}, + {"spacing", grid.spacing}, {"axes", grid.axes}, {"periodic", grid.periodic}, + {"z_up", grid.z_up}, {"max_level", grid.max_level}, {"cells", grid.cells}}; + } + if (!data_structure.meshes.empty()) { + json meshes = json::array(); + for (const auto &mesh : data_structure.meshes) { + json positions = json::array(); + for (std::size_t vertex = 0; vertex < mesh.vertex_count(); ++vertex) + positions.push_back({static_cast(static_cast(mesh.positions[vertex * 3])), + static_cast(static_cast(mesh.positions[vertex * 3 + 1])), + static_cast(static_cast(mesh.positions[vertex * 3 + 2]))}); + json feature_rows = json::object(); + for (const auto &[name, rows] : mesh.feature_rows) feature_rows[name] = rows; + json item = {{"name", mesh.name}, {"positions_float32", positions}, {"feature_rows", feature_rows}}; + if (mesh.indices) item["indices"] = *mesh.indices; + if (!mesh.display.empty()) { + json display = json::object(); + for (const auto &[name, values] : mesh.display) display[name] = column_json(Column(values)); + item["display"] = display; + } + meshes.push_back(item); + } + entry["meshes"] = meshes; + } + data_structures.push_back(entry); + } + json mappings = json::array(); + for (const auto &mapping : frame.mappings) { + json item = table_json(mapping); + item["source"] = {mapping.source.first, mapping.source.second}; + item["target"] = {mapping.target.first, mapping.target.second}; + item["method"] = mapping.method ? json(*mapping.method) : json(nullptr); + mappings.push_back(item); + } + json out = {{"time", frame.time}, {"time_unit", frame.time_unit}, {"step", frame.step}, + {"data_structures", data_structures}, {"mappings", mappings}}; + if (frame.update) { // a partial frame: its sender and removed entities, sorted (std::map) + json removed = json::array(); + for (const auto &[key, ids] : frame.update->removed) + removed.push_back({{"data_structure", key.first}, {"entity_set", key.second}, {"ids", ids}}); + out["update"] = {{"participant", frame.update->participant}, {"removed", removed}}; + } + return out; +} + +std::vector compare(const json &actual, const json &expected, const std::string &path) { + std::vector found; + if (expected.is_object()) { + if (!actual.is_object()) return {path + ": an object expected"}; + for (const auto &item : expected.items()) + if (!actual.contains(item.key())) found.push_back(path + ": missing key '" + item.key() + "'"); + for (const auto &item : actual.items()) + if (!expected.contains(item.key())) found.push_back(path + ": unexpected key '" + item.key() + "'"); + for (const auto &item : expected.items()) + if (actual.contains(item.key())) { + auto more = compare(actual[item.key()], item.value(), path + "." + item.key()); + found.insert(found.end(), more.begin(), more.end()); + } + return found; + } + if (expected.is_array()) { + if (!actual.is_array()) return {path + ": an array expected"}; + if (actual.size() != expected.size()) + return {path + ": " + std::to_string(actual.size()) + " items, " + std::to_string(expected.size()) + + " expected"}; + for (std::size_t item = 0; item < expected.size(); ++item) { + auto more = compare(actual[item], expected[item], path + "[" + std::to_string(item) + "]"); + found.insert(found.end(), more.begin(), more.end()); + } + return found; + } + if (is_number(expected)) { + if (is_number(actual) && actual.get() == expected.get()) return {}; + return {path + ": " + actual.dump() + ", " + expected.dump() + " expected"}; + } + if (actual != expected) return {path + ": " + actual.dump() + ", " + expected.dump() + " expected"}; + return {}; +} + +} // namespace encoding3d diff --git a/bindings/cpp/src/glb.cpp b/bindings/cpp/src/glb.cpp new file mode 100644 index 0000000..287b777 --- /dev/null +++ b/bindings/cpp/src/glb.cpp @@ -0,0 +1,72 @@ +// The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary +// chunk padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. +#include "encoding3d/encoding3d.hpp" + +#include "bytes.hpp" + +namespace encoding3d { + +namespace { +constexpr std::uint32_t GLB_MAGIC = 0x46546C67; // "glTF" +constexpr std::uint32_t GLB_VERSION = 2; +constexpr std::uint32_t JSON_CHUNK = 0x4E4F534A; // "JSON" +constexpr std::uint32_t BINARY_CHUNK = 0x004E4942; // "BIN\0" + +void append_word(std::vector &bytes, std::uint32_t value) { detail::append_values(bytes, &value, 1); } +} // namespace + +std::vector pack(const json &document, const std::vector &binary) { + std::string text = document.dump(-1, ' ', false, json::error_handler_t::strict); + text.append(detail::padding(text.size()), ' '); + std::vector padded = binary; + padded.resize(binary.size() + detail::padding(binary.size()), 0); + const std::size_t total = 12 + 8 + text.size() + (padded.empty() ? 0 : 8 + padded.size()); + if (total > 0xFFFFFFFFull) throw Error("a GLB file is at most 4 GiB"); + std::vector bytes; + bytes.reserve(total); + append_word(bytes, GLB_MAGIC); + append_word(bytes, GLB_VERSION); + append_word(bytes, static_cast(total)); + append_word(bytes, static_cast(text.size())); + append_word(bytes, JSON_CHUNK); + bytes.insert(bytes.end(), text.begin(), text.end()); + if (!padded.empty()) { + append_word(bytes, static_cast(padded.size())); + append_word(bytes, BINARY_CHUNK); + bytes.insert(bytes.end(), padded.begin(), padded.end()); + } + return bytes; +} + +std::pair> unpack(const std::vector &data) { + if (data.size() < 20) throw Error("not a GLB file: too short"); + if (detail::read_word(data, 0) != GLB_MAGIC) throw Error("not a GLB file: wrong magic"); + const auto version = detail::read_word(data, 4); + if (version != GLB_VERSION) throw Error("GLB version " + std::to_string(version) + ", 2 expected"); + const auto total = detail::read_word(data, 8); + if (total != data.size()) + throw Error("GLB header length " + std::to_string(total) + ", the file holds " + std::to_string(data.size()) + + " bytes"); + std::size_t offset = 12; + bool found_document = false; + json document; + std::vector binary; + while (offset < total) { + const std::size_t chunk_length = detail::read_word(data, offset); + const auto chunk_type = detail::read_word(data, offset + 4); + if (offset + 8 + chunk_length > total) throw Error("a GLB chunk runs past the end of the file"); + const auto begin = data.begin() + static_cast(offset + 8); + const auto end = begin + static_cast(chunk_length); + if (chunk_type == JSON_CHUNK) { + document = json::parse(begin, end); + found_document = true; + } else if (chunk_type == BINARY_CHUNK) { + binary.assign(begin, end); + } + offset += 8 + chunk_length; + } + if (!found_document) throw Error("GLB file without a JSON chunk"); + return {std::move(document), std::move(binary)}; +} + +} // namespace encoding3d diff --git a/bindings/cpp/src/model.cpp b/bindings/cpp/src/model.cpp new file mode 100644 index 0000000..bd04fea --- /dev/null +++ b/bindings/cpp/src/model.cpp @@ -0,0 +1,148 @@ +#include "encoding3d/model.hpp" + +#include +#include + +namespace encoding3d { + +std::size_t Column::size() const { + return std::visit([](const auto &column_values) { return column_values.size(); }, values); +} + +namespace { + +template struct TypeName; +template <> struct TypeName { static constexpr const char *value = "int8"; }; +template <> struct TypeName { static constexpr const char *value = "uint8"; }; +template <> struct TypeName { static constexpr const char *value = "int16"; }; +template <> struct TypeName { static constexpr const char *value = "uint16"; }; +template <> struct TypeName { static constexpr const char *value = "int32"; }; +template <> struct TypeName { static constexpr const char *value = "uint32"; }; +template <> struct TypeName { static constexpr const char *value = "int64"; }; +template <> struct TypeName { static constexpr const char *value = "uint64"; }; +template <> struct TypeName { static constexpr const char *value = "float32"; }; +template <> struct TypeName { static constexpr const char *value = "float64"; }; +template <> struct TypeName { static constexpr const char *value = "bool"; }; +template <> struct TypeName { static constexpr const char *value = "string"; }; + +const std::vector NO_RESERVED = {}; +const std::vector ENTITY_SET_RESERVED = {"id", "label", "complex", "parent", "edge_type", + "level", "index", "sibling_order"}; +const std::vector RELATION_RESERVED = {"id", "source", "target", "relation"}; +const std::vector MAPPING_RESERVED = {"source", "target", "weight"}; + +} // namespace + +bool Column::has_missing() const { + for (const bool flag : missing) + if (flag) return true; + return false; +} + +std::string Column::dtype() const { + return std::visit([](const auto &column_values) -> std::string { + using T = typename std::decay_t::value_type; + return TypeName::value; + }, values); +} + +bool Variable::operator==(const Variable &other) const { + return unit == other.unit && standard_name == other.standard_name && kind == other.kind && role == other.role && + description == other.description && minimum == other.minimum && maximum == other.maximum && + references == other.references && doi == other.doi && format == other.format; +} + +std::size_t Table::row_count() const { return columns.empty() ? 0 : columns.front().second.rows(); } + +bool Table::has_column(const std::string &column_name) const { + return std::any_of(columns.begin(), columns.end(), [&](const auto &item) { return item.first == column_name; }); +} + +const Column &Table::column(const std::string &column_name) const { + for (const auto &item : columns) + if (item.first == column_name) return item.second; + throw Error("table '" + name + "' has no column '" + column_name + "'"); +} + +Column &Table::column(const std::string &column_name) { + return const_cast(static_cast(*this).column(column_name)); +} + +Table &Table::set(const std::string &column_name, Column values) { + for (auto &item : columns) + if (item.first == column_name) { + item.second = std::move(values); + return *this; + } + columns.emplace_back(column_name, std::move(values)); + return *this; +} + +Table &Table::declare(const std::string &column_name, Variable variable) { + variables[column_name] = std::move(variable); + return *this; +} + +const std::vector &Table::reserved() const { + switch (table_kind) { + case TableKind::entity_set: return ENTITY_SET_RESERVED; + case TableKind::relation: return RELATION_RESERVED; + case TableKind::mapping: return MAPPING_RESERVED; + default: return NO_RESERVED; + } +} + +bool Table::is_reserved(const std::string &column_name) const { + const auto &names = reserved(); + return std::find(names.begin(), names.end(), column_name) != names.end(); +} + +std::vector Table::variable_names() const { + std::vector names; + for (const auto &item : columns) + if (!is_reserved(item.first)) names.push_back(item.first); + return names; +} + +void Table::check_lengths() const { + for (const auto &item : columns) { + if (item.second.width == 0 || item.second.size() % item.second.width != 0) + throw Error("column '" + item.first + "' of '" + name + "': its values are not whole rows"); + if (item.second.rows() != row_count()) + throw Error("table '" + name + "': columns of different lengths"); + if (!item.second.missing.empty() && item.second.missing.size() != item.second.rows()) + throw Error("column '" + item.first + "' of '" + name + "': one missing flag per row"); + if (item.second.has_missing() && (is_reserved(item.first) || item.second.width != 1)) + throw Error("column '" + item.first + "' of '" + name + "': missing values only in variables of one value " + "per row"); + } +} + +std::vector EntitySet::ids() const { + if (has_column("id")) return column("id").as(); + std::vector rows(row_count()); + std::iota(rows.begin(), rows.end(), 0); + return rows; +} + +const EntitySet &DataStructure::entity_set(const std::string &name) const { + for (const auto &candidate : entity_sets) + if (candidate.name == name) return candidate; + throw Error("data structure '" + id + "' has no entity set '" + name + "'"); +} + +EntitySet &DataStructure::entity_set(const std::string &name) { + return const_cast(static_cast(*this).entity_set(name)); +} + +const DataStructure &Frame::data_structure(const std::string &id) const { + for (const auto &candidate : data_structures) + if (candidate.id == id) return candidate; + throw Error("the frame has no data structure '" + id + "'"); +} + +DataStructure &Frame::data_structure(const std::string &id) { + return const_cast(static_cast(*this).data_structure(id)); +} + +} // namespace encoding3d diff --git a/bindings/cpp/src/sequence.cpp b/bindings/cpp/src/sequence.cpp new file mode 100644 index 0000000..c364619 --- /dev/null +++ b/bindings/cpp/src/sequence.cpp @@ -0,0 +1,65 @@ +// Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. +#include "encoding3d/encoding3d.hpp" + +#include +#include +#include +#include +#include + +namespace encoding3d { + +void write_index(const std::string &path, const Index &index) { + json frames = json::array(); + for (const auto &entry : index.frames) + frames.push_back({{"time", entry.time}, {"step", entry.step}, {"file", entry.file}}); + const json document = {{"fspmIndex", index.version}, {"timeUnit", index.time_unit}, {"frames", frames}}; + std::ofstream file(path, std::ios::binary); + if (!file) throw Error("cannot write '" + path + "'"); + file << document.dump(1) << "\n"; +} + +Index read_index(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error("cannot read '" + path + "'"); + const json document = json::parse(file); + Index index; + index.version = document.value("fspmIndex", std::string(SPECIFICATION_VERSION)); + index.time_unit = document.value("timeUnit", std::string("d")); + const auto directory = std::filesystem::absolute(std::filesystem::path(path)).parent_path(); + for (const auto &item : document.value("frames", json::array())) { + IndexEntry entry; + entry.time = item.at("time").get(); + entry.step = item.at("step").get(); + entry.file = item.at("file").get(); + entry.path = (directory / entry.file).string(); + index.frames.push_back(entry); + } + return index; +} + +std::string write_sequence(const std::string &directory, const std::vector &frames, const std::string &name) { + std::filesystem::create_directories(directory); + Index index; + std::set time_units; + for (std::size_t number = 0; number < frames.size(); ++number) { + std::ostringstream file; + file << name << "_" << std::setw(6) << std::setfill('0') << number << ".glb"; + write_frame((std::filesystem::path(directory) / file.str()).string(), frames[number]); + index.frames.push_back({frames[number].time, frames[number].step, file.str(), ""}); + time_units.insert(frames[number].time_unit); + } + if (time_units.size() > 1) throw Error("the frames' times are in several units"); + if (!time_units.empty()) index.time_unit = *time_units.begin(); + const auto path = (std::filesystem::path(directory) / INDEX_FILE).string(); + write_index(path, index); + return path; +} + +std::vector read_sequence(const std::string &path) { + std::vector frames; + for (const auto &entry : read_index(path).frames) frames.push_back(read_frame(entry.path)); + return frames; +} + +} // namespace encoding3d diff --git a/bindings/cpp/test/test_encoding3d.cpp b/bindings/cpp/test/test_encoding3d.cpp new file mode 100644 index 0000000..8a9dab9 --- /dev/null +++ b/bindings/cpp/test/test_encoding3d.cpp @@ -0,0 +1,259 @@ +// The C++ codec's tests: the conformance files read as their expected values, written and read back the same, every +// column type, meshes, sequences, and what the specification forbids refused. +// +// encoding3d_tests CONFORMANCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "encoding3d/encoding3d.hpp" + +namespace filesystem = std::filesystem; +using namespace encoding3d; + +namespace { + +int failures = 0, checks = 0; + +void check(bool condition, const std::string &what) { + ++checks; + if (!condition) { + ++failures; + std::cerr << "FAILED: " << what << "\n"; + } +} + +template void check_throws(Function function, const std::string &what) { + bool thrown = false; + try { + function(); + } catch (const Error &) { + thrown = true; + } + check(thrown, what + " (an Error expected)"); +} + +std::vector conformance_files(const filesystem::path &conformance) { + std::vector files; + for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) + if (entry.is_regular_file() && entry.path().extension() == ".glb" && + entry.path().string().find("coupling") == std::string::npos) // the coupling sessions: their own check + files.push_back(entry.path()); + std::sort(files.begin(), files.end()); + return files; +} + +json expected_of(const filesystem::path &glb) { + auto path = glb; + path.replace_extension(".expected.json"); + std::ifstream file(path, std::ios::binary); + return json::parse(file); +} + +std::vector file_bytes(const filesystem::path &path) { + std::ifstream file(path, std::ios::binary); + return {std::istreambuf_iterator(file), std::istreambuf_iterator()}; +} + +void report(const std::vector &differences, const std::string &what) { + for (std::size_t item = 0; item < std::min(differences.size(), 5); ++item) + std::cerr << " " << differences[item] << "\n"; + check(differences.empty(), what); +} + +void test_conformance(const filesystem::path &conformance) { + const auto files = conformance_files(conformance); + check(files.size() == 10, "ten conformance files"); + for (const auto &path : files) { + const auto name = filesystem::relative(path, conformance).string(); + const auto data = file_bytes(path); + auto [document, binary] = unpack(data); // the container, repacked + auto [again, binary_again] = unpack(pack(document, binary)); + check(binary_again == binary && again == document && binary.size() % 8 == 0, "GLB repacked: " + name); + const auto frame = read_frame(path.string()); // reading + report(compare(json::parse(frame_to_json(frame).dump()), expected_of(path)), "read as expected: " + name); + const auto written = from_bytes(to_bytes(frame)); // writing + report(compare(json::parse(frame_to_json(written).dump()), expected_of(path)), "written and read: " + name); + } +} + +void test_columns() { + EntitySet organs("Organ", 1); + organs.set("id", std::vector{0, 1, 2}) + .set("label", {"Leaf", "Internode", "Leaf"}) + .set("parent", std::vector{-1, 0, 1}) + .set("edge_type", {"none", "successor", "branch"}) + .set("with_nan", std::vector{1.0, std::nan(""), -2.5}) + .set("with_infinities", std::vector{std::numeric_limits::infinity(), 1.f, + -std::numeric_limits::infinity()}) + .set("flag", std::vector{true, false, true}) + .set("name", {"\xc3\xa9", "", "a b"}) + .set("position", Column(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9}, 3)) + .set("column_int8", std::vector{1, 2, 3}) + .set("column_uint8", std::vector{1, 2, 3}) + .set("column_int16", std::vector{1, 2, 3}) + .set("column_uint16", std::vector{1, 2, 3}) + .set("column_int32", std::vector{1, 2, 3}) + .set("column_uint32", std::vector{1, 2, 3}) + .set("column_uint64", std::vector{1, 2, 3}); + Variable length; + length.unit = "m"; + length.standard_name = "length"; + length.kind = "extensive"; + length.description = "a length"; + length.minimum = 0; + length.references = {"a reference"}; + organs.declare("with_nan", length); + DataStructure plants("plants"); + plants.entity_sets.push_back(organs); + plants.origin = {1, 0, 0, 0.5, 0, 1, 0, 0, 0, 0, 1, -0.1, 0, 0, 0, 1}; + plants.extras = {{"platform", {1, 2}}}; + Frame frame; + frame.data_structures.push_back(plants); + frame.time = 2.5; + frame.time_unit = "h"; + frame.step = 3; + + const auto read_back = from_bytes(to_bytes(frame)); + const auto &set = read_back.data_structure("plants").entity_set("Organ"); + check(set.column("id").holds(), "ids are int64"); + const auto &with_nan = set.column("with_nan").as(); + check(with_nan[0] == 1.0 && std::isnan(with_nan[1]) && with_nan[2] == -2.5, "NaN read back"); + const auto &infinities = set.column("with_infinities").as(); + check(std::isinf(infinities[0]) && infinities[0] > 0 && infinities[1] == 1.f && std::isinf(infinities[2]) && + infinities[2] < 0, "infinities read back"); + check(set.column("flag").as() == std::vector{true, false, true}, "booleans read back"); + check(set.column("name").as() == std::vector{"\xc3\xa9", "", "a b"}, "text read back"); + check(set.column("position").width == 3 && + set.column("position").as() == std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9}, "vectors read back"); + check(set.column("label").as() == std::vector{"Leaf", "Internode", "Leaf"}, "labels"); + check(set.column("column_uint16").as() == std::vector{1, 2, 3}, "uint16 read back"); + check(set.column("column_uint64").as() == std::vector{1, 2, 3}, "uint64 read back"); + check(set.variables.at("with_nan") == length, "declaration read back"); + check(read_back.data_structure("plants").origin == plants.origin, "origin read back"); + check(read_back.data_structure("plants").extras == plants.extras, "extras read back"); + check(read_back.time == 2.5 && read_back.time_unit == "h" && read_back.step == 3, "time read back"); +} + +void test_mesh() { + EntitySet segments("SubOrgan", 6); + segments.set("id", std::vector{10, 11}); + Mesh mesh; + mesh.name = "roots"; + mesh.positions = {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}; + mesh.feature_rows = {{"SubOrgan", {0, 1, 1, -1}}}; + mesh.indices = std::vector{0, 1, 2, 0, 2, 3}; + mesh.display = {{"radius", {1.f, 2.f, 2.f, 0.f}}}; + mesh.display_sources = {{"radius", "SubOrgan"}}; + DataStructure roots("roots"); + roots.entity_sets.push_back(segments); + roots.meshes.push_back(mesh); + Frame frame; + frame.data_structures.push_back(roots); + const auto read_back = from_bytes(to_bytes(frame)).data_structures[0].meshes[0]; + check(read_back.positions == mesh.positions && read_back.feature_rows == mesh.feature_rows && + read_back.indices == mesh.indices && read_back.display == mesh.display && + read_back.display_sources == mesh.display_sources, "mesh read back"); +} + +void test_refused() { + auto frame_with = [](EntitySet set) { + DataStructure plants("plants"); + plants.entity_sets.push_back(std::move(set)); + Frame frame; + frame.data_structures.push_back(plants); + return frame; + }; + EntitySet bad_column("Organ"); + bad_column.set("id", std::vector{0}).set("1st", std::vector{1.0}); + check_throws([&] { to_bytes(frame_with(bad_column)); }, "a column name that is not an identifier"); + Frame bad_id; + bad_id.data_structures.emplace_back("bad id"); + check_throws([&] { to_bytes(bad_id); }, "a data structure id that is not an identifier"); + EntitySet bad_edge("Organ"); + bad_edge.set("id", std::vector{0}).set("parent", std::vector{-1}).set("edge_type", {"root"}); + check_throws([&] { to_bytes(frame_with(bad_edge)); }, "an unknown edge type"); + EntitySet both("Organ"); + both.set("id", std::vector{0, 1}) + .set("x", std::vector{std::nan(""), std::numeric_limits::lowest()}); + check_throws([&] { to_bytes(frame_with(both)); }, "NaN and its stand-in in one column"); + EntitySet lengths("Organ"); + lengths.set("id", std::vector{0, 1}).set("x", std::vector{1.0}); + check_throws([&] { to_bytes(frame_with(lengths)); }, "columns of different lengths"); + check_throws([] { unpack({1, 2, 3}); }, "not a GLB file"); +} + +void test_sequence(const filesystem::path &conformance) { + const auto folder = filesystem::temp_directory_path() / "encoding3d_cpp_sequence"; + filesystem::remove_all(folder); + std::vector frames; + for (int step = 0; step < 3; ++step) { + EntitySet plant("Plant", 1); + plant.set("id", std::vector{0}).set("biomass", std::vector{double(step)}); + DataStructure plants("plants"); + plants.entity_sets.push_back(plant); + Frame frame; + frame.data_structures.push_back(plants); + frame.time = step / 24.0; + frame.step = step; + frames.push_back(frame); + } + const auto index = write_sequence(folder.string(), frames); + const auto entries = read_index(index).frames; + check(entries.size() == 3 && entries[2].step == 2, "index written and read"); + const auto read_back = read_sequence(index); + check(read_back.size() == 3 && read_back[2].data_structures[0].entity_sets[0].column("biomass").as()[0] == 2.0, + "sequence read back"); + const auto growing = read_index((conformance / "growing_sequence" / INDEX_FILE).string()); + check(std::all_of(growing.frames.begin(), growing.frames.end(), + [](const IndexEntry &entry) { return filesystem::exists(entry.path); }), + "the conformance sequence's files found"); + filesystem::remove_all(folder); +} + +} // namespace + +void test_partial_frame() { + // A coupling session's message: the sender and the removed entities kept, the reference checks lifted + EntitySet segments("SubOrgan", 6); + segments.set("id", Column(std::vector{10, 11})) + .set("complex", Column(std::vector{3, 3})) + .set("sugar", Column(std::vector{1.0, 2.0})); + DataStructure plant("plant"); + plant.entity_sets.push_back(segments); + Frame frame; + frame.data_structures.push_back(plant); + frame.update = Update{"vpl", {{{"plant", "SubOrgan"}, {5, 6}}}}; + const auto again = from_bytes(to_bytes(frame)); + check(again.update && again.update->participant == "vpl", "partial frame: its sender"); + check(again.update->removed.at({"plant", "SubOrgan"}) == std::vector{5, 6}, + "partial frame: its removed entities"); + check(frame_to_json(again)["update"]["removed"][0]["ids"] == json::array({5, 6}), "partial frame: its dump"); +} + +int main(int argument_count, char **arguments) { + if (argument_count != 2) { + std::cerr << "usage: encoding3d_tests CONFORMANCE\n"; + return 2; + } + const filesystem::path conformance = arguments[1]; + const std::vector>> tests = { + {"conformance", [&] { test_conformance(conformance); }}, {"columns", test_columns}, {"mesh", test_mesh}, + {"refused", test_refused}, {"sequence", [&] { test_sequence(conformance); }}, + {"partial frame", test_partial_frame}}; + for (const auto &[name, test] : tests) { + try { + test(); + } catch (const std::exception &error) { + check(false, name + ": " + error.what()); + } + } + std::cout << checks - failures << " of " << checks << " checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/bindings/cpp/tools/conformance_run.cpp b/bindings/cpp/tools/conformance_run.cpp new file mode 100644 index 0000000..dfbc0b9 --- /dev/null +++ b/bindings/cpp/tools/conformance_run.cpp @@ -0,0 +1,38 @@ +// The C++ codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump +// it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). +// Then .github/actions/conformance compares the dumps with the expected values and validates the written files. +// +// encoding3d_conformance_run CONFORMANCE OUT +#include +#include +#include + +#include "encoding3d/encoding3d.hpp" + +namespace filesystem = std::filesystem; + +int main(int argument_count, char **arguments) { + if (argument_count != 3) { + std::cerr << "usage: encoding3d_conformance_run CONFORMANCE OUT\n"; + return 2; + } + const filesystem::path conformance = arguments[1], out = arguments[2]; + for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) { + if (!entry.is_regular_file() || entry.path().extension() != ".glb") continue; + if (filesystem::relative(entry.path(), conformance).begin()->string() == "coupling") continue; // sessions + const auto relative = filesystem::relative(entry.path(), conformance); + const auto frame = encoding3d::read_frame(entry.path().string()); + auto dump = out / "dumps" / relative; + dump.replace_extension(".json"); + const auto written = out / "written" / relative; + filesystem::create_directories(dump.parent_path()); + filesystem::create_directories(written.parent_path()); + std::ofstream(dump, std::ios::binary) << encoding3d::frame_to_json(frame).dump(1) << "\n"; + encoding3d::write_frame(written.string(), frame); // written again from what was read + } + const auto index = conformance / "growing_sequence" / encoding3d::INDEX_FILE; + if (filesystem::exists(index)) + filesystem::copy_file(index, out / "written" / "growing_sequence" / encoding3d::INDEX_FILE, + filesystem::copy_options::overwrite_existing); + return 0; +} diff --git a/bindings/cpp/tools/coupling_conformance.cpp b/bindings/cpp/tools/coupling_conformance.cpp new file mode 100644 index 0000000..918d322 --- /dev/null +++ b/bindings/cpp/tools/coupling_conformance.cpp @@ -0,0 +1,42 @@ +// The C++ participant under test in the coupling conformance sessions (conformance/coupling, specification §9): +// launched by the coordinator as `encoding3d_coupling_conformance SESSION OUT`, it joins with the session's +// subject/declaration.json, writes what it reads from each turn to OUT/turn_.json, and sends +// subject/update_.glb as read and written again by this library. +// +// encoding3d couple-conformance conformance/coupling --participant build/encoding3d_coupling_conformance +#include +#include +#include +#include +#include + +#include "encoding3d/coupling.hpp" +#include "encoding3d/encoding3d.hpp" + +namespace filesystem = std::filesystem; + +std::string numbered(std::int64_t step) { + std::ostringstream text; + text << std::setw(4) << std::setfill('0') << step; + return text.str(); +} + +int main(int count, char **arguments) { + if (count != 3) { + std::cerr << "usage: encoding3d_coupling_conformance SESSION OUT\n"; + return 2; + } + const filesystem::path session = arguments[1], out = arguments[2]; + filesystem::create_directories(out); + std::ifstream declaration_file(session / "subject" / "declaration.json"); + encoding3d::coupling::Participant participant(encoding3d::json::parse(declaration_file), "", encoding3d::json::array(), + 120.0); + while (auto turn = participant.next_turn()) { + std::ofstream dump(out / ("turn_" + numbered(turn->step) + ".json")); + dump << encoding3d::frame_to_json(turn->frame).dump(1); + dump.close(); + const auto update = encoding3d::read_frame((session / "subject" / ("update_" + numbered(turn->step) + ".glb")).string()); + participant.send_update(encoding3d::from_bytes(encoding3d::to_bytes(update))); + } + return 0; +} diff --git a/bindings/java/LICENSE b/bindings/java/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/bindings/java/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/bindings/java/README.md b/bindings/java/README.md new file mode 100644 index 0000000..89702fa --- /dev/null +++ b/bindings/java/README.md @@ -0,0 +1,20 @@ +# encoding3d (Java) + +The Java codec of encoding3d (Java 17, Maven): reading and writing frames and sequences (spec 0.3), with one dependency, +Jackson. GroIMP's adapter is in `groimp/`. + +- `src/main/java/org/openalea/encoding3d/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB + container (`Glb`), reading and writing (`Codec`), sequences (`Sequences`), the conformance dump (`Dump`) and tool + (`ConformanceRun`). +- `src/test/`: the tests, the conformance files included. +- `groimp/`: GroIMP's adapter (`org.openalea.encoding3d.groimp.GroImp`, Java 21), on GroIMP's Graph module. + +```bash +mvn -f bindings/java/pom.xml verify +``` + +`org.openalea.encoding3d.coupling` is a participant of coupling sessions (its ZeroMQ transport with JeroMQ, an optional +dependency). + +Licence: GPL-3.0-or-later (`LICENSE`), as GroIMP; the rest of encoding3d is CeCILL-C. See +[the documentation](../../doc/java.md). diff --git a/bindings/java/groimp-plugin/README.md b/bindings/java/groimp-plugin/README.md new file mode 100644 index 0000000..a34daa4 --- /dev/null +++ b/bindings/java/groimp-plugin/README.md @@ -0,0 +1,42 @@ +# GroIMP's encoding3d plugin: a proposal for GroIMP + +**Status: a first proposal, for GroIMP's team to take over** (listing in GroIMP's plugin manager, an import filter). It +lives here so that it could be built and tested before GroIMP's team is contacted; encoding3d's requirements and CI do +not depend on it. + +A GroIMP plugin (`org.openalea.encoding3d`, GroIMP 3.0): **File > Export** of the 3D view to a `.glb` file, or the CLI's +`export3d file.glb`, writes the project as an encoding3d frame: + +- the graph below the project's root: its scales, topology and the nodes' fields (`GroImp.toDataStructure`, + `../groimp`); +- the scene's geometry as one mesh, each triangle carrying the rows of its node and of the node's complexes. + +The shapes are meshed by GroIMP's glTF plugin (`de.grogra.gltf`, from GroIMP's plugin manager or +gitlab.com/grogra/groimp-plugins/gltf), which it requires. + +```bash +mvn -f bindings/java/pom.xml install && mvn -f bindings/java/groimp/pom.xml install +mvn -f bindings/java/groimp-plugin/pom.xml package # the plugin's folder: target/plugin/encoding3d +``` + +To install it, copy `target/plugin/encoding3d` into GroIMP's `plugins/` folder, next to the glTF plugin's. +`test/run_in_groimp.sh` does this in a GroIMP install, grows an example headless, exports it and checks the frame, +then writes GroIMP's log (a frame after each of four runs), which a coupling session replays. To run it by hand +(GroIMP 3.0.0, and GroIMP's glTF plugin at the commit it was last tested with): + +```bash +curl -sSL -o groimp.deb https://download.grogra.de/3_0_0/GroIMP-3.0.0-all.deb +echo "b9b35811ff82482dfa676347639e40fe9601c92105f54095361a9a89f7e85dcf groimp.deb" | sha256sum -c +dpkg-deb -x groimp.deb groimp-install +git clone https://gitlab.com/grogra/groimp-plugins/gltf.git gltf +git -C gltf checkout d093aaebe5197317dfb81fadbe0ef67268bdaf1c +mvn -f gltf/pom.xml install -DskipTests -Dmaven.javadoc.skip=true -Dgroimp.core.version=3.0.0 +mkdir -p gltf-plugin/ext +cp gltf/target/gltf.jar gltf/src/main/resources/plugin.xml gltf/src/main/resources/plugin*.properties gltf-plugin/ +cp gltf/target/lib/ext/*.jar gltf-plugin/ext/ +bash bindings/java/groimp-plugin/test/run_in_groimp.sh groimp-install/usr/share/GroIMP gltf-plugin \ + bindings/java/groimp-plugin/target/plugin/encoding3d out +ENCODING3D_GROIMP_LOG=out/log/scene.fspm.json pytest bindings/python/test/test_coupling.py -k platform_log +``` + +Licence: GPL-3.0-or-later, as GroIMP. diff --git a/bindings/java/groimp-plugin/pom.xml b/bindings/java/groimp-plugin/pom.xml new file mode 100644 index 0000000..d360437 --- /dev/null +++ b/bindings/java/groimp-plugin/pom.xml @@ -0,0 +1,138 @@ + + + + 4.0.0 + + org.openalea + encoding3d-groimp-plugin + 0.1.0 + jar + encoding3d, GroIMP's plugin + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + + + + + 21 + UTF-8 + 3.0.0 + + + + + gitlab-maven + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d-groimp + 0.1.0 + + + de.grogra + gltf + 0.5 + provided + + + de.javagl + jgltf-model + 2.0.4 + provided + + + de.grogra + imp3d + ${groimp.version} + provided + + + de.grogra + platform.core + ${groimp.version} + provided + + + de.grogra + platform + ${groimp.version} + provided + + + de.grogra + graph + ${groimp.version} + provided + + + de.grogra + vecmath + ${groimp.version} + provided + + + + + encoding3d + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + libraries + package + copy-dependencies + + ${project.build.directory}/plugin/encoding3d/ext + runtime + com.fasterxml.jackson.core + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + folder + package + run + + + + + + + + + + + + + + diff --git a/bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java b/bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java new file mode 100644 index 0000000..643c8dc --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java @@ -0,0 +1,182 @@ +package org.openalea.encoding3d.groimp.plugin; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Mesh; +import org.openalea.encoding3d.groimp.GroImp; + +import de.grogra.gltf.exporter.GLTFExport; +import de.grogra.gltf.exporter.GLTFNodeWrapper; +import de.grogra.graph.Graph; +import de.grogra.graph.impl.Node; +import de.grogra.pf.io.FilterItem; +import de.grogra.pf.io.FilterSource; +import de.javagl.jgltf.model.AccessorData; +import de.javagl.jgltf.model.AccessorFloatData; +import de.javagl.jgltf.model.AccessorIntData; +import de.javagl.jgltf.model.AccessorShortData; +import de.javagl.jgltf.model.AccessorByteData; +import de.javagl.jgltf.model.MeshModel; +import de.javagl.jgltf.model.MeshPrimitiveModel; +import de.javagl.jgltf.model.NodeModel; +import de.javagl.jgltf.model.impl.DefaultNodeModel; + +/** + * The 3D view of a GroIMP project as an encoding3d frame (a .glb file): the graph below the project's root (its scales, + * topology and fields, by {@link GroImp#toDataStructure}) and the scene's geometry as one mesh, each triangle carrying + * the rows of its node and of the node's complexes. The shapes are meshed by GroIMP's glTF plugin (its per-shape + * exporters, as for a .gltf file); this class collects their triangles in the scene's frame (GroIMP's: z up), node by + * node, instead of writing glTF. + */ +public class Encoding3dExport extends GLTFExport { + /** The id of no node, for meshes whose node is not an entity (e.g. the scene's ground). */ + private static final long NO_NODE = -1; + + public Encoding3dExport(FilterItem item, FilterSource source) { + super(item, source); + } + + @Override + public void write(File file) throws IOException { + write(); // the glTF plugin's scene walk: one wrapper per node + DefaultNodeModel root = assembleTree(); + Node graphRoot = (Node) getGraphState().getGraph().getRoot(Graph.MAIN_GRAPH); + GroImp.Options options = new GroImp.Options(); + options.id = "plants"; + DataStructure plants = GroImp.toDataStructure(graphRoot, options); + + java.util.Set entities = new java.util.HashSet<>(); + for (EntitySet set : plants.entitySets) for (long id : set.column("id").longs()) entities.add(id); + List positions = new ArrayList<>(); + List nodes = new ArrayList<>(); + collect(root, identity(), NO_NODE, entities, positions, nodes); + if (!positions.isEmpty()) plants.meshes.add(mesh(plants, positions, nodes)); + + Frame frame = new Frame(); + frame.dataStructures.add(plants); + frame.writer.put("platform", "GroIMP").put("adapter", "org.openalea.encoding3d.groimp.plugin"); + frame.vocabularyVersion = "0.2"; + Codec.write(file.toPath(), frame); + } + + /** The glTF plugin's node tree, its wrappers linked as GLTFExport links them (a child under its parent's node). */ + private DefaultNodeModel assembleTree() { + DefaultNodeModel root = new DefaultNodeModel(); + for (String key : nodeRegistry.getKeys()) { + GLTFNodeWrapper wrapper = (GLTFNodeWrapper) nodeRegistry.get(key); + GLTFNodeWrapper parent = wrapper.getParentId() == -1 ? null + : (GLTFNodeWrapper) nodeRegistry.get("" + wrapper.getParentId()); + if (parent != null && parent.getOutNode() != null) parent.addChild(wrapper); + else root.addChild(wrapper.getInNode()); + } + return root; + } + + private static float[] identity() { + return new float[] {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + } + + /** Column-major 4×4 product a·b. */ + private static float[] multiply(float[] a, float[] b) { + float[] product = new float[16]; + for (int column = 0; column < 4; column++) + for (int row = 0; row < 4; row++) { + float sum = 0; + for (int k = 0; k < 4; k++) sum += a[k * 4 + row] * b[column * 4 + k]; + product[column * 4 + row] = sum; + } + return product; + } + + /** The GroIMP node id a glTF node stands for ("12" or "12_b"), else NO_NODE. */ + private static long nodeId(NodeModel node) { + String name = node.getName(); + if (name == null || name.isEmpty()) return NO_NODE; + try { + return Long.parseLong(name.endsWith("_b") ? name.substring(0, name.length() - 2) : name); + } catch (NumberFormatException error) { + return NO_NODE; + } + } + + /** Each triangle of the tree below {@code node}, its vertices in the scene's frame, with the id of its node. A glTF + * node that is no entity (unnamed, or an instance's, whose id the glTF plugin makes up) takes its nearest ancestor's + * entity: an instance's geometry, the node instantiating it. */ + private void collect(NodeModel node, float[] parentTransform, long inheritedId, java.util.Set entities, + List positions, List nodes) { + float[] transform = multiply(parentTransform, node.computeLocalTransform(null)); + long own = nodeId(node); + long id = entities.contains(own) ? own : inheritedId; + for (MeshModel mesh : node.getMeshModels()) + for (MeshPrimitiveModel primitive : mesh.getMeshPrimitiveModels()) { + if (primitive.getMode() != 4 || !primitive.getAttributes().containsKey("POSITION")) continue; + AccessorFloatData points = (AccessorFloatData) primitive.getAttributes().get("POSITION").getAccessorData(); + int[] indices = primitive.getIndices() == null ? null : indices(primitive.getIndices().getAccessorData()); + int count = indices == null ? points.getNumElements() : indices.length; + for (int corner = 0; corner < count; corner++) { + int vertex = indices == null ? corner : indices[corner]; + float x = points.get(vertex, 0), y = points.get(vertex, 1), z = points.get(vertex, 2); + for (int axis = 0; axis < 3; axis++) + positions.add((double) (transform[axis] * x + transform[4 + axis] * y + transform[8 + axis] * z + + transform[12 + axis])); + nodes.add(id); + } + } + for (NodeModel child : node.getChildren()) collect(child, transform, id, entities, positions, nodes); + } + + private static int[] indices(AccessorData data) { + int[] values = new int[data.getNumElements()]; + for (int index = 0; index < values.length; index++) { + if (data instanceof AccessorShortData shorts) values[index] = shorts.getInt(index); + else if (data instanceof AccessorIntData ints) values[index] = ints.get(index); + else if (data instanceof AccessorByteData bytes) values[index] = bytes.getInt(index); + else throw new IllegalArgumentException("indices of type " + data.getComponentType()); + } + return values; + } + + /** The triangles as an encoding3d mesh, three vertices each, with each vertex's row in every entity set: its node's, + * and its node's complexes' up the scales (NO_ID where it has none). */ + private static Mesh mesh(DataStructure plants, List positions, List nodes) { + Map where = new HashMap<>(); // id -> (entity set, row) + Map complexOf = new HashMap<>(); + for (int setIndex = 0; setIndex < plants.entitySets.size(); setIndex++) { + EntitySet set = plants.entitySets.get(setIndex); + long[] ids = set.column("id").longs(); + long[] complexes = set.has("complex") ? set.column("complex").longs() : null; + for (int row = 0; row < ids.length; row++) { + where.put(ids[row], new int[] {setIndex, row}); + if (complexes != null && complexes[row] != EntitySet.NO_ID) complexOf.put(ids[row], complexes[row]); + } + } + double[] points = new double[positions.size()]; + for (int index = 0; index < points.length; index++) points[index] = positions.get(index); + Mesh mesh = new Mesh("scene", points); + long[][] rows = new long[plants.entitySets.size()][nodes.size()]; + for (long[] setRows : rows) java.util.Arrays.fill(setRows, EntitySet.NO_ID); + for (int vertex = 0; vertex < nodes.size(); vertex++) { + Long id = nodes.get(vertex); + while (id != null && id != NO_NODE && where.containsKey(id)) { // the node, then its complexes + int[] place = where.get(id); + rows[place[0]][vertex] = place[1]; + id = complexOf.get(id); + } + } + for (int setIndex = 0; setIndex < rows.length; setIndex++) { + boolean any = false; + for (long row : rows[setIndex]) any |= row != EntitySet.NO_ID; + if (any) mesh.featureRows.put(plants.entitySets.get(setIndex).name, rows[setIndex]); + } + return mesh; + } +} diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.properties b/bindings/java/groimp-plugin/src/main/resources/plugin.properties new file mode 100644 index 0000000..63a920b --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.properties @@ -0,0 +1,4 @@ +pluginName = encoding3d +provider = OpenAlea (encoding3d) +/io/filetypes/encoding3d.Name = encoding3d frame (*.glb) +/io/mimetypes/application/x-java-jvm-local-objectref/encoding3dExport.Name = encoding3d frame diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.xml b/bindings/java/groimp-plugin/src/main/resources/plugin.xml new file mode 100644 index 0000000..013a83d --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bindings/java/groimp-plugin/test/run_in_groimp.sh b/bindings/java/groimp-plugin/test/run_in_groimp.sh new file mode 100755 index 0000000..a112bd5 --- /dev/null +++ b/bindings/java/groimp-plugin/test/run_in_groimp.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# GroIMP's encoding3d plugin in GroIMP itself, headless: install it (and GroIMP's glTF plugin, which meshes the shapes) +# into a GroIMP 3.0 install, grow one of GroIMP's examples, export its 3D view as an encoding3d frame, and check the file: +# valid encoding3d (encoding3d validate), every triangle with its node. Then GroIMP's log: the example grown again, a frame +# exported after each of four runs, with their index (OUT/log/scene.fspm.json), which a coupling session replays +# (bindings/python/test/test_coupling.py, ENCODING3D_GROIMP_LOG). +# +# run_in_groimp.sh GROIMP GLTF_PLUGIN ENCODING3D_PLUGIN OUT +# GROIMP a GroIMP 3.0 install (the folder holding core.jar and plugins/) +# GLTF_PLUGIN GroIMP's glTF plugin's folder (plugin.xml, gltf.jar, ext/) +# ENCODING3D_PLUGIN this plugin's folder (target/plugin/encoding3d) +# OUT where the frame is written +set -euo pipefail +groimp=$(realpath "$1"); gltf=$(realpath "$2"); encoding3d=$(realpath "$3"); out=$(realpath -m "$4") +mkdir -p "$out" "$groimp/plugins/gltf" "$groimp/plugins/encoding3d" "$out/home" +cp -r "$gltf/." "$groimp/plugins/gltf/" +cp -r "$encoding3d/." "$groimp/plugins/encoding3d/" +frame="$out/rapeseed.glb" +rm -f "$frame" +cd "$groimp" +# the export runs in the background: the commands' input stays open while it writes +(printf '$loadExample simpleRapeseed\nrun\nrun\nrun\nexport3d %s\n' "$frame"; sleep 30; printf 'listFunctions\n') | + java -Xmx1g -Djava.util.prefs.userRoot="$out/home" -jar core.jar -a cli > "$out/groimp.log" 2>&1 +test -s "$frame" || { echo "no frame written"; tail -40 "$out/groimp.log"; exit 1; } +encoding3d validate "$frame" +python - "$frame" <<'PYTHON' +import sys +import numpy as np +from openalea import encoding3d +plants = encoding3d.read(sys.argv[1]).data_structure("plants") +mesh = plants.meshes[0] +rows = mesh.feature_rows[plants.entity_sets[-1].name] +assert len(mesh.positions) > 0 and (rows >= 0).all(), "every vertex carries its node" +assert mesh.positions[:, 2].max() > 0, "the plant stands, z up" +print(f"{len(mesh.positions)} vertices, {len(np.unique(rows))} nodes drawn, of {plants.entity_sets[-1].count}") +PYTHON + +# GroIMP's log: a frame after each run +log="$out/log" +mkdir -p "$log" +(printf '$loadExample simpleRapeseed\n'; sleep 10 + for step in 0 1 2 3; do printf 'run\nexport3d %s/frame_%06d.glb\n' "$log" "$step"; sleep 25; done + printf 'listFunctions\n') | + java -Xmx1g -Djava.util.prefs.userRoot="$out/home" -jar core.jar -a cli > "$out/groimp_log.log" 2>&1 +python - "$log" <<'PYTHON' +import os +import sys +from openalea.encoding3d import read, write, write_index +folder = sys.argv[1] +files = sorted(name for name in os.listdir(folder) if name.endswith(".glb")) +assert len(files) == 4, f"four frames expected, {files}" +for step, name in enumerate(files): # GroIMP has no simulation clock: the log's time and step, its run count + frame = read(os.path.join(folder, name)) + frame.time, frame.step = float(step), step + write(os.path.join(folder, name), frame) +write_index(os.path.join(folder, "scene.fspm.json"), + [{"time": float(step), "step": step, "file": name} for step, name in enumerate(files)], time_unit="d") +print(f"GroIMP's log: {len(files)} frames in {folder}") +PYTHON +encoding3d validate "$log/scene.fspm.json" diff --git a/bindings/java/groimp/README.md b/bindings/java/groimp/README.md new file mode 100644 index 0000000..4081d62 --- /dev/null +++ b/bindings/java/groimp/README.md @@ -0,0 +1,29 @@ +# GroIMP's adapter: a proposal for GroIMP + +**Status: a first proposal, for GroIMP's team to take over.** It lives here so that the Java binding could be built +and tested before GroIMP's team is contacted. Once they adopt it, it moves into GroIMP (or GroIMP's plugins), with its +tests and CI. encoding3d itself does not depend on it: the Java codec (`bindings/java`) builds without it, and +encoding3d's CI does not run it. + +`org.openalea.encoding3d.groimp.GroImp`, on GroIMP's Graph module only (no GroIMP application): + +- `toDataStructure(root, options)`: a GroIMP graph (its nodes, their scales, successor and branch edges, their fields) + as an encoding3d graph data structure; +- `toGraph(dataStructure, nodes, idsOf)`: back, each row built as a node of the model's own classes. + +GroIMP's plugin, which exports a GroIMP project with its geometry, is in `../groimp-plugin` (its own README). + +## Building and testing it by hand + +GroIMP's modules (graph, platform, vecmath, …, 3.0.0) come from GroIMP's package registry, declared in `pom.xml`. + +```bash +mvn -f bindings/java/pom.xml install -DskipTests # the codec +mvn -f bindings/java/groimp/pom.xml verify # the adapter and its tests +encoding3d validate bindings/java/groimp/target/groimp_out/*.glb +``` + +The tests (`src/test/java/.../GroImpTest.java`) export a small GroIMP plant at several scales, read it back, rebuild it +as GroIMP nodes, and read another platform's frame. + +Licence: GPL-3.0-or-later, as GroIMP. diff --git a/bindings/java/groimp/pom.xml b/bindings/java/groimp/pom.xml new file mode 100644 index 0000000..fa65865 --- /dev/null +++ b/bindings/java/groimp/pom.xml @@ -0,0 +1,81 @@ + + + + 4.0.0 + + org.openalea + encoding3d-groimp + 0.1.0 + jar + encoding3d, GroIMP's adapter + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + + + + + 21 + UTF-8 + 3.0.0 + 5.11.4 + + + + + gitlab-maven + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d + 0.1.0 + + + de.grogra + graph + ${groimp.version} + provided + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + false + + ${project.basedir}/../../../conformance + ${project.build.directory}/groimp_out + + + + + + diff --git a/bindings/java/groimp/src/main/java/org/openalea/encoding3d/groimp/GroImp.java b/bindings/java/groimp/src/main/java/org/openalea/encoding3d/groimp/GroImp.java new file mode 100644 index 0000000..44f5485 --- /dev/null +++ b/bindings/java/groimp/src/main/java/org/openalea/encoding3d/groimp/GroImp.java @@ -0,0 +1,465 @@ +package org.openalea.encoding3d.groimp; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import org.openalea.encoding3d.Column; +import org.openalea.encoding3d.ComponentType; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Encoding3dException; +import org.openalea.encoding3d.Relation; +import org.openalea.encoding3d.Variable; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.Edge; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ManageableType; +import de.grogra.reflect.TypeId; + +/** + * A GroIMP graph as an encoding3d graph, and back. + * + *

Status: a first proposal for GroIMP's team to take over, not a part of encoding3d: encoding3d's requirements and CI do not depend on it; it is built and + * tested by hand (bindings/java/groimp/README.md), and moves into GroIMP once its team adopts it. + * + *

+ * + * + * + * + * + * + * + * + *
encoding3dGroIMP
one entity set per scale (rank: the scale, from 1)the nodes at that refinement depth below the + * root; a node reached by a successor or branch edge has its parent's scale, by a refinement edge one more
id{@code Node.getId()}
labelthe node's class's simple name
complexthe source of its incoming refinement edge, else its parent's complex
parent, edge_typethe source of its incoming successor (successor) or branch (branch) edge at the same + * scale
relationsthe other edge bits between exported nodes (containment, mark, notifies, user edges), one + * relation per bit and pair of entity sets
variablesthe nodes' managed fields declared below {@code Node} (numbers, booleans, chars and Strings; + * 3-component vecmath tuples as vectors)
+ * + *

The graph's root (e.g. an RGG root) is the data structure itself, not an entity. What encoding3d 0.1 cannot say is + * kept in {@code extras["groimp"]}: the classes' names. A field that other classes of the same scale lack is missing in + * their rows.

+ */ +public final class GroImp { + /** The name of GroIMP's entry in a data structure's extras. */ + public static final String PLATFORM = "groimp"; + private static final int TREE_EDGES = Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE | Graph.REFINEMENT_EDGE; + private static final Map RELATION_NAMES = Map.of(Graph.CONTAINMENT_EDGE, "containment", + Graph.CONTAINMENT_END_EDGE, "containment_end", Graph.MARK_EDGE, "mark", Graph.NOTIFIES_EDGE, "notifies"); + + /** How a graph is written. */ + public static final class Options { + /** The data structure's id. */ + public String id = "plants"; + /** The entity set name of each scale (1: the coarsest); by default its nodes' class name when they share one, + * else {@code Scale}. */ + public final Map scaleNames = new HashMap<>(); + /** The rank of each scale's entity set (by default its refinement depth); e.g. the vocabulary's base scales' + * positions (Plant 1, Axis 2, Organ 5, SubOrgan 6). */ + public final Map scaleRanks = new HashMap<>(); + /** The base scale of each scale's entity set (e.g. "Organ"), or "mixed". */ + public final Map scaleScales = new HashMap<>(); + /** The model's class names to the vocabulary's labels (e.g. Internode to StemInternode). */ + public final Map labels = new LinkedHashMap<>(); + /** The declarations of the fields (by name): unit, standard name, kind, role. */ + public final Map variables = new HashMap<>(); + public String model; + } + + private GroImp() {} + + /** A node's place in the exported graph. */ + private static final class Entity { + final Node node; + final int scale; + Node complex; + Node parent; + String edgeType; + + Entity(Node node, int scale, Node complex, Node parent, String edgeType) { + this.node = node; + this.scale = scale; + this.complex = complex; + this.parent = parent; + this.edgeType = edgeType; + } + } + + /** The graph below {@code root} (not included) as a {@code graph} data structure. */ + public static DataStructure toDataStructure(Node root, Options options) { + // Depth first along successor, branch and refinement edges, each node once, children in edge order + Map entities = new IdentityHashMap<>(); + List order = new ArrayList<>(); + ArrayDeque stack = new ArrayDeque<>(); + List rootChildren = children(new Entity(root, 0, null, null, "none"), root, true); + for (int index = rootChildren.size() - 1; index >= 0; index--) stack.push(rootChildren.get(index)); + while (!stack.isEmpty()) { + Entity entity = stack.pop(); + if (entities.containsKey(entity.node)) continue; + if (entity.node.getId() < 0) + throw new Encoding3dException("node " + entity.node + " has no id: it is not in a graph"); + entities.put(entity.node, entity); + order.add(entity); + List next = children(entity, root, false); + for (int index = next.size() - 1; index >= 0; index--) stack.push(next.get(index)); + } + + // Complex and parent from each node's incoming edges, whichever edge reached it first: a lateral's first + // component has both a refinement edge from its complex and a branch edge from its bearer + for (Entity entity : order) { + Node node = entity.node; + Node complex = null, parent = null; + String edgeType = "none"; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getTarget() != node || !entities.containsKey(edge.getSource())) continue; + int bits = edge.getEdgeBits(); + if ((bits & Graph.REFINEMENT_EDGE) != 0) complex = edge.getSource(); + else if ((bits & (Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE)) != 0 + && entities.get(edge.getSource()).scale == entity.scale) { + parent = edge.getSource(); + edgeType = (bits & Graph.SUCCESSOR_EDGE) != 0 ? "successor" : "branch"; + } + } + entity.complex = complex; + entity.parent = parent; + entity.edgeType = edgeType; + } + for (Entity entity : order) { // no refinement edge: its parent's complex (MTG's rule) + Entity ancestor = entity; + while (ancestor.complex == null && ancestor.parent != null) ancestor = entities.get(ancestor.parent); + entity.complex = ancestor.complex; + } + + Map> byScale = new java.util.TreeMap<>(); + for (Entity entity : order) byScale.computeIfAbsent(entity.scale, scale -> new ArrayList<>()).add(entity); + Map setNames = new HashMap<>(); + for (Map.Entry> item : byScale.entrySet()) { + Set classes = new LinkedHashSet<>(); + for (Entity entity : item.getValue()) classes.add(entity.node.getClass().getSimpleName()); + setNames.put(item.getKey(), options.scaleNames.getOrDefault(item.getKey(), + classes.size() == 1 ? classes.iterator().next() : "Scale" + item.getKey())); + } + + DataStructure dataStructure = new DataStructure(options.id, "graph"); + dataStructure.platform = "GroIMP"; + dataStructure.model = options.model; + ObjectNode extras = JsonNodeFactory.instance.objectNode(); + ObjectNode classes = extras.putObject("classes"); + int coarsest = byScale.isEmpty() ? 0 : byScale.keySet().iterator().next(); + for (Map.Entry> item : byScale.entrySet()) { + String setName = setNames.get(item.getKey()); + List rows = item.getValue(); + EntitySet set = new EntitySet(setName, options.scaleRanks.getOrDefault(item.getKey(), (long) item.getKey())); + set.scale = options.scaleScales.get(item.getKey()); + long[] ids = new long[rows.size()], complexes = new long[rows.size()], parents = new long[rows.size()]; + String[] labels = new String[rows.size()], edgeTypes = new String[rows.size()]; + for (int row = 0; row < rows.size(); row++) { + Entity entity = rows.get(row); + ids[row] = entity.node.getId(); + labels[row] = entity.node.getClass().getSimpleName(); + complexes[row] = entity.complex == null ? EntitySet.NO_ID : entity.complex.getId(); + parents[row] = entity.parent == null ? EntitySet.NO_ID : entity.parent.getId(); + edgeTypes[row] = entity.edgeType; + classes.put(labels[row], entity.node.getClass().getName()); + } + set.set("id", Column.of(ids)).set("label", Column.of(labels)); + if (item.getKey() != coarsest) set.set("complex", Column.of(complexes)); + set.set("parent", Column.of(parents)).set("edge_type", Column.of(edgeTypes)); + addFields(set, rows, options); + dataStructure.entitySets.add(set); + } + addRelations(dataStructure, order, entities, setNames); + dataStructure.extras.set(PLATFORM, extras); + dataStructure.labels.putAll(options.labels); + return dataStructure; + } + + /** The nodes {@code entity}'s outgoing tree edges lead to, as entities. */ + private static List children(Entity entity, Node root, boolean fromRoot) { + List found = new ArrayList<>(); + Node node = entity.node; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getSource() != node) continue; + int bits = edge.getEdgeBits(); + if ((bits & TREE_EDGES) == 0) continue; + Node target = edge.getTarget(); + if (target == root) continue; + if (fromRoot) { // the root's children: the coarsest scale's first nodes + found.add(new Entity(target, 1, null, null, "none")); + } else if ((bits & Graph.REFINEMENT_EDGE) != 0) { // one scale finer: its complex + found.add(new Entity(target, entity.scale + 1, node, null, "none")); + } else { // the same scale: its parent + found.add(new Entity(target, entity.scale, entity.complex, node, + (bits & Graph.SUCCESSOR_EDGE) != 0 ? "successor" : "branch")); + } + } + return found; + } + + private static boolean isOwnField(ManageableType.Field field) { + return field.getDeclaringType() != Node.$TYPE && (field.getModifiers() & ManageableType.Field.HIDDEN) == 0; + } + + /** A field's value, as a column value: Double, Float, Long, Integer, Short, Byte, Boolean, String, double[3]; null + * when not written. */ + private static Object fieldValue(ManageableType.Field field, Node node) { + return switch (field.getType().getTypeId()) { + case TypeId.BOOLEAN -> field.getBoolean(node); + case TypeId.BYTE -> field.getByte(node); + case TypeId.SHORT -> field.getShort(node); + case TypeId.CHAR -> String.valueOf(field.getChar(node)); + case TypeId.INT -> field.getInt(node); + case TypeId.LONG -> field.getLong(node); + case TypeId.FLOAT -> field.getFloat(node); + case TypeId.DOUBLE -> field.getDouble(node); + case TypeId.OBJECT -> objectValue(field.getObject(node)); + default -> null; + }; + } + + /** Strings as they are, vecmath's Tuple3d and Tuple3f (public x, y, z) as 3 doubles; anything else is not written. */ + private static Object objectValue(Object value) { + if (value instanceof String) return value; + if (value == null) return null; + for (Class type = value.getClass(); type != null; type = type.getSuperclass()) + if (type.getName().equals("javax.vecmath.Tuple3d") || type.getName().equals("javax.vecmath.Tuple3f")) { + try { + double[] vector = new double[3]; + String[] names = {"x", "y", "z"}; + for (int axis = 0; axis < 3; axis++) + vector[axis] = ((Number) type.getField(names[axis]).get(value)).doubleValue(); + return vector; + } catch (ReflectiveOperationException error) { + return null; + } + } + return null; + } + + /** The rows' managed fields as columns (in order of first appearance), missing where a row's class has none. */ + private static void addFields(EntitySet set, List rows, Options options) { + Map values = new LinkedHashMap<>(); + for (int row = 0; row < rows.size(); row++) { + Node node = rows.get(row).node; + ManageableType type = node.getNType(); + for (int index = 0; index < type.getManagedFieldCount(); index++) { + ManageableType.Field field = type.getManagedField(index); + if (!isOwnField(field) || set.isReserved(field.getName())) continue; + Object value = fieldValue(field, node); + if (value == null) continue; + values.computeIfAbsent(field.getName(), name -> new Object[rows.size()])[row] = value; + } + } + for (Map.Entry item : values.entrySet()) { + Column column = column(item.getValue()); + if (column == null) continue; // values of several kinds: not written + boolean[] absent = new boolean[item.getValue().length]; + boolean any = false; + for (int row = 0; row < absent.length; row++) any |= absent[row] = item.getValue()[row] == null; + if (any && column.width == 1) column.withMissing(absent); // a missing vector: NaN + set.set(item.getKey(), column); + Variable declared = options.variables.get(item.getKey()); + if (declared != null) set.declare(item.getKey(), declared); + } + } + + /** A column of values of one kind (null where a row has none: NaN, 0, false or ""); null for mixed kinds. */ + private static Column column(Object[] values) { + Class kind = null; + for (Object value : values) + if (value != null) { + if (kind != null && kind != value.getClass()) return null; + kind = value.getClass(); + } + int rows = values.length; + if (kind == Double.class) { + double[] column = new double[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? Double.NaN : (Double) values[row]; + return Column.of(column); + } else if (kind == Float.class) { + float[] column = new float[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? Float.NaN : (Float) values[row]; + return Column.of(column); + } else if (kind == Long.class) { + long[] column = new long[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Long) values[row]; + return Column.of(column); + } else if (kind == Integer.class) { + int[] column = new int[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Integer) values[row]; + return Column.of(column); + } else if (kind == Short.class) { + short[] column = new short[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Short) values[row]; + return new Column(ComponentType.INT16, column); + } else if (kind == Byte.class) { + byte[] column = new byte[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Byte) values[row]; + return new Column(ComponentType.INT8, column); + } else if (kind == Boolean.class) { + boolean[] column = new boolean[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] != null && (Boolean) values[row]; + return Column.of(column); + } else if (kind == String.class) { + String[] column = new String[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? "" : (String) values[row]; + return Column.of(column); + } else if (kind == double[].class) { + double[] column = new double[rows * 3]; + for (int row = 0; row < rows; row++) + for (int axis = 0; axis < 3; axis++) + column[row * 3 + axis] = values[row] == null ? Double.NaN : ((double[]) values[row])[axis]; + return Column.vectors(3, column); + } + return null; + } + + /** The other edge bits between exported nodes (also on tree edges): one relation per bit and pair of sets. */ + private static void addRelations(DataStructure dataStructure, List order, Map entities, + Map setNames) { + Map> edges = new LinkedHashMap<>(); + Map ends = new HashMap<>(); + for (Entity entity : order) { + Node node = entity.node; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getSource() != node || !entities.containsKey(edge.getTarget())) continue; + int others = edge.getEdgeBits() & ~TREE_EDGES; + for (int bit = 1; bit != 0 && others != 0; bit <<= 1) { + if ((others & bit) == 0) continue; + others &= ~bit; + String kind = RELATION_NAMES.getOrDefault(bit, "edge_" + Integer.toHexString(bit)); + String source = setNames.get(entity.scale), target = setNames.get(entities.get(edge.getTarget()).scale); + String key = kind + "|" + source + "|" + target; + edges.computeIfAbsent(key, k -> new ArrayList<>()).add(new long[] {node.getId(), edge.getTarget().getId()}); + ends.put(key, new String[] {kind, source, target}); + } + } + } + Map kindCount = new HashMap<>(); + for (String[] end : ends.values()) kindCount.merge(end[0], 1, Integer::sum); + for (Map.Entry> item : edges.entrySet()) { + String[] end = ends.get(item.getKey()); + String name = kindCount.get(end[0]) == 1 ? end[0] : end[0] + "_" + end[1] + "_" + end[2]; + Relation relation = new Relation(name, end[1], end[2]); + long[] sources = new long[item.getValue().size()], targets = new long[item.getValue().size()]; + for (int index = 0; index < sources.length; index++) { + sources[index] = item.getValue().get(index)[0]; + targets[index] = item.getValue().get(index)[1]; + } + relation.set("source", Column.of(sources)).set("target", Column.of(targets)); + dataStructure.relations.add(relation); + } + } + + // ── Data structure → graph ────────────────────────────────────────── + + /** + * A graph data structure as GroIMP nodes below a new root: each entity a node made by {@code nodes} from its label + * (e.g. {@code label -> new Internode()}), its fields set from its row, linked by refinement edges from its complex + * (the first of each complex's components only, as GroIMP's multiscale graphs), successor or branch edges from its + * parent, and its relations' edge bits. The root's children: the coarsest scale's entities without parent. Returns + * the root; GroIMP gives the nodes new ids when the root is put in a graph, {@code idsOf} maps each entity's id to + * its node. + */ + public static Node toGraph(DataStructure dataStructure, Function nodes, Map idsOf) { + List sets = new ArrayList<>(); + for (EntitySet set : dataStructure.entitySets) if (set.rank != null && set.has("id")) sets.add(set); + sets.sort((left, right) -> Long.compare(left.rank, right.rank)); + Node root = new Node(); + for (EntitySet set : sets) { + long[] ids = set.column("id").longs(); + String[] labels = set.has("label") ? set.column("label").strings() : null; + for (int row = 0; row < ids.length; row++) { + Node node = nodes.apply(labels == null ? set.name : labels[row]); + if (node == null) throw new Encoding3dException("no node made for label '" + labels[row] + "'"); + setFields(node, set, row); + idsOf.put(ids[row], node); + } + } + Set refined = new java.util.HashSet<>(); // complexes already linked to a first component + for (EntitySet set : sets) { + long[] ids = set.column("id").longs(); + long[] parents = set.has("parent") ? set.column("parent").longs() : null; + String[] edgeTypes = set.has("edge_type") ? set.column("edge_type").strings() : null; + long[] complexes = set.has("complex") ? set.column("complex").longs() : null; + for (int row = 0; row < ids.length; row++) { + Node node = idsOf.get(ids[row]); + long parent = parents == null ? EntitySet.NO_ID : parents[row]; + long complex = complexes == null ? EntitySet.NO_ID : complexes[row]; + if (parent != EntitySet.NO_ID) { + idsOf.get(parent).addEdgeBitsTo(node, + "successor".equals(edgeTypes[row]) ? Graph.SUCCESSOR_EDGE : Graph.BRANCH_EDGE, null); + Long parentComplex = complexOf(sets, parent); + if (complex != EntitySet.NO_ID && (parentComplex == null || parentComplex != complex) + && refined.add(complex)) // the first component of its complex + idsOf.get(complex).addEdgeBitsTo(node, Graph.REFINEMENT_EDGE, null); + } else if (complex != EntitySet.NO_ID) { + idsOf.get(complex).addEdgeBitsTo(node, Graph.REFINEMENT_EDGE, null); + refined.add(complex); + } else { + root.addEdgeBitsTo(node, Graph.BRANCH_EDGE, null); + } + } + } + for (Relation relation : dataStructure.relations) { + int bit = 0; + for (Map.Entry item : RELATION_NAMES.entrySet()) + if (relation.name.equals(item.getValue()) || relation.name.startsWith(item.getValue() + "_")) bit = item.getKey(); + if (bit == 0 && relation.name.startsWith("edge_")) { + String hex = relation.name.substring(5).split("_")[0]; + bit = Integer.parseInt(hex, 16); + } + if (bit == 0) continue; // another platform's relation: not an edge bit + long[] sources = relation.column("source").longs(), targets = relation.column("target").longs(); + for (int index = 0; index < sources.length; index++) + idsOf.get(sources[index]).addEdgeBitsTo(idsOf.get(targets[index]), bit, null); + } + return root; + } + + private static Long complexOf(List sets, long id) { + for (EntitySet set : sets) { + if (!set.has("complex")) continue; + long[] ids = set.column("id").longs(); + for (int row = 0; row < ids.length; row++) if (ids[row] == id) return set.column("complex").longs()[row]; + } + return null; + } + + private static void setFields(Node node, EntitySet set, int row) { + ManageableType type = node.getNType(); + for (String name : set.variableNames()) { + ManageableType.Field field = type.getManagedField(name); + if (field == null || !isOwnField(field)) continue; + Column column = set.column(name); + if (column.isMissing(row)) continue; // the field keeps its class's default + switch (field.getType().getTypeId()) { + case TypeId.BOOLEAN -> field.setBoolean(node, column.booleans()[row]); + case TypeId.BYTE -> field.setByte(node, (byte) column.doubleAt(row)); + case TypeId.SHORT -> field.setShort(node, (short) column.doubleAt(row)); + case TypeId.CHAR -> field.setChar(node, column.strings()[row].charAt(0)); + case TypeId.INT -> field.setInt(node, (int) column.doubleAt(row)); + case TypeId.LONG -> field.setLong(node, (long) column.doubleAt(row)); + case TypeId.FLOAT -> field.setFloat(node, (float) column.doubleAt(row)); + case TypeId.DOUBLE -> field.setDouble(node, column.doubleAt(row)); + case TypeId.OBJECT -> { + if (column.type == ComponentType.STRING) field.setObject(node, column.strings()[row]); + } + default -> { } + } + } + } +} diff --git a/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/GroImpTest.java b/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/GroImpTest.java new file mode 100644 index 0000000..34cc3dd --- /dev/null +++ b/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/GroImpTest.java @@ -0,0 +1,129 @@ +package org.openalea.encoding3d.groimp; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Variable; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** GroIMP's adapter: a multiscale GroIMP graph (plant, internodes and leaves) exported, written, read and rebuilt. */ +class GroImpTest { + static final Path OUT = Path.of(System.getProperty("encoding3d.out", "target/groimp_out")); + + /** root -> Plant /-> Internode < Internode, each internode bearing a leaf (+); a mark edge from a leaf to the plant. */ + static Node plant(GraphManager graph) { + Node root = new Node(); + Organs.Plant plant = new Organs.Plant(); + plant.species = "Wheat"; + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Organs.Internode first = new Organs.Internode(), second = new Organs.Internode(); + first.length = 0.12; first.radius = 0.004f; first.rank = 1; + second.length = 0.08; second.radius = 0.003f; second.rank = 2; + Organs.Leaf firstLeaf = new Organs.Leaf(), secondLeaf = new Organs.Leaf(); + firstLeaf.length = 0.2; secondLeaf.length = 0.15; secondLeaf.alive = false; + plant.addEdgeBitsTo(first, Graph.REFINEMENT_EDGE, null); + first.addEdgeBitsTo(second, Graph.SUCCESSOR_EDGE, null); + first.addEdgeBitsTo(firstLeaf, Graph.BRANCH_EDGE, null); + second.addEdgeBitsTo(secondLeaf, Graph.BRANCH_EDGE, null); + firstLeaf.addEdgeBitsTo(plant, Graph.MARK_EDGE, null); + graph.setRoot(Graph.MAIN_GRAPH, root); + return root; + } + + static GraphManager newGraph() { + return new GraphManager(new ServerConnection(null), "test", false, false); + } + + @Test + void exportedWrittenAndRebuilt() throws IOException { + Node root = plant(newGraph()); + GroImp.Options options = new GroImp.Options(); + options.scaleNames.put(2, "Organ"); + options.scaleRanks.put(2, 5L); // the vocabulary's Organ + options.scaleScales.put(2, "Organ"); + options.labels.put("Internode", "StemInternode"); + options.variables.put("length", new Variable("m", "length", "extensive", "state")); + DataStructure written = GroImp.toDataStructure(root, options); + + assertEquals(List.of("Plant", "Organ"), written.entitySets.stream().map(set -> set.name).toList()); + EntitySet plants = written.entitySet("Plant"), organs = written.entitySet("Organ"); + assertArrayEquals(new String[] {"Wheat"}, plants.column("species").strings()); + assertEquals(5L, organs.rank); + assertArrayEquals(new String[] {"Internode", "Internode", "Leaf", "Leaf"}, organs.column("label").strings()); + long[] ids = organs.column("id").longs(); + long plantId = plants.column("id").longs()[0]; + assertArrayEquals(new long[] {plantId, plantId, plantId, plantId}, organs.column("complex").longs()); + assertArrayEquals(new long[] {-1, ids[0], ids[1], ids[0]}, organs.column("parent").longs()); + assertArrayEquals(new String[] {"none", "successor", "branch", "branch"}, organs.column("edge_type").strings()); + double[] lengths = organs.column("length").doubles(); + assertArrayEquals(new double[] {0.12, 0.08, 0.15, 0.2}, lengths); + assertTrue(Float.isNaN(organs.column("radius").floats()[2])); // leaves have no radius + assertArrayEquals(new boolean[] {false, false, false, true}, organs.column("alive").booleans()); + assertEquals("mark", written.relations.get(0).name); + assertArrayEquals(new long[] {ids[3]}, written.relations.get(0).column("source").longs()); + assertArrayEquals(new boolean[] {true, true, false, false}, organs.column("alive").missing); // internodes + assertArrayEquals(new boolean[] {false, false, true, true}, organs.column("rank").missing); // leaves + assertEquals("Organ", organs.scale); + assertEquals("StemInternode", written.labels.get("Internode")); + assertTrue(!written.extras.get("groimp").has("missing")); + + Files.createDirectories(OUT); + Frame frame = new Frame(); + frame.dataStructures.add(written); + Codec.write(OUT.resolve("groimp_plant.glb"), frame); + DataStructure readBack = Codec.read(OUT.resolve("groimp_plant.glb")).dataStructures.get(0); + + Map nodes = new HashMap<>(); + Map> classes = Map.of("Plant", Organs.Plant::new, + "Internode", Organs.Internode::new, "Leaf", Organs.Leaf::new); + Node rebuilt = GroImp.toGraph(readBack, label -> classes.get(label).get(), nodes); + newGraph().setRoot(Graph.MAIN_GRAPH, rebuilt); + DataStructure again = GroImp.toDataStructure(rebuilt, options); + EntitySet againOrgans = again.entitySet("Organ"); + assertArrayEquals(organs.column("label").strings(), againOrgans.column("label").strings()); + assertArrayEquals(lengths, againOrgans.column("length").doubles()); + assertArrayEquals(organs.column("edge_type").strings(), againOrgans.column("edge_type").strings()); + assertArrayEquals(organs.column("alive").booleans(), againOrgans.column("alive").booleans()); + assertArrayEquals(organs.column("alive").missing, againOrgans.column("alive").missing); + assertEquals(0.004f, ((Organs.Internode) nodes.get(ids[0])).radius); + assertEquals("Wheat", ((Organs.Plant) nodes.get(plantId)).species); + assertEquals(1, again.relations.size()); + assertNull(again.entitySet("Plant").has("complex") ? "complex" : null); + } + + @Test + void anotherPlatformsGraph() throws IOException { + // The conformance root system (metafspm: Plant, Axis, SubOrgan) as GroIMP nodes: three scales by refinement + Path conformance = Path.of(System.getProperty("encoding3d.conformance", "../../../conformance")); + DataStructure roots = Codec.read(conformance.resolve("root_system.glb")).dataStructures.get(0); + Node root = GroImp.toGraph(roots, label -> new Node(), new HashMap<>()); + newGraph().setRoot(Graph.MAIN_GRAPH, root); + DataStructure again = GroImp.toDataStructure(root, new GroImp.Options()); + assertEquals(3, again.entitySets.size()); + for (int scale = 0; scale < 3; scale++) { + EntitySet written = roots.entitySets.get(scale), read = again.entitySets.get(scale); + assertEquals(written.rowCount(), read.rowCount(), written.name); + if (written.has("edge_type")) + assertEquals(List.of(written.column("edge_type").strings()).stream().sorted().toList(), + List.of(read.column("edge_type").strings()).stream().sorted().toList(), written.name); + } + } +} diff --git a/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/Organs.java b/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/Organs.java new file mode 100644 index 0000000..461fc62 --- /dev/null +++ b/bindings/java/groimp/src/test/java/org/openalea/encoding3d/groimp/Organs.java @@ -0,0 +1,168 @@ +package org.openalea.encoding3d.groimp; + +import de.grogra.graph.impl.Node; + +/** Node classes as GroIMP models declare them (managed fields, as the XL compiler generates them): a plant, internodes + * and leaves. */ +final class Organs { + private Organs() {} + + static final class Plant extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + String species = ""; + + static { + $TYPE = new NType(Plant.class); + $TYPE.addManagedField(new Field("species", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.STRING, null, 0)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Plant.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + protected void setObjectImpl(Object o, Object value) { + if (id == 0) ((Plant) o).species = (String) value; else super.setObjectImpl(o, value); + } + + @Override + public Object getObject(Object o) { + return id == 0 ? ((Plant) o).species : super.getObject(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Plant(); + } + } + + static final class Internode extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + double length; + float radius; + int rank; + + static { + $TYPE = new NType(Internode.class); + $TYPE.addManagedField(new Field("length", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.DOUBLE, null, 0)); + $TYPE.addManagedField(new Field("radius", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.FLOAT, null, 1)); + $TYPE.addManagedField(new Field("rank", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.INT, null, 2)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Internode.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + public void setDouble(Object o, double value) { + if (id == 0) ((Internode) o).length = value; else super.setDouble(o, value); + } + + @Override + public double getDouble(Object o) { + return id == 0 ? ((Internode) o).length : super.getDouble(o); + } + + @Override + public void setFloat(Object o, float value) { + if (id == 1) ((Internode) o).radius = value; else super.setFloat(o, value); + } + + @Override + public float getFloat(Object o) { + return id == 1 ? ((Internode) o).radius : super.getFloat(o); + } + + @Override + public void setInt(Object o, int value) { + if (id == 2) ((Internode) o).rank = value; else super.setInt(o, value); + } + + @Override + public int getInt(Object o) { + return id == 2 ? ((Internode) o).rank : super.getInt(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Internode(); + } + } + + static final class Leaf extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + double length; + boolean alive = true; + + static { + $TYPE = new NType(Leaf.class); + $TYPE.addManagedField(new Field("length", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.DOUBLE, null, 0)); + $TYPE.addManagedField(new Field("alive", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.BOOLEAN, null, 1)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Leaf.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + public void setDouble(Object o, double value) { + if (id == 0) ((Leaf) o).length = value; else super.setDouble(o, value); + } + + @Override + public double getDouble(Object o) { + return id == 0 ? ((Leaf) o).length : super.getDouble(o); + } + + @Override + public void setBoolean(Object o, boolean value) { + if (id == 1) ((Leaf) o).alive = value; else super.setBoolean(o, value); + } + + @Override + public boolean getBoolean(Object o) { + return id == 1 ? ((Leaf) o).alive : super.getBoolean(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Leaf(); + } + } +} diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml new file mode 100644 index 0000000..869ba04 --- /dev/null +++ b/bindings/java/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + org.openalea + encoding3d + 0.1.0 + jar + encoding3d (Java) + Reading and writing encoding3d files: functional-structural plant model scenes as glTF 2.0 binaries + https://github.com/openalea/encoding3d + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + As GroIMP, the platform this codec serves (bindings/java/LICENSE); the rest of encoding3d is CeCILL-C + + + + + 17 + UTF-8 + 2.18.2 + 5.11.4 + ${project.basedir}/../../conformance + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + org.zeromq + jeromq + 0.6.0 + true + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + -Xlint:all + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + ${encoding3d.conformance} + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + org.openalea.encoding3d.ConformanceRun + + + org.openalea.encoding3d + + + + + + + diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Codec.java b/bindings/java/src/main/java/org/openalea/encoding3d/Codec.java new file mode 100644 index 0000000..f350839 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Codec.java @@ -0,0 +1,959 @@ +package org.openalea.encoding3d; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), + * meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data + * structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, + * standard names, kinds and roles. + */ +public final class Codec { + /** The version of the specification this library reads and writes. */ + public static final String SPECIFICATION_VERSION = "0.3"; + public static final List EDGE_TYPES = List.of("none", "successor", "branch", "unknown"); + + private static final String EXTENSION = "FSPM_scene", METADATA = "EXT_structural_metadata", + FEATURES = "EXT_mesh_features", EDGE_TYPE_ENUM = "FSPM_edge_type"; + private static final List ENUM_COLUMNS = List.of("label", "edge_type", "relation"); + private static final List ID_COLUMNS = List.of("id", "complex", "parent", "source", "target", "level", "index", + "sibling_order"); + private static final int BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing + private static final int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; + private static final int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; + private static final long MAX_FLOAT_FEATURE = 1L << 24; // feature rows are float vertex attributes + private static final Pattern IDENTIFIER = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); + // The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about + // x, column-major). Readers of the tables use the scene's frame, not this node. + private static final double[] Z_UP_TO_Y_UP = {1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1}; + + private Codec() {} + + // ── Writing ────────────────────────────────────────────────────────── + + /** The GLB file of a frame. */ + public static byte[] toBytes(Frame frame) { + Glb.BufferBuilder buffers = new Glb.BufferBuilder(); + ObjectNode schema = Glb.MAPPER.createObjectNode().put("id", "fspm_scene"); + schema.putObject("classes"); + ObjectNode edgeEnum = schema.putObject("enums").putObject(EDGE_TYPE_ENUM).put("valueType", "UINT8"); + ArrayNode edgeValues = edgeEnum.putArray("values"); + for (int value = 0; value < EDGE_TYPES.size(); value++) + edgeValues.addObject().put("name", EDGE_TYPES.get(value)).put("value", value); + ArrayNode tables = Glb.MAPPER.createArrayNode(); + ObjectNode variables = Glb.MAPPER.createObjectNode(); + ObjectNode document = Glb.MAPPER.createObjectNode(); + document.putObject("asset").put("version", "2.0").put("generator", "encoding3d (Java) " + SPECIFICATION_VERSION); + document.put("scene", 0); + document.putArray("scenes").addObject().putArray("nodes").add(0); + ArrayNode nodes = document.putArray("nodes"); + ObjectNode sceneNode = nodes.addObject().put("name", "fspm_scene"); + putNumbers(sceneNode.putArray("matrix"), Z_UP_TO_Y_UP); + sceneNode.putArray("children"); + ArrayNode accessors = Glb.MAPPER.createArrayNode(), meshes = Glb.MAPPER.createArrayNode(); + + ArrayNode dataStructures = Glb.MAPPER.createArrayNode(); + for (DataStructure dataStructure : frame.dataStructures) { + checkIdentifier(dataStructure.id, "data structure id"); + ObjectNode entry = dataStructures.addObject().put("id", dataStructure.id).put("kind", dataStructure.kind); + if (dataStructure.platform != null) entry.put("platform", dataStructure.platform); + if (dataStructure.model != null) entry.put("model", dataStructure.model); + double[] columnMajor = columnMajor(dataStructure.origin); + putNumbers(entry.putArray("origin"), columnMajor); // column-major, as glTF matrices + ArrayNode sets = entry.putArray("entitySets"); + for (EntitySet set : dataStructure.entitySets) { + ObjectNode item = sets.addObject().put("name", set.name); + if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + if (set.scale != null) item.put("scale", set.scale); + item.put("propertyTable", addTable(set, dataStructure.id + "__" + set.name, buffers, schema, tables, variables)); + } + if (!dataStructure.relations.isEmpty()) { + ArrayNode relations = entry.putArray("relations"); + for (Relation relation : dataStructure.relations) + relations.addObject().put("name", relation.name).put("sourceSet", relation.sourceSet) + .put("targetSet", relation.targetSet).put("propertyTable", + addTable(relation, dataStructure.id + "__" + relation.name, buffers, schema, tables, variables)); + } + if (dataStructure.scalars != null) + entry.put("scalars", addTable(dataStructure.scalars, dataStructure.id + "__scalars", buffers, schema, tables, variables)); + if (dataStructure.grid != null) entry.set("grid", gridJson(dataStructure.grid)); + if (!dataStructure.labels.isEmpty()) { + ObjectNode labels = entry.putObject("labels"); + dataStructure.labels.forEach(labels::put); + } + if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); + ObjectNode node = nodes.addObject().put("name", dataStructure.id); + if (!isIdentity(dataStructure.origin)) putNumbers(node.putArray("matrix"), columnMajor); + int nodeIndex = nodes.size() - 1; + ((ArrayNode) sceneNode.get("children")).add(nodeIndex); + entry.put("node", nodeIndex); + if (!dataStructure.meshes.isEmpty()) { + ArrayNode children = node.putArray("children"); + ArrayNode meshEntries = entry.putArray("meshes"); + for (Mesh mesh : dataStructure.meshes) { + meshes.add(encodeMesh(mesh, dataStructure, entry, buffers, accessors)); + nodes.addObject().put("name", mesh.name).put("mesh", meshes.size() - 1); + children.add(nodes.size() - 1); + ObjectNode meshEntry = meshEntries.addObject().put("name", mesh.name).put("mesh", meshes.size() - 1); + if (!mesh.display.isEmpty()) { + ArrayNode display = meshEntry.putArray("display"); + for (String variable : mesh.display.keySet()) { + ObjectNode item = display.addObject().put("attribute", displayAttribute(variable)) + .put("variable", variable); + String source = mesh.displaySources.get(variable); + if (source == null) item.putNull("entitySet"); else item.put("entitySet", source); + } + } + } + } + } + + ArrayNode mappings = Glb.MAPPER.createArrayNode(); + for (Mapping mapping : frame.mappings) { + checkIdentifier(mapping.name, "mapping name"); + ObjectNode entry = mappings.addObject().put("name", mapping.name); + entry.putObject("source").put("dataStructure", mapping.sourceDataStructure).put("entitySet", mapping.sourceSet); + entry.putObject("target").put("dataStructure", mapping.targetDataStructure).put("entitySet", mapping.targetSet); + if (mapping.method != null) entry.put("method", mapping.method); + entry.put("propertyTable", addTable(mapping, "mapping__" + mapping.name, buffers, schema, tables, variables)); + } + + ObjectNode update = null; + if (frame.update != null) { + checkIdentifier(frame.update.participant, "participant"); + update = Glb.MAPPER.createObjectNode().put("participant", frame.update.participant); + if (!frame.update.removed.isEmpty()) { + ArrayNode removed = update.putArray("removed"); + for (Map.Entry item : frame.update.removed.entrySet()) { + String[] key = item.getKey().split("/", 2); + Table table = new Table(key[1] + "__removed"); + table.set("id", Column.of(item.getValue())); + removed.addObject().put("dataStructure", key[0]).put("entitySet", key[1]).put("propertyTable", + addTable(table, key[0] + "__" + key[1] + "__removed", buffers, schema, tables, variables)); + } + } + } + ObjectNode scene = Glb.MAPPER.createObjectNode().put("version", SPECIFICATION_VERSION); + scene.putObject("frame").put("time", frame.time).put("timeUnit", frame.timeUnit).put("step", frame.step); + if (frame.vocabularyVersion != null) scene.put("vocabulary", frame.vocabularyVersion); + if (!frame.writer.isEmpty()) scene.set("writer", frame.writer); + if (update != null) scene.set("update", update); + scene.set("dataStructures", dataStructures); + if (!mappings.isEmpty()) scene.set("mappings", mappings); + if (!variables.isEmpty()) scene.set("variables", variables); + + ObjectNode extensions = Glb.MAPPER.createObjectNode(); + ObjectNode metadata = extensions.putObject(METADATA); + metadata.set("schema", schema); + metadata.set("propertyTables", tables); + extensions.set(EXTENSION, scene); + ArrayNode used = document.putArray("extensionsUsed").add(METADATA).add(EXTENSION); + if (!meshes.isEmpty()) used.add(FEATURES); + JsonNode others = frame.unknown.get("extensions"); + if (others != null) + for (Iterator> items = others.fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + extensions.set(item.getKey(), item.getValue()); + used.add(item.getKey()); + } + document.set("extensions", extensions); + if (frame.unknown.has("extras")) document.set("extras", frame.unknown.get("extras")); + if (!meshes.isEmpty()) { + document.set("meshes", meshes); + document.set("accessors", accessors); + } + byte[] binary = buffers.data.toByteArray(); + if (!buffers.views.isEmpty()) { + // the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + binary = java.util.Arrays.copyOf(binary, binary.length + Glb.padding(binary.length)); + document.putArray("buffers").addObject().put("byteLength", binary.length); + document.set("bufferViews", buffers.views); + } + return Glb.pack(document, binary); + } + + /** Writes a frame to the GLB file {@code path}. */ + public static void write(Path path, Frame frame) throws IOException { + Files.write(path, toBytes(frame)); + } + + static void checkIdentifier(String name, String what) { + if (name == null || !IDENTIFIER.matcher(name).matches()) + throw new Encoding3dException(what + " '" + name + "' must be an identifier (letters, digits, '_'; not starting with a digit)"); + } + + private static void putNumbers(ArrayNode array, double[] values) { + for (double value : values) array.add(value); + } + + private static double[] columnMajor(double[] rowMajor) { + double[] values = new double[16]; + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) values[column * 4 + row] = rowMajor[row * 4 + column]; + return values; + } + + private static boolean isIdentity(double[] origin) { + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) + if (origin[row * 4 + column] != (row == column ? 1.0 : 0.0)) return false; + return true; + } + + private static ObjectNode declarationJson(Variable variable) { + ObjectNode declaration = Glb.MAPPER.createObjectNode().put("unit", variable.unit).put("role", variable.role); + if (variable.standardName != null) declaration.put("standardName", variable.standardName); + if (variable.kind != null) declaration.put("kind", variable.kind); + if (variable.minimum != null) declaration.put("minimum", variable.minimum); + if (variable.maximum != null) declaration.put("maximum", variable.maximum); + if (!variable.references.isEmpty()) variable.references.forEach(declaration.putArray("references")::add); + if (!variable.doi.isEmpty()) variable.doi.forEach(declaration.putArray("doi")::add); + if (variable.format != null) declaration.put("format", variable.format); + return declaration; + } + + private static int addTable(Table table, String classId, Glb.BufferBuilder buffers, ObjectNode schema, ArrayNode tables, + ObjectNode variables) { + checkIdentifier(classId, "class"); + table.checkLengths(); + ObjectNode properties = Glb.MAPPER.createObjectNode(), stored = Glb.MAPPER.createObjectNode(); + for (Map.Entry item : table.columns.entrySet()) + encodeColumn(table, classId, item.getKey(), item.getValue(), buffers, schema, properties, stored); + ((ObjectNode) schema.get("classes")).putObject(classId).put("name", table.name).set("properties", properties); + ObjectNode description = tables.addObject().put("name", table.name).put("class", classId).put("count", table.rowCount()); + description.set("properties", stored); + ObjectNode declared = Glb.MAPPER.createObjectNode(); + for (String name : table.variableNames()) + if (table.variables.containsKey(name)) declared.set(name, declarationJson(table.variables.get(name))); + if (!declared.isEmpty()) variables.set(classId, declared); + return tables.size() - 1; + } + + private static void encodeColumn(Table table, String classId, String name, Column column, Glb.BufferBuilder buffers, + ObjectNode schema, ObjectNode properties, ObjectNode stored) { + checkIdentifier(name, "column of '" + table.name + "'"); + String where = "column '" + name + "' of '" + table.name + "'"; + ObjectNode property = properties.putObject(name); + Variable declared = table.variables.get(name); + if (declared != null && declared.description != null) property.put("description", declared.description); + boolean reserved = table.isReserved(name); + if (reserved && ENUM_COLUMNS.contains(name)) { + if (column.type != ComponentType.STRING) throw new Encoding3dException(where + ": labels are text"); + property.put("type", "ENUM"); + property.put("enumType", encodeEnum(name, column.strings(), classId, schema, buffers, stored)); + } else if (column.type == ComponentType.STRING) { + if (column.width != 1) throw new Encoding3dException(where + ": text vectors are not supported"); + property.put("type", "STRING"); + String[] values = column.strings(); + if (column.hasMissing()) { // missing text: noData, a string the column lacks + String noData = freeString(column); + property.put("noData", noData); + values = values.clone(); + for (int row = 0; row < values.length; row++) if (column.missing[row]) values[row] = noData; + } + byte[][] encoded = new byte[values.length][]; + long[] offsets = new long[values.length + 1]; + for (int row = 0; row < values.length; row++) { + encoded[row] = values[row].getBytes(StandardCharsets.UTF_8); + offsets[row + 1] = offsets[row] + encoded[row].length; + } + ByteBuffer text = ByteBuffer.allocate((int) offsets[values.length]); + for (byte[] value : encoded) text.put(value); + ObjectNode item = stored.putObject(name).put("values", buffers.add(text.array())); + boolean narrow = offsets[values.length] < (1L << 32); + ByteBuffer offsetBytes = little(offsets.length * (narrow ? 4 : 8)); + for (long offset : offsets) if (narrow) offsetBytes.putInt((int) offset); else offsetBytes.putLong(offset); + item.put("stringOffsets", buffers.add(offsetBytes.array())).put("stringOffsetType", narrow ? "UINT32" : "UINT64"); + } else if (column.type == ComponentType.BOOLEAN && column.hasMissing()) { // UINT8 0, 1, 255 missing + property.put("type", "SCALAR").put("componentType", "UINT8").put("noData", BOOLEAN_NO_DATA); + property.putObject("extras").put("FSPM_boolean", true); + boolean[] values = column.booleans(); + byte[] codes = new byte[values.length]; + for (int row = 0; row < values.length; row++) + codes[row] = column.missing[row] ? (byte) BOOLEAN_NO_DATA : (byte) (values[row] ? 1 : 0); + stored.putObject(name).put("values", buffers.add(codes)); + } else if (column.type == ComponentType.BOOLEAN) { + if (column.width != 1) throw new Encoding3dException(where + ": boolean vectors are not supported"); + property.put("type", "BOOLEAN"); + boolean[] values = column.booleans(); + byte[] packed = new byte[(values.length + 7) / 8]; + for (int row = 0; row < values.length; row++) + if (values[row]) packed[row / 8] |= (byte) (1 << (row % 8)); // bit order: little + stored.putObject(name).put("values", buffers.add(packed)); + } else { + ComponentType type = column.type; + Column data = column; + if (reserved && ID_COLUMNS.contains(name) && !type.isFloat() && type != ComponentType.INT64) { + long[] ids = new long[column.size()]; // ids are INT64, whatever the platform's + for (int index = 0; index < ids.length; index++) ids[index] = (long) column.doubleAt(index); + data = new Column(ComponentType.INT64, ids, column.width); + type = ComponentType.INT64; + } + property.put("type", "SCALAR").put("componentType", type.name()); + if (column.width != 1) { + if (column.width < 2) throw new Encoding3dException(where + ": vectors have at least 2 components"); + property.put("array", true).put("count", column.width); + } + if (type == ComponentType.FLOAT64) { + double[] values = data.doubles().clone(); + for (int row = 0; column.missing != null && row < values.length; row++) // a missing float: NaN + if (column.missing[row]) values[row] = Double.NaN; + data = standIns(values, property, where); + } else if (type == ComponentType.FLOAT32) { + float[] values = data.floats().clone(); + for (int row = 0; column.missing != null && row < values.length; row++) + if (column.missing[row]) values[row] = Float.NaN; + data = standIns(values, property, where); + } else if (reserved && (name.equals("complex") || name.equals("parent"))) { + property.put("noData", EntitySet.NO_ID); + } else if (column.hasMissing()) { // missing integers: noData, a value the column lacks + long noData = freeInteger(column); + property.put("noData", noData); + data = withValueAt(column, noData); + } + if (data.width != column.width) data = new Column(data.type, data.values, column.width); + stored.putObject(name).put("values", buffers.add(bytesOf(data))); + } + if (name.equals("id") && table.kind() == Table.Kind.ENTITY_SET) property.put("required", true); + } + + /** A value of the column's integer type that it does not hold: its minimum (signed) or maximum (unsigned) first. */ + private static long freeInteger(Column column) { + java.util.Set present = new java.util.HashSet<>(); + for (int row = 0; row < column.size(); row++) + if (!column.isMissing(row)) present.add(rawLong(column, row)); + long minimum, maximum; + switch (column.type) { + case INT8 -> { minimum = Byte.MIN_VALUE; maximum = Byte.MAX_VALUE; } + case INT16 -> { minimum = Short.MIN_VALUE; maximum = Short.MAX_VALUE; } + case INT32 -> { minimum = Integer.MIN_VALUE; maximum = Integer.MAX_VALUE; } + case INT64 -> { minimum = Long.MIN_VALUE; maximum = Long.MAX_VALUE; } + case UINT8 -> { minimum = 0; maximum = 0xFFL; } + case UINT16 -> { minimum = 0; maximum = 0xFFFFL; } + case UINT32 -> { minimum = 0; maximum = 0xFFFFFFFFL; } + default -> { minimum = 0; maximum = -1; } // UINT64: its maximum's bits + } + boolean signed = !column.type.isUnsigned(); + long first = signed ? minimum : maximum, second = signed ? maximum : minimum; + if (!present.contains(first)) return first; + if (!present.contains(second)) return second; + long candidate = first; + while (present.contains(candidate)) candidate = signed ? candidate + 1 : candidate - 1; + return candidate; + } + + /** A value's bits as a long, unsigned values as stored (UINT8 255 as 255). */ + private static long rawLong(Column column, int index) { + return switch (column.type) { + case INT8 -> ((byte[]) column.values)[index]; + case UINT8 -> Byte.toUnsignedLong(((byte[]) column.values)[index]); + case INT16 -> ((short[]) column.values)[index]; + case UINT16 -> Short.toUnsignedLong(((short[]) column.values)[index]); + case INT32 -> ((int[]) column.values)[index]; + case UINT32 -> Integer.toUnsignedLong(((int[]) column.values)[index]); + case INT64, UINT64 -> ((long[]) column.values)[index]; + default -> throw new Encoding3dException("a " + column.type + " column has no integers"); + }; + } + + /** The column's values with {@code value} at its missing rows. */ + private static Column withValueAt(Column column, long value) { + Object copy = java.lang.reflect.Array.newInstance(column.values.getClass().getComponentType(), column.size()); + System.arraycopy(column.values, 0, copy, 0, column.size()); + for (int row = 0; row < column.size(); row++) + if (column.isMissing(row)) + switch (column.type) { + case INT8, UINT8 -> ((byte[]) copy)[row] = (byte) value; + case INT16, UINT16 -> ((short[]) copy)[row] = (short) value; + case INT32, UINT32 -> ((int[]) copy)[row] = (int) value; + default -> ((long[]) copy)[row] = value; + } + return new Column(column.type, copy, column.width); + } + + /** A string the column does not hold: the empty string first, then NUL characters. */ + private static String freeString(Column column) { + java.util.Set present = new java.util.HashSet<>(); + for (int row = 0; row < column.size(); row++) if (!column.isMissing(row)) present.add(column.strings()[row]); + String candidate = ""; + while (present.contains(candidate)) candidate += "\0"; + return candidate; + } + + // EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; + // +inf and -inf as the largest finite value and the next one after the most negative, declared in the property's + // extras (FSPM_infinity). + private static Column standIns(double[] values, ObjectNode property, String where) { + double positive = Double.MAX_VALUE, negative = Math.nextUp(-Double.MAX_VALUE), noData = -Double.MAX_VALUE; + boolean infinite = false, nan = false; + for (double value : values) { + infinite |= Double.isInfinite(value); + nan |= Double.isNaN(value); + } + if (infinite) { + for (double value : values) + if (value == positive || value == negative) + throw new Encoding3dException(where + " holds both infinities and the values standing for them"); + property.putObject("extras").putObject("FSPM_infinity").put("positive", positive).put("negative", negative); + for (int index = 0; index < values.length; index++) + if (Double.isInfinite(values[index])) values[index] = values[index] > 0 ? positive : negative; + } + if (nan) { + for (double value : values) + if (value == noData) throw new Encoding3dException(where + " holds both NaN and the value standing for NaN"); + property.put("noData", noData); + for (int index = 0; index < values.length; index++) if (Double.isNaN(values[index])) values[index] = noData; + } + return new Column(ComponentType.FLOAT64, values); + } + + private static Column standIns(float[] values, ObjectNode property, String where) { + float positive = Float.MAX_VALUE, negative = Math.nextUp(-Float.MAX_VALUE), noData = -Float.MAX_VALUE; + boolean infinite = false, nan = false; + for (float value : values) { + infinite |= Float.isInfinite(value); + nan |= Float.isNaN(value); + } + if (infinite) { + for (float value : values) + if (value == positive || value == negative) + throw new Encoding3dException(where + " holds both infinities and the values standing for them"); + property.putObject("extras").putObject("FSPM_infinity").put("positive", (double) positive) + .put("negative", (double) negative); + for (int index = 0; index < values.length; index++) + if (Float.isInfinite(values[index])) values[index] = values[index] > 0 ? positive : negative; + } + if (nan) { + for (float value : values) + if (value == noData) throw new Encoding3dException(where + " holds both NaN and the value standing for NaN"); + property.put("noData", (double) noData); + for (int index = 0; index < values.length; index++) if (Float.isNaN(values[index])) values[index] = noData; + } + return new Column(ComponentType.FLOAT32, values); + } + + private static String encodeEnum(String name, String[] values, String classId, ObjectNode schema, + Glb.BufferBuilder buffers, ObjectNode stored) { + if (name.equals("edge_type")) { + byte[] codes = new byte[values.length]; + for (int row = 0; row < values.length; row++) { + int code = EDGE_TYPES.indexOf(values[row]); + if (code < 0) + throw new Encoding3dException("edge_type of '" + classId + "': unknown value '" + values[row] + + "' (allowed: none, successor, branch)"); + codes[row] = (byte) code; + } + stored.putObject(name).put("values", buffers.add(codes)); + return EDGE_TYPE_ENUM; + } + List names = new ArrayList<>(new TreeSet<>(List.of(values))); + String enumName = classId + "__" + name; + boolean narrow = names.size() <= 256; + ObjectNode enumeration = ((ObjectNode) schema.get("enums")).putObject(enumName).put("valueType", narrow ? "UINT8" : "UINT16"); + ArrayNode enumValues = enumeration.putArray("values"); + Map codeOf = new HashMap<>(); + for (int code = 0; code < names.size(); code++) { + enumValues.addObject().put("name", names.get(code)).put("value", code); + codeOf.put(names.get(code), code); + } + ByteBuffer codes = little(values.length * (narrow ? 1 : 2)); + for (String value : values) if (narrow) codes.put((byte) (int) codeOf.get(value)); else codes.putShort((short) (int) codeOf.get(value)); + stored.putObject(name).put("values", buffers.add(codes.array())); + return enumName; + } + + private static ByteBuffer little(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + } + + /** A numeric column's values, little-endian. */ + static byte[] bytesOf(Column column) { + ByteBuffer bytes = little(column.size() * column.type.size); + switch (column.type) { + case INT8, UINT8 -> bytes.put((byte[]) column.values); + case INT16, UINT16 -> { for (short value : (short[]) column.values) bytes.putShort(value); } + case INT32, UINT32 -> { for (int value : (int[]) column.values) bytes.putInt(value); } + case INT64, UINT64 -> { for (long value : (long[]) column.values) bytes.putLong(value); } + case FLOAT32 -> { for (float value : (float[]) column.values) bytes.putFloat(value); } + case FLOAT64 -> { for (double value : (double[]) column.values) bytes.putDouble(value); } + default -> throw new Encoding3dException("a " + column.type + " column has no numbers"); + } + return bytes.array(); + } + + private static ObjectNode gridJson(Grid grid) { + ObjectNode entry = Glb.MAPPER.createObjectNode().put("kind", grid.kind); + ArrayNode shape = entry.putArray("shape"); + for (long size : grid.shape) shape.add(size); + putNumbers(entry.putArray("origin"), grid.origin); + putNumbers(entry.putArray("spacing"), grid.spacing); + grid.axes.forEach(entry.putArray("axes")::add); + ArrayNode periodic = entry.putArray("periodic"); + for (boolean value : grid.periodic) periodic.add(value); + entry.put("zUp", grid.zUp).put("cells", grid.cells); + if (grid.kind.equals("adaptive")) entry.put("maxLevel", grid.maxLevel); + return entry; + } + + private static String displayAttribute(String name) { + return "_" + name.toUpperCase(java.util.Locale.ROOT); + } + + private static ObjectNode encodeMesh(Mesh mesh, DataStructure dataStructure, ObjectNode entry, + Glb.BufferBuilder buffers, ArrayNode accessors) { + int vertices = mesh.vertexCount(); + if (mesh.positions.length != vertices * 3) throw new Encoding3dException("mesh '" + mesh.name + "': 3 coordinates per vertex"); + float[] positions = new float[mesh.positions.length]; + for (int index = 0; index < positions.length; index++) positions[index] = (float) mesh.positions[index]; + ObjectNode attributes = Glb.MAPPER.createObjectNode(); + ObjectNode position = accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, positions)), "VEC3", + FLOAT, ARRAY_BUFFER, vertices, attributes, "POSITION"); + if (vertices > 0) { + double[] minimum = {Double.MAX_VALUE, Double.MAX_VALUE, Double.MAX_VALUE}, maximum = {-Double.MAX_VALUE, -Double.MAX_VALUE, -Double.MAX_VALUE}; + for (int vertex = 0; vertex < vertices; vertex++) + for (int axis = 0; axis < 3; axis++) { + minimum[axis] = Math.min(minimum[axis], positions[vertex * 3 + axis]); + maximum[axis] = Math.max(maximum[axis], positions[vertex * 3 + axis]); + } + putNumbers(position.putArray("min"), minimum); + putNumbers(position.putArray("max"), maximum); + } + Map tables = new HashMap<>(); + for (JsonNode set : entry.get("entitySets")) tables.put(set.get("name").asText(), set.get("propertyTable").asInt()); + ArrayNode featureIds = Glb.MAPPER.createArrayNode(); + int setIndex = 0; + for (Map.Entry item : mesh.featureRows.entrySet()) { + String setName = item.getKey(); + long[] rows = item.getValue(); + if (!tables.containsKey(setName)) + throw new Encoding3dException("mesh '" + mesh.name + "': no entity set '" + setName + "' in '" + dataStructure.id + "'"); + if (rows.length != vertices) + throw new Encoding3dException("mesh '" + mesh.name + "': feature rows of '" + setName + "' are not one per vertex"); + long count = dataStructure.entitySet(setName).rowCount(); + if (count >= MAX_FLOAT_FEATURE) + throw new Encoding3dException("mesh '" + mesh.name + "': too many entities in '" + setName + "', feature IDs stop at 2^24"); + float[] storedRows = new float[rows.length]; + TreeSet present = new TreeSet<>(); + boolean hasNull = false; + for (int vertex = 0; vertex < rows.length; vertex++) { + long row = rows[vertex]; + if (row < EntitySet.NO_ID || row >= count) + throw new Encoding3dException("mesh '" + mesh.name + "': feature rows of '" + setName + "' outside its rows"); + hasNull |= row == EntitySet.NO_ID; + if (row != EntitySet.NO_ID) present.add(row); + storedRows[vertex] = row == EntitySet.NO_ID ? count : row; // null: one past the last row + } + accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, storedRows)), "SCALAR", FLOAT, ARRAY_BUFFER, + vertices, attributes, "_FEATURE_ID_" + setIndex); + ObjectNode feature = featureIds.addObject().put("featureCount", present.size()).put("attribute", setIndex) + .put("propertyTable", tables.get(setName)).put("label", setName); + if (hasNull) feature.put("nullFeatureId", count); + setIndex++; + } + for (Map.Entry item : mesh.display.entrySet()) { + if (item.getValue().length != vertices) + throw new Encoding3dException("mesh '" + mesh.name + "': display copy '" + item.getKey() + "' is not one value per vertex"); + accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, item.getValue())), "SCALAR", FLOAT, + ARRAY_BUFFER, vertices, attributes, displayAttribute(item.getKey())); + } + ObjectNode primitive = Glb.MAPPER.createObjectNode(); + primitive.set("attributes", attributes); + primitive.put("mode", 4); + if (mesh.indices != null) { + int[] indices = new int[mesh.indices.length]; + for (int index = 0; index < indices.length; index++) indices[index] = (int) mesh.indices[index]; + ObjectNode holder = Glb.MAPPER.createObjectNode(); + accessor(accessors, buffers, bytesOf(new Column(ComponentType.UINT32, indices)), "SCALAR", UNSIGNED_INT, + ELEMENT_ARRAY_BUFFER, indices.length, holder, "indices"); + primitive.set("indices", holder.get("indices")); + } + if (!featureIds.isEmpty()) primitive.putObject("extensions").putObject(FEATURES).set("featureIds", featureIds); + ObjectNode encoded = Glb.MAPPER.createObjectNode().put("name", mesh.name); + encoded.putArray("primitives").add(primitive); + return encoded; + } + + private static ObjectNode accessor(ArrayNode accessors, Glb.BufferBuilder buffers, byte[] bytes, String type, + int componentType, int target, int count, ObjectNode attributes, String attribute) { + ObjectNode item = accessors.addObject().put("bufferView", buffers.add(bytes, target)) + .put("componentType", componentType).put("count", count).put("type", type); + attributes.put(attribute, accessors.size() - 1); + return item; + } + + // ── Reading ────────────────────────────────────────────────────────── + + /** The frame of a GLB file's bytes. */ + public static Frame fromBytes(byte[] data) { + Glb.Contents contents = Glb.unpack(data); + JsonNode document = contents.document(); + byte[] binary = contents.binary(); + JsonNode extensions = document.path("extensions"); + if (!extensions.has(EXTENSION)) throw new Encoding3dException("not an encoding3d file: no " + EXTENSION + " extension"); + JsonNode scene = extensions.get(EXTENSION); + JsonNode metadata = extensions.path(METADATA); + JsonNode variables = scene.path("variables"); + + Frame frame = new Frame(); + JsonNode frameEntry = scene.path("frame"); + frame.time = frameEntry.path("time").asDouble(0.0); + frame.timeUnit = frameEntry.path("timeUnit").asText("d"); + frame.step = frameEntry.path("step").asLong(0); + if (scene.get("writer") instanceof ObjectNode writer) frame.writer = writer; + if (scene.hasNonNull("vocabulary")) frame.vocabularyVersion = scene.get("vocabulary").asText(); + for (JsonNode entry : scene.path("dataStructures")) { + DataStructure dataStructure = new DataStructure(entry.get("id").asText(), entry.path("kind").asText("graph")); + if (entry.hasNonNull("platform")) dataStructure.platform = entry.get("platform").asText(); + if (entry.hasNonNull("model")) dataStructure.model = entry.get("model").asText(); + if (entry.get("extras") instanceof ObjectNode extras) dataStructure.extras = extras; + for (Iterator> labels = entry.path("labels").fields(); labels.hasNext(); ) { + Map.Entry label = labels.next(); + dataStructure.labels.put(label.getKey(), label.getValue().asText()); + } + if (entry.has("origin")) { + JsonNode origin = entry.get("origin"); + if (origin.size() != 16) throw new Encoding3dException("data structure '" + dataStructure.id + "': an origin of 16 numbers"); + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) + dataStructure.origin[row * 4 + column] = origin.get(column * 4 + row).asDouble(); + } + Map tableNames = new HashMap<>(); + for (JsonNode item : entry.path("entitySets")) { + EntitySet set = new EntitySet("", item.hasNonNull("rank") ? item.get("rank").asLong() : null); + if (item.hasNonNull("scale")) set.scale = item.get("scale").asText(); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), set); + tableNames.put(item.get("propertyTable").asInt(), item.get("name").asText()); + dataStructure.entitySets.add(set); + } + for (JsonNode item : entry.path("relations")) { + Relation relation = new Relation("", item.get("sourceSet").asText(), item.get("targetSet").asText()); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), relation); + dataStructure.relations.add(relation); + } + if (entry.has("scalars")) { + Table scalars = new Table(""); + decodeTable(document, binary, metadata, variables, entry.get("scalars").asInt(), scalars); + dataStructure.scalars = scalars; + } + if (entry.has("grid")) dataStructure.grid = decodeGrid(entry.get("grid")); + for (JsonNode meshEntry : entry.path("meshes")) + dataStructure.meshes.add(decodeMesh(document, binary, meshEntry, tableNames)); + frame.dataStructures.add(dataStructure); + } + for (JsonNode entry : scene.path("mappings")) { + Mapping mapping = new Mapping("", entry.get("source").get("dataStructure").asText(), + entry.get("source").get("entitySet").asText(), entry.get("target").get("dataStructure").asText(), + entry.get("target").get("entitySet").asText(), entry.hasNonNull("method") ? entry.get("method").asText() : null); + decodeTable(document, binary, metadata, variables, entry.get("propertyTable").asInt(), mapping); + frame.mappings.add(mapping); + } + if (scene.has("update")) { + JsonNode entry = scene.get("update"); + frame.update = new Update(entry.get("participant").asText()); + for (JsonNode item : entry.path("removed")) { + Table removed = new Table(""); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), removed); + frame.update.removed.put(Update.key(item.get("dataStructure").asText(), item.get("entitySet").asText()), + removed.column("id").longs()); + } + } + ObjectNode others = Glb.MAPPER.createObjectNode(); + for (Iterator> items = extensions.fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + if (!List.of(METADATA, EXTENSION, FEATURES).contains(item.getKey())) others.set(item.getKey(), item.getValue()); + } + if (!others.isEmpty()) frame.unknown.set("extensions", others); + if (document.has("extras")) frame.unknown.set("extras", document.get("extras")); + return frame; + } + + /** The frame of the GLB file {@code path}. */ + public static Frame read(Path path) throws IOException { + return fromBytes(Files.readAllBytes(path)); + } + + private static ByteBuffer view(JsonNode document, byte[] binary, int index) { + JsonNode view = document.get("bufferViews").get(index); + if (view.path("buffer").asInt(0) != 0) throw new Encoding3dException("encoding3d files keep every buffer view in the GLB binary chunk"); + int start = view.path("byteOffset").asInt(0), length = view.get("byteLength").asInt(); + if (start < 0 || length < 0 || start + (long) length > binary.length) throw new Encoding3dException("a buffer view runs past the binary chunk"); + return ByteBuffer.wrap(binary, start, length).slice().order(ByteOrder.LITTLE_ENDIAN); + } + + private static long[] unsignedValues(String type, ByteBuffer bytes, int count) { + long[] values = new long[count]; + for (int index = 0; index < count; index++) + values[index] = switch (type) { + case "UINT8" -> Byte.toUnsignedLong(bytes.get(index)); + case "UINT16" -> Short.toUnsignedLong(bytes.getShort(index * 2)); + case "UINT32" -> Integer.toUnsignedLong(bytes.getInt(index * 4)); + case "UINT64" -> bytes.getLong(index * 8); + default -> throw new Encoding3dException("offsets or enum values of type " + type + " are not part of encoding3d 0.1"); + }; + return values; + } + + private static Column readScalars(ComponentType type, ByteBuffer bytes, int count, int width) { + int size = count * width; + if ((long) size * type.size > bytes.remaining()) throw new Encoding3dException("a buffer view is too short for its values"); + return switch (type) { + case INT8, UINT8 -> { byte[] values = new byte[size]; bytes.get(values); yield new Column(type, values, width); } + case INT16, UINT16 -> { short[] values = new short[size]; bytes.asShortBuffer().get(values); yield new Column(type, values, width); } + case INT32, UINT32 -> { int[] values = new int[size]; bytes.asIntBuffer().get(values); yield new Column(type, values, width); } + case INT64, UINT64 -> { long[] values = new long[size]; bytes.asLongBuffer().get(values); yield new Column(type, values, width); } + case FLOAT32 -> { float[] values = new float[size]; bytes.asFloatBuffer().get(values); yield new Column(type, values, width); } + case FLOAT64 -> { double[] values = new double[size]; bytes.asDoubleBuffer().get(values); yield new Column(type, values, width); } + default -> throw new Encoding3dException("component type " + type + " is not a number"); + }; + } + + private static void decodeTable(JsonNode document, byte[] binary, JsonNode metadata, JsonNode variables, int index, + Table table) { + JsonNode description = metadata.get("propertyTables").get(index); + String classId = description.get("class").asText(); + table.name = description.path("name").asText(classId); + JsonNode schema = metadata.get("schema"); + JsonNode properties = schema.get("classes").get(classId).get("properties"); + int count = description.get("count").asInt(); + for (Iterator> items = description.get("properties").fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + String name = item.getKey(); + JsonNode stored = item.getValue(), property = properties.get(name); + ByteBuffer raw = view(document, binary, stored.get("values").asInt()); + String type = property.get("type").asText(); + Column column; + switch (type) { + case "ENUM" -> { + JsonNode enumeration = schema.get("enums").get(property.get("enumType").asText()); + long[] codes = unsignedValues(enumeration.path("valueType").asText("UINT16"), raw, count); + Map nameOf = new HashMap<>(); + for (JsonNode value : enumeration.get("values")) nameOf.put(value.get("value").asLong(), value.get("name").asText()); + String[] labels = new String[count]; + for (int row = 0; row < count; row++) labels[row] = nameOf.get(codes[row]); + column = new Column(ComponentType.STRING, labels); + } + case "STRING" -> { + long[] offsets = unsignedValues(stored.path("stringOffsetType").asText("UINT32"), + view(document, binary, stored.get("stringOffsets").asInt()), count + 1); + String[] text = new String[count]; + byte[] bytes = new byte[raw.remaining()]; + raw.get(bytes); + for (int row = 0; row < count; row++) { + if (offsets[row + 1] > bytes.length || offsets[row] > offsets[row + 1]) + throw new Encoding3dException("'" + classId + "." + name + "': string offsets outside its values"); + text[row] = new String(bytes, (int) offsets[row], (int) (offsets[row + 1] - offsets[row]), StandardCharsets.UTF_8); + } + column = new Column(ComponentType.STRING, text); + if (property.has("noData")) { // missing text + String noData = property.get("noData").asText(); + boolean[] missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) + if (text[row].equals(noData)) { + missing[row] = any = true; + text[row] = ""; + } + if (any) column.missing = missing; + } + } + case "BOOLEAN" -> { + if (raw.remaining() * 8L < count) throw new Encoding3dException("'" + classId + "." + name + "': too few bits"); + boolean[] values = new boolean[count]; + for (int row = 0; row < count; row++) values[row] = ((raw.get(row / 8) >> (row % 8)) & 1) == 1; + column = new Column(ComponentType.BOOLEAN, values); + } + case "SCALAR" -> { + boolean vector = property.path("array").asBoolean(false); + if (vector && !property.has("count")) + throw new Encoding3dException("'" + classId + "." + name + "': variable-length arrays are not part of encoding3d 0.1"); + int width = vector ? property.get("count").asInt() : 1; + ComponentType componentType; + try { + componentType = ComponentType.valueOf(property.get("componentType").asText()); + } catch (IllegalArgumentException error) { + throw new Encoding3dException("component type " + property.get("componentType") + " is not part of encoding3d 0.1"); + } + column = readScalars(componentType, raw, count, width); + restoreStandIns(column, property); + if (property.path("extras").path("FSPM_boolean").asBoolean(false)) { // booleans with missing values + long noData = property.path("noData").asLong(BOOLEAN_NO_DATA); + boolean[] values = new boolean[count], missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) { + long code = (long) column.doubleAt(row); + missing[row] = code == noData; + any |= missing[row]; + values[row] = !missing[row] && code == 1; + } + column = new Column(ComponentType.BOOLEAN, values); + if (any) column.missing = missing; + } else if (width == 1 && !componentType.isFloat() && property.has("noData") + && !name.equals("complex") && !name.equals("parent")) { // missing integers + long noData = property.get("noData").asLong(); + boolean[] missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) { + missing[row] = rawLong(column, row) == noData; + any |= missing[row]; + } + if (any) column.missing = missing; + } + } + default -> throw new Encoding3dException("'" + classId + "." + name + "': property type " + type + " is not part of encoding3d 0.1"); + } + table.columns.put(name, column); + JsonNode declared = variables.path(classId).get(name); + if (declared != null || property.has("description")) { + Variable variable = new Variable(); + if (declared != null) { + variable.unit = declared.path("unit").asText("1"); + variable.role = declared.path("role").asText("state"); + if (declared.hasNonNull("standardName")) variable.standardName = declared.get("standardName").asText(); + if (declared.hasNonNull("kind")) variable.kind = declared.get("kind").asText(); + if (declared.hasNonNull("minimum")) variable.minimum = declared.get("minimum").asDouble(); + if (declared.hasNonNull("maximum")) variable.maximum = declared.get("maximum").asDouble(); + for (JsonNode reference : declared.path("references")) variable.references.add(reference.asText()); + for (JsonNode doi : declared.path("doi")) variable.doi.add(doi.asText()); + if (declared.hasNonNull("format")) variable.format = declared.get("format").asText(); + } + if (property.has("description")) variable.description = property.get("description").asText(); + table.variables.put(name, variable); + } + } + } + + private static void restoreStandIns(Column column, JsonNode property) { + JsonNode infinity = property.path("extras").path("FSPM_infinity"); + if (column.type == ComponentType.FLOAT64) { + double[] values = column.doubles(); + if (property.has("noData")) { + double noData = property.get("noData").asDouble(); + for (int index = 0; index < values.length; index++) if (values[index] == noData) values[index] = Double.NaN; + } + if (!infinity.isMissingNode()) { + double positive = infinity.get("positive").asDouble(), negative = infinity.get("negative").asDouble(); + for (int index = 0; index < values.length; index++) { + if (values[index] == positive) values[index] = Double.POSITIVE_INFINITY; + else if (values[index] == negative) values[index] = Double.NEGATIVE_INFINITY; + } + } + } else if (column.type == ComponentType.FLOAT32) { + float[] values = column.floats(); + if (property.has("noData")) { + float noData = (float) property.get("noData").asDouble(); + for (int index = 0; index < values.length; index++) if (values[index] == noData) values[index] = Float.NaN; + } + if (!infinity.isMissingNode()) { + float positive = (float) infinity.get("positive").asDouble(), negative = (float) infinity.get("negative").asDouble(); + for (int index = 0; index < values.length; index++) { + if (values[index] == positive) values[index] = Float.POSITIVE_INFINITY; + else if (values[index] == negative) values[index] = Float.NEGATIVE_INFINITY; + } + } + } + } + + private static Grid decodeGrid(JsonNode item) { + Grid grid = new Grid(); + grid.kind = item.get("kind").asText(); + grid.shape = new long[item.get("shape").size()]; + for (int index = 0; index < grid.shape.length; index++) grid.shape[index] = item.get("shape").get(index).asLong(); + grid.origin = doubles(item.get("origin")); + grid.spacing = doubles(item.get("spacing")); + List axes = new ArrayList<>(); + for (JsonNode axis : item.get("axes")) axes.add(axis.asText()); + grid.axes = axes; + grid.periodic = new boolean[item.get("periodic").size()]; + for (int index = 0; index < grid.periodic.length; index++) grid.periodic[index] = item.get("periodic").get(index).asBoolean(); + grid.zUp = item.path("zUp").asBoolean(true); + grid.maxLevel = item.path("maxLevel").asLong(0); + grid.cells = item.get("cells").asText(); + return grid; + } + + private static double[] doubles(JsonNode array) { + double[] values = new double[array.size()]; + for (int index = 0; index < values.length; index++) values[index] = array.get(index).asDouble(); + return values; + } + + /** An accessor's values, as doubles, row after row. */ + private static double[] accessorValues(JsonNode document, byte[] binary, int index) { + JsonNode accessor = document.get("accessors").get(index); + int width = switch (accessor.get("type").asText()) { + case "SCALAR" -> 1; + case "VEC2" -> 2; + case "VEC3" -> 3; + case "VEC4" -> 4; + default -> throw new Encoding3dException("accessor type " + accessor.get("type") + " is not used by encoding3d"); + }; + int componentType = accessor.get("componentType").asInt(); + int size = switch (componentType) { + case FLOAT, UNSIGNED_INT -> 4; + case UNSIGNED_SHORT -> 2; + case UNSIGNED_BYTE -> 1; + default -> throw new Encoding3dException("accessor component type " + componentType + " is not used by encoding3d"); + }; + int viewIndex = accessor.get("bufferView").asInt(); + ByteBuffer raw = view(document, binary, viewIndex); + int stride = document.get("bufferViews").get(viewIndex).path("byteStride").asInt(size * width); + int start = accessor.path("byteOffset").asInt(0), count = accessor.get("count").asInt(); + double[] values = new double[count * width]; + for (int item = 0; item < count; item++) + for (int component = 0; component < width; component++) { + int at = start + item * stride + component * size; + if (at + size > raw.limit()) throw new Encoding3dException("an accessor runs past its buffer view"); + values[item * width + component] = switch (componentType) { + case FLOAT -> raw.getFloat(at); + case UNSIGNED_INT -> Integer.toUnsignedLong(raw.getInt(at)); + case UNSIGNED_SHORT -> Short.toUnsignedInt(raw.getShort(at)); + default -> Byte.toUnsignedInt(raw.get(at)); + }; + } + return values; + } + + private static Mesh decodeMesh(JsonNode document, byte[] binary, JsonNode meshEntry, Map tableNames) { + JsonNode primitive = document.get("meshes").get(meshEntry.get("mesh").asInt()).get("primitives").get(0); + JsonNode attributes = primitive.get("attributes"); + Mesh mesh = new Mesh(meshEntry.get("name").asText(), accessorValues(document, binary, attributes.get("POSITION").asInt())); + for (JsonNode feature : primitive.path("extensions").path(FEATURES).path("featureIds")) { + double[] stored = accessorValues(document, binary, attributes.get("_FEATURE_ID_" + feature.get("attribute").asInt()).asInt()); + long[] rows = new long[stored.length]; + long nullRow = feature.has("nullFeatureId") ? feature.get("nullFeatureId").asLong() : Long.MIN_VALUE; + for (int vertex = 0; vertex < rows.length; vertex++) + rows[vertex] = (long) stored[vertex] == nullRow ? EntitySet.NO_ID : (long) stored[vertex]; + mesh.featureRows.put(tableNames.get(feature.get("propertyTable").asInt()), rows); + } + for (JsonNode item : meshEntry.path("display")) { + double[] stored = accessorValues(document, binary, attributes.get(item.get("attribute").asText()).asInt()); + float[] values = new float[stored.length]; + for (int vertex = 0; vertex < values.length; vertex++) values[vertex] = (float) stored[vertex]; + String variable = item.get("variable").asText(); + mesh.display.put(variable, values); + if (item.hasNonNull("entitySet")) mesh.displaySources.put(variable, item.get("entitySet").asText()); + } + if (primitive.has("indices")) { + double[] stored = accessorValues(document, binary, primitive.get("indices").asInt()); + mesh.indices = new long[stored.length]; + for (int index = 0; index < stored.length; index++) mesh.indices[index] = (long) stored[index]; + } + return mesh; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Column.java b/bindings/java/src/main/java/org/openalea/encoding3d/Column.java new file mode 100644 index 0000000..2ff760f --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Column.java @@ -0,0 +1,110 @@ +package org.openalea.encoding3d; + +import java.lang.reflect.Array; +import java.util.Arrays; + +/** + * A column: its values (a Java array, see {@link ComponentType}), {@code width} per row (1: one value per row; 2 or more: + * a fixed-size vector per row, row after row), and which rows are missing ({@code missing}, one flag per row, null when + * none is; NaN in a float column). + */ +public final class Column { + public final ComponentType type; + public final Object values; + public final int width; + public boolean[] missing; + + public Column(ComponentType type, Object values, int width) { + if (!values.getClass().isArray()) throw new Encoding3dException("a column's values are an array"); + Class expected = switch (type) { + case INT8, UINT8 -> byte.class; + case INT16, UINT16 -> short.class; + case INT32, UINT32 -> int.class; + case INT64, UINT64 -> long.class; + case FLOAT32 -> float.class; + case FLOAT64 -> double.class; + case BOOLEAN -> boolean.class; + case STRING -> String.class; + }; + if (values.getClass().getComponentType() != expected) + throw new Encoding3dException("a " + type + " column holds " + expected.getSimpleName() + " values"); + if (width < 1) throw new Encoding3dException("a column's width is at least 1"); + this.type = type; + this.values = values; + this.width = width; + } + + public Column(ComponentType type, Object values) { + this(type, values, 1); + } + + public static Column of(long... values) { return new Column(ComponentType.INT64, values); } + public static Column of(int... values) { return new Column(ComponentType.INT32, values); } + public static Column of(double... values) { return new Column(ComponentType.FLOAT64, values); } + public static Column of(float... values) { return new Column(ComponentType.FLOAT32, values); } + public static Column of(boolean... values) { return new Column(ComponentType.BOOLEAN, values); } + public static Column of(String... values) { return new Column(ComponentType.STRING, values); } + /** A vector column: {@code width} values per row, row after row. */ + public static Column vectors(int width, double... values) { return new Column(ComponentType.FLOAT64, values, width); } + + /** Marks rows as missing (one flag per row); returns the column, for chaining. */ + public Column withMissing(boolean... rows) { + if (rows.length != rows()) throw new Encoding3dException("one missing flag per row"); + missing = rows; + return this; + } + + public boolean isMissing(int row) { + return missing != null && missing[row]; + } + + public boolean hasMissing() { + if (missing != null) for (boolean flag : missing) if (flag) return true; + return false; + } + + /** The number of values (rows × width). */ + public int size() { + return Array.getLength(values); + } + + public int rows() { + return size() / width; + } + + public long[] longs() { return (long[]) values; } + public int[] ints() { return (int[]) values; } + public double[] doubles() { return (double[]) values; } + public float[] floats() { return (float[]) values; } + public boolean[] booleans() { return (boolean[]) values; } + public String[] strings() { return (String[]) values; } + + /** The value at {@code index} as a double (unsigned integers read as unsigned). */ + public double doubleAt(int index) { + return switch (type) { + case INT8 -> ((byte[]) values)[index]; + case UINT8 -> Byte.toUnsignedInt(((byte[]) values)[index]); + case INT16 -> ((short[]) values)[index]; + case UINT16 -> Short.toUnsignedInt(((short[]) values)[index]); + case INT32 -> ((int[]) values)[index]; + case UINT32 -> Integer.toUnsignedLong(((int[]) values)[index]); + case INT64 -> ((long[]) values)[index]; + case UINT64 -> Double.parseDouble(Long.toUnsignedString(((long[]) values)[index])); + case FLOAT32 -> ((float[]) values)[index]; + case FLOAT64 -> ((double[]) values)[index]; + default -> throw new Encoding3dException("a " + type + " column has no numbers"); + }; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Column column)) return false; + return type == column.type && width == column.width && Arrays.deepEquals(new Object[] {values}, new Object[] {column.values}) + && hasMissing() == column.hasMissing() && (!hasMissing() || Arrays.equals(missing, column.missing)); + } + + @Override + public int hashCode() { + return type.hashCode() * 31 + width; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/ComponentType.java b/bindings/java/src/main/java/org/openalea/encoding3d/ComponentType.java new file mode 100644 index 0000000..2a22b27 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/ComponentType.java @@ -0,0 +1,33 @@ +package org.openalea.encoding3d; + +/** + * The type of a column's values. Integers are held in the Java array of their width (UINT8 in a byte[], UINT64 in a + * long[]), their bits as stored; booleans in a boolean[]; text, labels and edge types in a String[]. + */ +public enum ComponentType { + INT8(1, "int8"), UINT8(1, "uint8"), INT16(2, "int16"), UINT16(2, "uint16"), INT32(4, "int32"), + UINT32(4, "uint32"), INT64(8, "int64"), UINT64(8, "uint64"), FLOAT32(4, "float32"), FLOAT64(8, "float64"), + BOOLEAN(0, "bool"), STRING(0, "string"); + + /** Bytes per value (0: booleans and text). */ + public final int size; + /** The type's name in the conformance files' expected values (numpy's, or "string"). */ + public final String dtype; + + ComponentType(int size, String dtype) { + this.size = size; + this.dtype = dtype; + } + + public boolean isNumber() { + return size > 0; + } + + public boolean isFloat() { + return this == FLOAT32 || this == FLOAT64; + } + + public boolean isUnsigned() { + return this == UINT8 || this == UINT16 || this == UINT32 || this == UINT64; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/ConformanceRun.java b/bindings/java/src/main/java/org/openalea/encoding3d/ConformanceRun.java new file mode 100644 index 0000000..fe53a5e --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/ConformanceRun.java @@ -0,0 +1,47 @@ +package org.openalea.encoding3d; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.stream.Stream; + +/** + * The Java codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump it + * as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). Then + * .github/actions/conformance compares the dumps with the expected values and validates the written files. + * + * 
java -jar encoding3d.jar CONFORMANCE OUT
+ */ +public final class ConformanceRun { + private ConformanceRun() {} + + public static void main(String[] arguments) throws IOException { + if (arguments.length != 2) { + System.err.println("usage: java -jar encoding3d.jar CONFORMANCE OUT"); + System.exit(2); + } + Path conformance = Path.of(arguments[0]), out = Path.of(arguments[1]); + List files; + try (Stream paths = Files.walk(conformance)) { + files = paths.filter(path -> path.toString().endsWith(".glb")) + .filter(path -> !conformance.relativize(path).startsWith("coupling")).sorted().toList(); // sessions + } + for (Path file : files) { + Path relative = conformance.relativize(file); + Frame frame = Codec.read(file); + String name = relative.toString(); + Path dump = out.resolve("dumps").resolve(name.substring(0, name.length() - ".glb".length()) + ".json"); + Path written = out.resolve("written").resolve(relative); + Files.createDirectories(dump.getParent()); + Files.createDirectories(written.getParent()); + Files.writeString(dump, Glb.MAPPER.writeValueAsString(Dump.frameToJson(frame)) + "\n"); + Codec.write(written, frame); // written again from what was read + } + Path index = conformance.resolve("growing_sequence").resolve(Sequences.INDEX_FILE); + if (Files.exists(index)) + Files.copy(index, out.resolve("written").resolve("growing_sequence").resolve(Sequences.INDEX_FILE), + StandardCopyOption.REPLACE_EXISTING); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/DataStructure.java b/bindings/java/src/main/java/org/openalea/encoding3d/DataStructure.java new file mode 100644 index 0000000..2885685 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/DataStructure.java @@ -0,0 +1,42 @@ +package org.openalea.encoding3d; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * One plant population's graph, or one environment model's grid. origin: its frame in the scene, a 4×4 matrix row by + * row. extras: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); other + * readers keep them without interpreting them. + */ +public final class DataStructure { + public String id; + public String kind = "graph"; + public final List entitySets = new ArrayList<>(); + public final List relations = new ArrayList<>(); + public Table scalars; + public Grid grid; + public final List meshes = new ArrayList<>(); + public double[] origin = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + public String platform; + public String model; + public ObjectNode extras = JsonNodeFactory.instance.objectNode(); + /** The platform's labels to the vocabulary's. */ + public final java.util.Map labels = new java.util.LinkedHashMap<>(); + + public DataStructure(String id) { + this.id = id; + } + + public DataStructure(String id, String kind) { + this.id = id; + this.kind = kind; + } + + public EntitySet entitySet(String name) { + for (EntitySet set : entitySets) if (set.name.equals(name)) return set; + throw new Encoding3dException("data structure '" + id + "' has no entity set '" + name + "'"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Dump.java b/bindings/java/src/main/java/org/openalea/encoding3d/Dump.java new file mode 100644 index 0000000..915e394 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Dump.java @@ -0,0 +1,211 @@ +package org.openalea.encoding3d; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md), and + * the comparison {@code encoding3d compare} makes. + */ +public final class Dump { + private Dump() {} + + private static void addValue(ArrayNode array, Column column, int index) { + if (column.width == 1 && column.isMissing(index)) { // missing values: null + array.addNull(); + return; + } + switch (column.type) { + case FLOAT32, FLOAT64 -> { + double value = column.type == ComponentType.FLOAT32 ? column.floats()[index] : column.doubles()[index]; + if (Double.isNaN(value)) array.addNull(); + else if (Double.isInfinite(value)) array.add(value > 0 ? "Infinity" : "-Infinity"); + else array.add(value); + } + case BOOLEAN -> array.add(column.booleans()[index]); + case STRING -> array.add(column.strings()[index]); + case UINT64 -> array.add(new BigInteger(Long.toUnsignedString(column.longs()[index]))); + case INT64 -> array.add(column.longs()[index]); + default -> array.add((long) column.doubleAt(index)); + } + } + + private static ArrayNode columnJson(Column column) { + ArrayNode items = Glb.MAPPER.createArrayNode(); + if (column.width == 1) { + for (int index = 0; index < column.size(); index++) addValue(items, column, index); + } else { + for (int row = 0; row < column.rows(); row++) { + ArrayNode vector = items.addArray(); + for (int component = 0; component < column.width; component++) addValue(vector, column, row * column.width + component); + } + } + return items; + } + + private static ObjectNode tableJson(Table table) { + ObjectNode entry = Glb.MAPPER.createObjectNode().put("name", table.name).put("count", table.rowCount()); + ObjectNode columns = entry.putObject("columns"); + for (Map.Entry item : table.columns.entrySet()) { + ObjectNode column = columns.putObject(item.getKey()).put("dtype", item.getValue().type.dtype); + column.set("values", columnJson(item.getValue())); + } + ObjectNode declarations = Glb.MAPPER.createObjectNode(); + for (Map.Entry item : table.variables.entrySet()) { + Variable variable = item.getValue(); + ObjectNode declared = declarations.putObject(item.getKey()).put("unit", variable.unit).put("role", variable.role); + if (variable.standardName != null) declared.put("standard_name", variable.standardName); + if (variable.kind != null) declared.put("kind", variable.kind); + if (variable.description != null) declared.put("description", variable.description); + if (variable.minimum != null) declared.put("minimum", variable.minimum); + if (variable.maximum != null) declared.put("maximum", variable.maximum); + if (!variable.references.isEmpty()) variable.references.forEach(declared.putArray("references")::add); + if (!variable.doi.isEmpty()) variable.doi.forEach(declared.putArray("doi")::add); + if (variable.format != null) declared.put("format", variable.format); + } + if (!declarations.isEmpty()) entry.set("variables", declarations); + return entry; + } + + private static ArrayNode numbers(double[] values) { + ArrayNode array = Glb.MAPPER.createArrayNode(); + for (double value : values) array.add(value); + return array; + } + + /** A frame as plain JSON values: NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. */ + public static ObjectNode frameToJson(Frame frame) { + ObjectNode dump = Glb.MAPPER.createObjectNode().put("time", frame.time).put("time_unit", frame.timeUnit).put("step", frame.step); + ArrayNode dataStructures = dump.putArray("data_structures"); + for (DataStructure dataStructure : frame.dataStructures) { + ObjectNode entry = dataStructures.addObject().put("id", dataStructure.id).put("kind", dataStructure.kind); + ArrayNode origin = entry.putArray("origin"); + for (int row = 0; row < 4; row++) { + ArrayNode values = origin.addArray(); + for (int column = 0; column < 4; column++) values.add(dataStructure.origin[row * 4 + column]); + } + ArrayNode sets = entry.putArray("entity_sets"); + for (EntitySet set : dataStructure.entitySets) { + ObjectNode item = tableJson(set); + if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + if (set.scale != null) item.put("scale", set.scale); + sets.add(item); + } + if (dataStructure.platform != null) entry.put("platform", dataStructure.platform); + if (dataStructure.model != null) entry.put("model", dataStructure.model); + if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); + if (!dataStructure.labels.isEmpty()) { + ObjectNode labels = entry.putObject("labels"); + dataStructure.labels.forEach(labels::put); + } + if (!dataStructure.relations.isEmpty()) { + ArrayNode relations = entry.putArray("relations"); + for (Relation relation : dataStructure.relations) + relations.add(tableJson(relation).put("source_set", relation.sourceSet).put("target_set", relation.targetSet)); + } + if (dataStructure.scalars != null) entry.set("scalars", tableJson(dataStructure.scalars)); + if (dataStructure.grid != null) { + Grid grid = dataStructure.grid; + ObjectNode item = entry.putObject("grid").put("kind", grid.kind); + ArrayNode shape = item.putArray("shape"); + for (long size : grid.shape) shape.add(size); + item.set("origin", numbers(grid.origin)); + item.set("spacing", numbers(grid.spacing)); + grid.axes.forEach(item.putArray("axes")::add); + ArrayNode periodic = item.putArray("periodic"); + for (boolean value : grid.periodic) periodic.add(value); + item.put("z_up", grid.zUp).put("max_level", grid.maxLevel).put("cells", grid.cells); + } + if (!dataStructure.meshes.isEmpty()) { + ArrayNode meshes = entry.putArray("meshes"); + for (Mesh mesh : dataStructure.meshes) { + ObjectNode item = meshes.addObject().put("name", mesh.name); + ArrayNode positions = item.putArray("positions_float32"); + for (int vertex = 0; vertex < mesh.vertexCount(); vertex++) { + ArrayNode position = positions.addArray(); + for (int axis = 0; axis < 3; axis++) position.add((double) (float) mesh.positions[vertex * 3 + axis]); + } + ObjectNode featureRows = item.putObject("feature_rows"); + for (Map.Entry rows : mesh.featureRows.entrySet()) { + ArrayNode values = featureRows.putArray(rows.getKey()); + for (long row : rows.getValue()) values.add(row); + } + if (mesh.indices != null) { + ArrayNode indices = item.putArray("indices"); + for (long index : mesh.indices) indices.add(index); + } + if (!mesh.display.isEmpty()) { + ObjectNode display = item.putObject("display"); + for (Map.Entry values : mesh.display.entrySet()) + display.set(values.getKey(), columnJson(new Column(ComponentType.FLOAT32, values.getValue()))); + } + } + } + } + ArrayNode mappings = dump.putArray("mappings"); + for (Mapping mapping : frame.mappings) { + ObjectNode item = tableJson(mapping); + item.putArray("source").add(mapping.sourceDataStructure).add(mapping.sourceSet); + item.putArray("target").add(mapping.targetDataStructure).add(mapping.targetSet); + if (mapping.method == null) item.putNull("method"); else item.put("method", mapping.method); + mappings.add(item); + } + if (frame.update != null) { // a partial frame: its sender and removed entities, sorted + ObjectNode update = dump.putObject("update").put("participant", frame.update.participant); + ArrayNode removed = update.putArray("removed"); + for (Map.Entry item : frame.update.removed.entrySet()) { + String[] key = item.getKey().split("/", 2); + ObjectNode entry = removed.addObject().put("data_structure", key[0]).put("entity_set", key[1]); + ArrayNode ids = entry.putArray("ids"); + for (long id : item.getValue()) ids.add(id); + } + } + return dump; + } + + /** + * The differences between two JSON values, as {@code encoding3d compare} finds them: objects by key, arrays in order, + * numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). + */ + public static List compare(JsonNode actual, JsonNode expected, String path) { + List found = new ArrayList<>(); + if (expected.isObject()) { + if (!actual.isObject()) return List.of(path + ": an object expected"); + for (Iterator names = expected.fieldNames(); names.hasNext(); ) { + String name = names.next(); + if (!actual.has(name)) found.add(path + ": missing key '" + name + "'"); + else found.addAll(compare(actual.get(name), expected.get(name), path + "." + name)); + } + for (Iterator names = actual.fieldNames(); names.hasNext(); ) { + String name = names.next(); + if (!expected.has(name)) found.add(path + ": unexpected key '" + name + "'"); + } + return found; + } + if (expected.isArray()) { + if (!actual.isArray()) return List.of(path + ": an array expected"); + if (actual.size() != expected.size()) + return List.of(path + ": " + actual.size() + " items, " + expected.size() + " expected"); + for (int index = 0; index < expected.size(); index++) + found.addAll(compare(actual.get(index), expected.get(index), path + "[" + index + "]")); + return found; + } + if (expected.isNumber()) { + if (actual.isNumber() && actual.asDouble() == expected.asDouble()) return found; + return List.of(path + ": " + actual + ", " + expected + " expected"); + } + if (!actual.equals(expected)) return List.of(path + ": " + actual + ", " + expected + " expected"); + return found; + } + + public static List compare(JsonNode actual, JsonNode expected) { + return compare(actual, expected, "$"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Encoding3dException.java b/bindings/java/src/main/java/org/openalea/encoding3d/Encoding3dException.java new file mode 100644 index 0000000..5355831 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Encoding3dException.java @@ -0,0 +1,14 @@ +package org.openalea.encoding3d; + +/** What the specification forbids, or a file that is not encoding3d. */ +public class Encoding3dException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public Encoding3dException(String message) { + super(message); + } + + public Encoding3dException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/EntitySet.java b/bindings/java/src/main/java/org/openalea/encoding3d/EntitySet.java new file mode 100644 index 0000000..d911750 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/EntitySet.java @@ -0,0 +1,38 @@ +package org.openalea.encoding3d; + +/** + * The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (no rank). Reserved + * columns: id (INT64, stable across frames, never reused), label (text), complex (the id of the containing entity at a + * coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), edge_type ("none", + * "successor", "branch"); an adaptive grid's cells: level and index. + */ +public class EntitySet extends Table { + /** The id of no entity. */ + public static final long NO_ID = -1; + + public Long rank; + /** Its base scale (Plant, Axis, ..., Connection), or "mixed". */ + public String scale; + + public EntitySet(String name, Long rank) { + super(name); + this.rank = rank; + } + + public EntitySet(String name) { + this(name, null); + } + + @Override + public Kind kind() { + return Kind.ENTITY_SET; + } + + /** The ids of its entities: its id column, or a grid's row numbers (from 0) when it has none. */ + public long[] ids() { + if (has("id")) return column("id").longs(); + long[] rows = new long[rowCount()]; + for (int row = 0; row < rows.length; row++) rows[row] = row; + return rows; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Frame.java b/bindings/java/src/main/java/org/openalea/encoding3d/Frame.java new file mode 100644 index 0000000..b915156 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Frame.java @@ -0,0 +1,29 @@ +package org.openalea.encoding3d; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A scene at one time step (time in timeUnit, step), or a parameter set. unknown: the content of a read file this + * library does not interpret (other root extensions, the root's extras), written back as it was. + */ +public final class Frame { + public final List dataStructures = new ArrayList<>(); + public final List mappings = new ArrayList<>(); + public double time = 0.0; + public String timeUnit = "d"; + public long step = 0; + public ObjectNode writer = JsonNodeFactory.instance.objectNode(); + public String vocabularyVersion; + /** Set for a partial frame, a coupling session's message. */ + public Update update; + public ObjectNode unknown = JsonNodeFactory.instance.objectNode(); + + public DataStructure dataStructure(String id) { + for (DataStructure dataStructure : dataStructures) if (dataStructure.id.equals(id)) return dataStructure; + throw new Encoding3dException("the frame has no data structure '" + id + "'"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Glb.java b/bindings/java/src/main/java/org/openalea/encoding3d/Glb.java new file mode 100644 index 0000000..dea2b44 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Glb.java @@ -0,0 +1,102 @@ +package org.openalea.encoding3d; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary chunk + * padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. + */ +public final class Glb { + private static final int MAGIC = 0x46546C67; // "glTF" + private static final int VERSION = 2; + private static final int JSON_CHUNK = 0x4E4F534A; // "JSON" + private static final int BINARY_CHUNK = 0x004E4942; // "BIN\0" + static final ObjectMapper MAPPER = new ObjectMapper(); + + /** A GLB file's JSON document and binary buffer. */ + public record Contents(JsonNode document, byte[] binary) {} + + private Glb() {} + + static int padding(long length) { + return (int) ((8 - length % 8) % 8); + } + + /** The bytes of a GLB file from its JSON document and its binary buffer (both padded to 8 bytes). */ + public static byte[] pack(JsonNode document, byte[] binary) { + byte[] text; + try { + text = MAPPER.writeValueAsBytes(document); + } catch (IOException error) { + throw new Encoding3dException("the document cannot be written as JSON", error); + } + int textLength = text.length + padding(text.length); + int binaryLength = binary.length + padding(binary.length); + long total = 12L + 8 + textLength + (binary.length == 0 ? 0 : 8 + binaryLength); + if (total > 0xFFFFFFFFL) throw new Encoding3dException("a GLB file is at most 4 GiB"); + ByteBuffer buffer = ByteBuffer.allocate((int) total).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(MAGIC).putInt(VERSION).putInt((int) total).putInt(textLength).putInt(JSON_CHUNK).put(text); + for (int index = text.length; index < textLength; index++) buffer.put((byte) ' '); + if (binary.length > 0) { + buffer.putInt(binaryLength).putInt(BINARY_CHUNK).put(binary); + buffer.put(new byte[binaryLength - binary.length]); + } + return buffer.array(); + } + + /** The JSON document (keys in their order) and the binary buffer of a GLB file's bytes. */ + public static Contents unpack(byte[] data) { + if (data.length < 20) throw new Encoding3dException("not a GLB file: too short"); + ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + if (buffer.getInt(0) != MAGIC) throw new Encoding3dException("not a GLB file: wrong magic"); + if (buffer.getInt(4) != VERSION) throw new Encoding3dException("GLB version " + buffer.getInt(4) + ", 2 expected"); + long total = Integer.toUnsignedLong(buffer.getInt(8)); + if (total != data.length) + throw new Encoding3dException("GLB header length " + total + ", the file holds " + data.length + " bytes"); + int offset = 12; + JsonNode document = null; + byte[] binary = new byte[0]; + while (offset < total) { + int length = buffer.getInt(offset), type = buffer.getInt(offset + 4); + if (length < 0 || offset + 8L + length > total) throw new Encoding3dException("a GLB chunk runs past the end of the file"); + if (type == JSON_CHUNK) { + try { + document = MAPPER.readTree(Arrays.copyOfRange(data, offset + 8, offset + 8 + length)); + } catch (IOException error) { + throw new Encoding3dException("the GLB file's JSON chunk is not JSON", error); + } + } else if (type == BINARY_CHUNK) { + binary = Arrays.copyOfRange(data, offset + 8, offset + 8 + length); + } + offset += 8 + length; + } + if (document == null) throw new Encoding3dException("GLB file without a JSON chunk"); + return new Contents(document, binary); + } + + /** Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. */ + static final class BufferBuilder { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final com.fasterxml.jackson.databind.node.ArrayNode views = MAPPER.createArrayNode(); + + int add(byte[] bytes, int target) { + data.write(new byte[padding(data.size())], 0, padding(data.size())); + var view = views.addObject(); + view.put("buffer", 0).put("byteOffset", data.size()).put("byteLength", bytes.length); + if (target != 0) view.put("target", target); + data.write(bytes, 0, bytes.length); + return views.size() - 1; + } + + int add(byte[] bytes) { + return add(bytes, 0); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Grid.java b/bindings/java/src/main/java/org/openalea/encoding3d/Grid.java new file mode 100644 index 0000000..364ffd4 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Grid.java @@ -0,0 +1,20 @@ +package org.openalea.encoding3d; + +import java.util.List; + +/** + * A grid's layout. Regular: shape cells of size spacing from origin, along axes, periodic per axis, z pointing up or + * down (zUp). Adaptive: the same for its base level, refined up to maxLevel, its leaf cells (an entity set with level + * and index columns) explicit. cells: the name of the cells' entity set. + */ +public final class Grid { + public String kind = "regular"; + public long[] shape = {}; + public double[] origin = {}; + public double[] spacing = {}; + public List axes = List.of("x", "y", "z"); + public boolean[] periodic = {}; + public boolean zUp = true; + public long maxLevel = 0; + public String cells = "Cell"; +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Mapping.java b/bindings/java/src/main/java/org/openalea/encoding3d/Mapping.java new file mode 100644 index 0000000..b8cb422 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Mapping.java @@ -0,0 +1,25 @@ +package org.openalea.encoding3d; + +/** + * Weights between the entities of two data structures: source and target ids, weight. source and target: a data + * structure's id and an entity set's name. + */ +public class Mapping extends Table { + public String sourceDataStructure, sourceSet, targetDataStructure, targetSet; + public String method; + + public Mapping(String name, String sourceDataStructure, String sourceSet, String targetDataStructure, + String targetSet, String method) { + super(name); + this.sourceDataStructure = sourceDataStructure; + this.sourceSet = sourceSet; + this.targetDataStructure = targetDataStructure; + this.targetSet = targetSet; + this.method = method; + } + + @Override + public Kind kind() { + return Kind.MAPPING; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Mesh.java b/bindings/java/src/main/java/org/openalea/encoding3d/Mesh.java new file mode 100644 index 0000000..304afa7 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Mesh.java @@ -0,0 +1,28 @@ +package org.openalea.encoding3d; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Triangles: positions (3 per vertex, in the data structure's frame), three consecutive vertices per triangle, or + * indices (3 per triangle, from 0) into them; featureRows: per entity set, the row (from 0) of each vertex's entity in + * that set, NO_ID for none; display: per variable, its float copy per vertex, from the entity set displaySources. + */ +public final class Mesh { + public String name; + public double[] positions; + public final Map featureRows = new LinkedHashMap<>(); + public long[] indices; + public final Map display = new LinkedHashMap<>(); + public final Map displaySources = new HashMap<>(); + + public Mesh(String name, double[] positions) { + this.name = name; + this.positions = positions; + } + + public int vertexCount() { + return positions.length / 3; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Relation.java b/bindings/java/src/main/java/org/openalea/encoding3d/Relation.java new file mode 100644 index 0000000..6ec58b8 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Relation.java @@ -0,0 +1,21 @@ +package org.openalea.encoding3d; + +/** + * An edge table: source and target entity ids (in the entity sets sourceSet and targetSet), an optional id per edge, + * an optional relation label per edge, and edge variables. + */ +public class Relation extends Table { + public String sourceSet; + public String targetSet; + + public Relation(String name, String sourceSet, String targetSet) { + super(name); + this.sourceSet = sourceSet; + this.targetSet = targetSet; + } + + @Override + public Kind kind() { + return Kind.RELATION; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Sequences.java b/bindings/java/src/main/java/org/openalea/encoding3d/Sequences.java new file mode 100644 index 0000000..642f0b2 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Sequences.java @@ -0,0 +1,72 @@ +package org.openalea.encoding3d; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. */ +public final class Sequences { + public static final String INDEX_FILE = "scene.fspm.json"; + + /** One frame of a sequence's index; path: its file's absolute path, when read. */ + public record Entry(double time, long step, String file, Path path) {} + + /** A sequence's index: its frames, in time order. */ + public record Index(String version, String timeUnit, List frames) {} + + private Sequences() {} + + public static void writeIndex(Path path, Index index) throws IOException { + ObjectNode document = Glb.MAPPER.createObjectNode().put("fspmIndex", index.version()).put("timeUnit", index.timeUnit()); + ArrayNode frames = document.putArray("frames"); + for (Entry entry : index.frames()) frames.addObject().put("time", entry.time()).put("step", entry.step()).put("file", entry.file()); + Files.writeString(path, Glb.MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(document) + "\n", + StandardCharsets.UTF_8); + } + + public static Index readIndex(Path path) throws IOException { + JsonNode document = Glb.MAPPER.readTree(path.toFile()); + Path directory = path.toAbsolutePath().getParent(); + List frames = new ArrayList<>(); + for (JsonNode item : document.path("frames")) + frames.add(new Entry(item.get("time").asDouble(), item.get("step").asLong(), item.get("file").asText(), + directory.resolve(item.get("file").asText()))); + return new Index(document.path("fspmIndex").asText(Codec.SPECIFICATION_VERSION), document.path("timeUnit").asText("d"), frames); + } + + /** Writes frames to {@code directory}, one GLB each (name_000000.glb, ...), and their index; the index's path. */ + public static Path writeSequence(Path directory, List frames, String name) throws IOException { + Files.createDirectories(directory); + List entries = new ArrayList<>(); + TreeSet timeUnits = new TreeSet<>(); + for (int number = 0; number < frames.size(); number++) { + String file = String.format("%s_%06d.glb", name, number); + Codec.write(directory.resolve(file), frames.get(number)); + entries.add(new Entry(frames.get(number).time, frames.get(number).step, file, null)); + timeUnits.add(frames.get(number).timeUnit); + } + if (timeUnits.size() > 1) throw new Encoding3dException("the frames' times are in several units: " + timeUnits); + Path index = directory.resolve(INDEX_FILE); + writeIndex(index, new Index(Codec.SPECIFICATION_VERSION, timeUnits.isEmpty() ? "d" : timeUnits.first(), entries)); + return index; + } + + public static Path writeSequence(Path directory, List frames) throws IOException { + return writeSequence(directory, frames, "frame"); + } + + /** The frames of the sequence whose index is {@code path}. */ + public static List readSequence(Path path) throws IOException { + List frames = new ArrayList<>(); + for (Entry entry : readIndex(path).frames()) frames.add(Codec.read(entry.path())); + return frames; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Table.java b/bindings/java/src/main/java/org/openalea/encoding3d/Table.java new file mode 100644 index 0000000..bf4c7c3 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Table.java @@ -0,0 +1,84 @@ +package org.openalea.encoding3d; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Columns of equal length, in their order, and the declarations of its variables: a data structure's scalars. */ +public class Table { + /** The kinds of tables, each with its reserved columns. */ + public enum Kind { + SCALARS(List.of()), + ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order")), + RELATION(List.of("id", "source", "target", "relation")), + MAPPING(List.of("source", "target", "weight")); + + public final List reserved; + + Kind(List reserved) { + this.reserved = reserved; + } + } + + public String name; + public final Map columns = new LinkedHashMap<>(); + public final Map variables = new LinkedHashMap<>(); + + public Table(String name) { + this.name = name; + } + + public Kind kind() { + return Kind.SCALARS; + } + + /** Sets a column (replacing it, or appending it); returns the table, for chaining. */ + public Table set(String columnName, Column column) { + columns.put(columnName, column); + return this; + } + + public Table declare(String columnName, Variable variable) { + variables.put(columnName, variable); + return this; + } + + public Column column(String columnName) { + Column column = columns.get(columnName); + if (column == null) throw new Encoding3dException("table '" + name + "' has no column '" + columnName + "'"); + return column; + } + + public boolean has(String columnName) { + return columns.containsKey(columnName); + } + + public int rowCount() { + return columns.isEmpty() ? 0 : columns.values().iterator().next().rows(); + } + + public boolean isReserved(String columnName) { + return kind().reserved.contains(columnName); + } + + /** The variable columns: all but the reserved ones. */ + public List variableNames() { + List names = new ArrayList<>(); + for (String columnName : columns.keySet()) if (!isReserved(columnName)) names.add(columnName); + return names; + } + + /** Throws when columns have different numbers of rows. */ + public void checkLengths() { + for (Map.Entry entry : columns.entrySet()) { + Column column = entry.getValue(); + if (column.hasMissing() && (isReserved(entry.getKey()) || column.width != 1)) + throw new Encoding3dException("column '" + entry.getKey() + "' of '" + name + + "': missing values only in variables of one value per row"); + if (column.size() % column.width != 0) + throw new Encoding3dException("column '" + entry.getKey() + "' of '" + name + "': its values are not whole rows"); + if (column.rows() != rowCount()) throw new Encoding3dException("table '" + name + "': columns of different lengths"); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Update.java b/bindings/java/src/main/java/org/openalea/encoding3d/Update.java new file mode 100644 index 0000000..a8a436e --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Update.java @@ -0,0 +1,23 @@ +package org.openalea.encoding3d; + +import java.util.Map; +import java.util.TreeMap; + +/** + * What makes a frame partial: a coupling session's message (specification §11). participant: its sender + * ("coordinator" for a turn's inputs); its tables hold some rows and columns only. removed: "data structure/entity set" + * keys (see {@link #key}) to the ids of the entities removed, sorted by key. + */ +public final class Update { + public String participant; + public final Map removed = new TreeMap<>(); + + public Update(String participant) { + this.participant = participant; + } + + /** The key of an entity set in {@link #removed}. */ + public static String key(String dataStructure, String entitySet) { + return dataStructure + "/" + entitySet; + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/Variable.java b/bindings/java/src/main/java/org/openalea/encoding3d/Variable.java new file mode 100644 index 0000000..eb77a6c --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/Variable.java @@ -0,0 +1,47 @@ +package org.openalea.encoding3d; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind + * (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance. + */ +public final class Variable { + public String unit = "1"; + public String standardName; + public String kind; + public String role = "state"; + public String description; + public Double minimum; + public Double maximum; + public List references = new ArrayList<>(); + public List doi = new ArrayList<>(); + /** A text column of dates or times: "date", "date-time" or "time" (ISO 8601). */ + public String format; + + public Variable() {} + + public Variable(String unit, String standardName, String kind, String role) { + this.unit = unit; + this.standardName = standardName; + this.kind = kind; + this.role = role; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Variable variable)) return false; + return Objects.equals(unit, variable.unit) && Objects.equals(standardName, variable.standardName) + && Objects.equals(kind, variable.kind) && Objects.equals(role, variable.role) + && Objects.equals(description, variable.description) && Objects.equals(minimum, variable.minimum) + && Objects.equals(maximum, variable.maximum) && references.equals(variable.references) + && doi.equals(variable.doi) && Objects.equals(format, variable.format); + } + + @Override + public int hashCode() { + return Objects.hash(unit, standardName, kind, role); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Channel.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Channel.java new file mode 100644 index 0000000..43b5426 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Channel.java @@ -0,0 +1,26 @@ +package org.openalea.encoding3d.coupling; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** A participant's channel to the coordinator: a message is a JSON header and, for turns and updates, a partial frame's GLB bytes. */ +public interface Channel extends AutoCloseable { + /** A received message: its header, and its payload or null. */ + record Message(ObjectNode header, byte[] payload) {} + + void send(ObjectNode header, byte[] payload); + + /** The next message; Encoding3dException after {@code timeout} seconds without one (null: wait). */ + Message receive(Double timeout); + + /** As the session says (its welcome's keepMessages): keep the messages read, or delete them (the folder transport). */ + default void keepMessages(boolean keep) {} + + @Override + default void close() {} + + /** A participant's channel to {@code endpoint}: "files:path" (a folder), or a ZeroMQ endpoint (JeroMQ). */ + static Channel of(String endpoint, String participant) { + if (endpoint.startsWith("files:")) return new FolderChannel(endpoint.substring("files:".length()), participant); + return new ZmqChannel(endpoint, participant); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ConformanceParticipant.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ConformanceParticipant.java new file mode 100644 index 0000000..4d849f5 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ConformanceParticipant.java @@ -0,0 +1,38 @@ +package org.openalea.encoding3d.coupling; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.Dump; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * The Java participant under test in the coupling conformance sessions (conformance/coupling, specification §9): + * launched by the coordinator as {@code java ... ConformanceParticipant SESSION OUT}, it joins with the session's + * subject/declaration.json, writes what it reads from each turn to OUT/turn_<step>.json, and sends + * subject/update_<step>.glb as read and written again by this codec. + */ +public final class ConformanceParticipant { + private ConformanceParticipant() {} + + public static void main(String[] arguments) throws Exception { + if (arguments.length != 2) { + System.err.println("usage: ConformanceParticipant SESSION OUT"); + System.exit(2); + } + Path session = Path.of(arguments[0]), out = Path.of(arguments[1]); + Files.createDirectories(out); + ObjectNode declaration = (ObjectNode) Json.MAPPER.readTree(session.resolve("subject").resolve("declaration.json").toFile()); + try (Participant participant = new Participant(declaration, null, null, 120.0)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + String step = String.format("%04d", turn.step()); + Json.MAPPER.writerWithDefaultPrettyPrinter().writeValue(out.resolve("turn_" + step + ".json").toFile(), + Dump.frameToJson(turn.frame())); + byte[] update = Files.readAllBytes(session.resolve("subject").resolve("update_" + step + ".glb")); + participant.sendUpdate(Codec.fromBytes(Codec.toBytes(Codec.fromBytes(update)))); + } + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/FolderChannel.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/FolderChannel.java new file mode 100644 index 0000000..a5aad52 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/FolderChannel.java @@ -0,0 +1,98 @@ +package org.openalea.encoding3d.coupling; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.openalea.encoding3d.Encoding3dException; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A participant's folder queues: {@code //to_coordinator/} and {@code to_participant/}. Message n + * is {@code n.json} (its header, n as 8 digits) and {@code n.glb} (its payload), the payload written first and the + * header renamed into place last. + */ +public final class FolderChannel implements Channel { + private final Path incoming, outgoing; + private long nextRead = 0, nextWrite = 0; + private boolean keep = true; // false: a message's files deleted once read + private final java.util.List readFiles = new java.util.ArrayList<>(); // those read and kept + + public FolderChannel(String folder, String participant) { + incoming = Path.of(folder, participant, "to_participant"); + outgoing = Path.of(folder, participant, "to_coordinator"); + try { + Files.createDirectories(incoming); + Files.createDirectories(outgoing); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + } + + private static String numbered(long number) { + return String.format("%08d", number); + } + + @Override + public void send(ObjectNode header, byte[] payload) { + String stem = numbered(nextWrite); + ObjectNode written = header.deepCopy(); + try { + if (payload != null) { + Files.write(outgoing.resolve(stem + ".glb"), payload); + written.put("payload", stem + ".glb"); + } + Path part = outgoing.resolve(stem + ".json.part"); + Files.writeString(part, written.toString()); + Files.move(part, outgoing.resolve(stem + ".json"), StandardCopyOption.ATOMIC_MOVE); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + nextWrite++; + } + + @Override + public void keepMessages(boolean keep) { + if (keep) return; + this.keep = false; + try { + for (Path file : readFiles) Files.deleteIfExists(file); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + readFiles.clear(); + } + + @Override + public Message receive(Double timeout) { + Path header = incoming.resolve(numbered(nextRead) + ".json"); + long start = System.nanoTime(); + try { + while (!Files.exists(header)) { + if (timeout != null && (System.nanoTime() - start) / 1e9 > timeout) + throw new Encoding3dException("no message in " + timeout + " s"); + Thread.sleep(5); + } + ObjectNode read = (ObjectNode) Json.MAPPER.readTree(Files.readString(header)); + java.util.List files = new java.util.ArrayList<>(java.util.List.of(header)); + byte[] payload = null; + if (read.has("payload")) { + files.add(incoming.resolve(read.get("payload").asText())); + payload = Files.readAllBytes(files.get(1)); + read.remove("payload"); + } + nextRead++; + if (keep) readFiles.addAll(files); + else for (Path file : files) Files.delete(file); + return new Message(read, payload); + } catch (IOException error) { + throw new UncheckedIOException(error); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new Encoding3dException("interrupted while waiting for a message"); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Json.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Json.java new file mode 100644 index 0000000..ff664c8 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Json.java @@ -0,0 +1,10 @@ +package org.openalea.encoding3d.coupling; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** The JSON mapper of the session's headers. */ +final class Json { + static final ObjectMapper MAPPER = new ObjectMapper(); + + private Json() {} +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Participant.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Participant.java new file mode 100644 index 0000000..5d26826 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Participant.java @@ -0,0 +1,101 @@ +package org.openalea.encoding3d.coupling; + +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.Encoding3dException; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Update; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A participant's side of a coupling session (specification §11). Joining sends the declaration (its JSON, see + * spec/schema/coupling.schema.json) and the links it proposes, and waits for the welcome; then + * + * 
+ *     for (Turn turn; (turn = participant.nextTurn()) != null; ) {   // until the coordinator stops the session
+ *         ... turn.frame(): its entity sets, their entities, its inputs
+ *         participant.sendUpdate(frame);                             // its outputs and topology changes
+ *     }
+ *
+ * + * The ids of the entities it creates are in [idOffset, idOffset + idCount). + */ +public final class Participant implements AutoCloseable { + public static final String PROTOCOL_VERSION = "0.3"; + public static final String ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT"; + public static final String PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT"; + + public final String name; + public final long idOffset; + public final long idCount; + public final JsonNode session; + public final JsonNode couplings; + public String stopped; + private final Channel channel; + private final Double timeout; + private long step = -1; + + /** Joins the session at {@code endpoint} (null: the environment's ENCODING3D_ENDPOINT); the declaration's participant + * name, if empty, from ENCODING3D_PARTICIPANT; {@code links}: null for none; {@code timeout}: seconds, null to wait. */ + public Participant(ObjectNode declaration, String endpoint, ArrayNode links, Double timeout) { + this.timeout = timeout; + if (endpoint == null) endpoint = System.getenv(ENDPOINT_VARIABLE); + if (endpoint == null || endpoint.isEmpty()) throw new Encoding3dException("no endpoint: give one, or set " + ENDPOINT_VARIABLE); + String own = declaration.path("participant").asText(""); + if (own.isEmpty()) { + own = System.getenv(PARTICIPANT_VARIABLE); + declaration.put("participant", own); + } + name = own; + channel = Channel.of(endpoint, name); + ObjectNode hello = Json.MAPPER.createObjectNode().put("type", "hello").put("protocol", PROTOCOL_VERSION); + hello.set("declaration", declaration); + hello.set("links", links == null ? Json.MAPPER.createArrayNode() : links); + channel.send(hello, null); + Channel.Message welcome = channel.receive(timeout); + String type = welcome.header().path("type").asText(); + if (type.equals("stop")) + throw new Encoding3dException("the coordinator refused " + name + ": " + welcome.header().path("reason").asText()); + if (!type.equals("welcome")) throw new Encoding3dException("expected a welcome, got " + type); + idOffset = welcome.header().get("idOffset").asLong(); + idCount = welcome.header().get("idCount").asLong(); + session = welcome.header().get("session"); + channel.keepMessages(session.path("keepMessages").asBoolean(true)); + couplings = welcome.header().path("couplings"); + } + + /** The next turn, or null when the coordinator stopped the session ({@link #stopped}: its reason). */ + public Turn nextTurn() { + Channel.Message message = channel.receive(timeout); + String type = message.header().path("type").asText(); + if (type.equals("stop")) { + stopped = message.header().path("reason").asText(""); + return null; + } + if (!type.equals("turn")) throw new Encoding3dException("expected a turn, got " + type); + JsonNode header = message.header(); + step = header.get("step").asLong(); + return new Turn(step, header.get("phase").asText(), header.get("time").asDouble(), header.get("until").asDouble(), + Codec.fromBytes(message.payload())); + } + + /** Sends the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the + * entities it adds or changes); in frame.update.removed, the entities it removed. */ + public void sendUpdate(Frame frame) { + if (frame.update == null) frame.update = new Update(name); + frame.update.participant = name; + channel.send(Json.MAPPER.createObjectNode().put("type", "update").put("step", step), Codec.toBytes(frame)); + } + + /** Tells the coordinator this participant cannot go on (the session stops). */ + public void fail(String message) { + channel.send(Json.MAPPER.createObjectNode().put("type", "error").put("message", message), null); + } + + @Override + public void close() throws Exception { + channel.close(); + } +} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Turn.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Turn.java new file mode 100644 index 0000000..520ad24 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/Turn.java @@ -0,0 +1,9 @@ +package org.openalea.encoding3d.coupling; + +import org.openalea.encoding3d.Frame; + +/** + * A turn: step (0: the initial state, phase "init"), the time it starts from and the time it reaches, and its frame, + * the participant's inputs (frame.update.removed: the entities removed since its last turn). + */ +public record Turn(long step, String phase, double time, double until, Frame frame) {} diff --git a/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ZmqChannel.java b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ZmqChannel.java new file mode 100644 index 0000000..eb67cf0 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/encoding3d/coupling/ZmqChannel.java @@ -0,0 +1,58 @@ +package org.openalea.encoding3d.coupling; + +import java.nio.charset.StandardCharsets; + +import org.openalea.encoding3d.Encoding3dException; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A DEALER socket named after the participant (JeroMQ, an optional dependency: needed only for this transport), + * connected to the coordinator's ROUTER socket; a message is one or two frames, its header then its payload. + */ +public final class ZmqChannel implements Channel { + private final ZContext context = new ZContext(); + private final ZMQ.Socket socket; + + public ZmqChannel(String endpoint, String participant) { + socket = context.createSocket(SocketType.DEALER); + socket.setIdentity(participant.getBytes(StandardCharsets.UTF_8)); + socket.setLinger(1000); + socket.connect(endpoint); + } + + @Override + public void send(ObjectNode header, byte[] payload) { + byte[] text = header.toString().getBytes(StandardCharsets.UTF_8); + if (payload == null) socket.send(text, 0); + else { + socket.send(text, ZMQ.SNDMORE); + socket.send(payload, 0); + } + } + + @Override + public Message receive(Double timeout) { + try (ZMQ.Poller poller = context.createPoller(1)) { + poller.register(socket, ZMQ.Poller.POLLIN); + if (poller.poll(timeout == null ? -1 : (long) (timeout * 1000)) <= 0) + throw new Encoding3dException("no message in " + timeout + " s"); + } + byte[] header = socket.recv(0); + byte[] payload = socket.hasReceiveMore() ? socket.recv(0) : null; + try { + return new Message((ObjectNode) Json.MAPPER.readTree(header), payload); + } catch (java.io.IOException error) { + throw new Encoding3dException("an unreadable message header: " + error.getMessage()); + } + } + + @Override + public void close() { + context.close(); + } +} diff --git a/bindings/java/src/test/java/org/openalea/encoding3d/CodecTest.java b/bindings/java/src/test/java/org/openalea/encoding3d/CodecTest.java new file mode 100644 index 0000000..9f08a94 --- /dev/null +++ b/bindings/java/src/test/java/org/openalea/encoding3d/CodecTest.java @@ -0,0 +1,206 @@ +package org.openalea.encoding3d; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import com.fasterxml.jackson.databind.JsonNode; + +/** The Java codec: the conformance files read as their expected values, written and read back the same, every column type, meshes, sequences, and what the specification forbids refused. */ +class CodecTest { + static final Path CONFORMANCE = Path.of(System.getProperty("encoding3d.conformance", "../../conformance")); + + static Stream conformanceFiles() throws IOException { + try (Stream paths = Files.walk(CONFORMANCE)) { + return paths.filter(path -> path.toString().endsWith(".glb")) + .filter(path -> !CONFORMANCE.relativize(path).startsWith("coupling")).sorted().toList().stream(); + } + } + + static JsonNode expectedOf(Path glb) throws IOException { + String name = glb.getFileName().toString(); + return Glb.MAPPER.readTree(glb.resolveSibling(name.substring(0, name.length() - 4) + ".expected.json").toFile()); + } + + /** The dump as it is written and read as JSON (numbers as JSON numbers). */ + static JsonNode asJson(Frame frame) throws IOException { + return Glb.MAPPER.readTree(Glb.MAPPER.writeValueAsString(Dump.frameToJson(frame))); + } + + @Test + void tenConformanceFiles() throws IOException { + assertEquals(10, conformanceFiles().count()); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void glbRepackedAsItWas(Path file) throws IOException { + Glb.Contents contents = Glb.unpack(Files.readAllBytes(file)); + Glb.Contents again = Glb.unpack(Glb.pack(contents.document(), contents.binary())); + assertArrayEquals(contents.binary(), again.binary()); + assertEquals(contents.document(), again.document()); + assertEquals(0, contents.binary().length % 8); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void readAsExpected(Path file) throws IOException { + List differences = Dump.compare(asJson(Codec.read(file)), expectedOf(file)); + assertTrue(differences.isEmpty(), () -> String.join("\n", differences.subList(0, Math.min(5, differences.size())))); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void writtenAndReadBack(Path file) throws IOException { + Frame frame = Codec.fromBytes(Codec.toBytes(Codec.read(file))); + List differences = Dump.compare(asJson(frame), expectedOf(file)); + assertTrue(differences.isEmpty(), () -> String.join("\n", differences.subList(0, Math.min(5, differences.size())))); + } + + @Test + void columnsOfEveryType() { + EntitySet organs = new EntitySet("Organ", 1L); + organs.set("id", Column.of(0, 1, 2)).set("label", Column.of("Leaf", "Internode", "Leaf")) + .set("parent", Column.of(-1L, 0L, 1L)).set("edge_type", Column.of("none", "successor", "branch")) + .set("with_nan", Column.of(1.0, Double.NaN, -2.5)) + .set("with_infinities", Column.of(Float.POSITIVE_INFINITY, 1f, Float.NEGATIVE_INFINITY)) + .set("flag", Column.of(true, false, true)).set("name", Column.of("é", "", "a b")) + .set("position", Column.vectors(3, 1, 2, 3, 4, 5, 6, 7, 8, 9)) + .set("column_int8", new Column(ComponentType.INT8, new byte[] {1, 2, -3})) + .set("column_uint8", new Column(ComponentType.UINT8, new byte[] {1, 2, (byte) 255})) + .set("column_int16", new Column(ComponentType.INT16, new short[] {1, 2, 3})) + .set("column_uint16", new Column(ComponentType.UINT16, new short[] {1, 2, (short) 65535})) + .set("column_uint32", new Column(ComponentType.UINT32, new int[] {1, 2, -1})) + .set("column_uint64", new Column(ComponentType.UINT64, new long[] {1, 2, -1})); + Variable length = new Variable("m", "length", "extensive", "state"); + length.description = "a length"; + length.minimum = 0.0; + length.references.add("a reference"); + organs.declare("with_nan", length); + DataStructure plants = new DataStructure("plants"); + plants.entitySets.add(organs); + plants.origin = new double[] {1, 0, 0, 0.5, 0, 1, 0, 0, 0, 0, 1, -0.1, 0, 0, 0, 1}; + plants.extras.putArray("platform").add(1).add(2); + Frame frame = new Frame(); + frame.dataStructures.add(plants); + frame.time = 2.5; + frame.timeUnit = "h"; + frame.step = 3; + + Frame readBack = Codec.fromBytes(Codec.toBytes(frame)); + EntitySet set = readBack.dataStructure("plants").entitySet("Organ"); + assertEquals(ComponentType.INT64, set.column("id").type); // ids are INT64 + assertArrayEquals(new double[] {1.0, Double.NaN, -2.5}, set.column("with_nan").doubles()); + assertArrayEquals(new float[] {Float.POSITIVE_INFINITY, 1f, Float.NEGATIVE_INFINITY}, set.column("with_infinities").floats()); + assertArrayEquals(new boolean[] {true, false, true}, set.column("flag").booleans()); + assertArrayEquals(new String[] {"é", "", "a b"}, set.column("name").strings()); + assertEquals(3, set.column("position").width); + assertArrayEquals(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9}, set.column("position").doubles()); + assertArrayEquals(new String[] {"Leaf", "Internode", "Leaf"}, set.column("label").strings()); + for (String name : List.of("column_int8", "column_uint8", "column_int16", "column_uint16", "column_uint32", "column_uint64")) + assertEquals(organs.column(name), set.column(name), name); + assertEquals(255.0, set.column("column_uint8").doubleAt(2)); + assertEquals(length, set.variables.get("with_nan")); + assertArrayEquals(plants.origin, readBack.dataStructure("plants").origin); + assertEquals(plants.extras, readBack.dataStructure("plants").extras); + assertEquals(2.5, readBack.time); + assertEquals("h", readBack.timeUnit); + assertEquals(3, readBack.step); + } + + @Test + void meshes() { + EntitySet segments = new EntitySet("SubOrgan", 6L); + segments.set("id", Column.of(10L, 11L)); + Mesh mesh = new Mesh("roots", new double[] {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}); + mesh.featureRows.put("SubOrgan", new long[] {0, 1, 1, -1}); + mesh.indices = new long[] {0, 1, 2, 0, 2, 3}; + mesh.display.put("radius", new float[] {1, 2, 2, 0}); + mesh.displaySources.put("radius", "SubOrgan"); + DataStructure roots = new DataStructure("roots"); + roots.entitySets.add(segments); + roots.meshes.add(mesh); + Frame frame = new Frame(); + frame.dataStructures.add(roots); + Mesh readBack = Codec.fromBytes(Codec.toBytes(frame)).dataStructures.get(0).meshes.get(0); + assertArrayEquals(mesh.positions, readBack.positions); + assertArrayEquals(mesh.featureRows.get("SubOrgan"), readBack.featureRows.get("SubOrgan")); + assertArrayEquals(mesh.indices, readBack.indices); + assertArrayEquals(mesh.display.get("radius"), readBack.display.get("radius")); + assertEquals(mesh.displaySources, readBack.displaySources); + } + + @Test + void partialFrames() throws IOException { + EntitySet segments = new EntitySet("SubOrgan", 6L); + segments.set("id", Column.of(10L, 11L)).set("complex", Column.of(3L, 3L)).set("sugar", Column.of(1.0, 2.0)); + Frame frame = frameWith(segments); + frame.update = new Update("vpl"); + frame.update.removed.put(Update.key("plants", "SubOrgan"), new long[] {5, 6}); + Frame again = Codec.fromBytes(Codec.toBytes(frame)); + assertEquals("vpl", again.update.participant); + assertArrayEquals(new long[] {5, 6}, again.update.removed.get(Update.key("plants", "SubOrgan"))); + assertEquals(5, asJson(again).get("update").get("removed").get(0).get("ids").get(0).asLong()); + } + + static Frame frameWith(EntitySet set) { + DataStructure plants = new DataStructure("plants"); + plants.entitySets.add(set); + Frame frame = new Frame(); + frame.dataStructures.add(plants); + return frame; + } + + @Test + void refused() { + EntitySet badColumn = new EntitySet("Organ"); + badColumn.set("id", Column.of(0L)).set("1st", Column.of(1.0)); + assertThrows(Encoding3dException.class, () -> Codec.toBytes(frameWith(badColumn))); + Frame badId = new Frame(); + badId.dataStructures.add(new DataStructure("bad id")); + assertThrows(Encoding3dException.class, () -> Codec.toBytes(badId)); + EntitySet badEdge = new EntitySet("Organ"); + badEdge.set("id", Column.of(0L)).set("parent", Column.of(-1L)).set("edge_type", Column.of("root")); + assertThrows(Encoding3dException.class, () -> Codec.toBytes(frameWith(badEdge))); + EntitySet both = new EntitySet("Organ"); + both.set("id", Column.of(0L, 1L)).set("x", Column.of(Double.NaN, -Double.MAX_VALUE)); + assertThrows(Encoding3dException.class, () -> Codec.toBytes(frameWith(both))); + EntitySet lengths = new EntitySet("Organ"); + lengths.set("id", Column.of(0L, 1L)).set("x", Column.of(1.0)); + assertThrows(Encoding3dException.class, () -> Codec.toBytes(frameWith(lengths))); + assertThrows(Encoding3dException.class, () -> Glb.unpack(new byte[] {1, 2, 3})); + assertThrows(Encoding3dException.class, () -> new Column(ComponentType.INT64, new double[] {1.0})); + } + + @Test + void sequences(@TempDir Path folder) throws IOException { + List frames = new ArrayList<>(); + for (int step = 0; step < 3; step++) { + EntitySet plant = new EntitySet("Plant", 1L); + plant.set("id", Column.of(0L)).set("biomass", Column.of((double) step)); + Frame frame = frameWith(plant); + frame.time = step / 24.0; + frame.step = step; + frames.add(frame); + } + Path index = Sequences.writeSequence(folder, frames); + assertEquals(2, Sequences.readIndex(index).frames().get(2).step()); + List readBack = Sequences.readSequence(index); + assertEquals(2.0, readBack.get(2).dataStructures.get(0).entitySets.get(0).column("biomass").doubles()[0]); + for (Sequences.Entry entry : Sequences.readIndex(CONFORMANCE.resolve("growing_sequence").resolve(Sequences.INDEX_FILE)).frames()) + assertTrue(Files.exists(entry.path())); + } +} diff --git a/bindings/julia/README.md b/bindings/julia/README.md new file mode 100644 index 0000000..79b2e8b --- /dev/null +++ b/bindings/julia/README.md @@ -0,0 +1,32 @@ +# encoding3d.jl + +The Julia codec of encoding3d: reading and writing frames and sequences (spec 0.3), a coupling session's participant, +and, as package extensions proposed to their platforms, MultiScaleTreeGraph.jl MTGs and VirtualPlantLab graphs to and +from `graph` data structures. + +- `encoding3d/src/`: the GLB container (`glb.jl`), the data model (`model.jl`), reading and writing (`codec.jl`), + sequences (`sequence.jl`), the conformance dump (`dump.jl`). +- `encoding3d/ext/encoding3dMultiScaleTreeGraphExt.jl`: the MTG adapter, loaded with MultiScaleTreeGraph (PlantSimEngine's, + PlantGeom's graphs). +- `encoding3d/ext/encoding3dPlantGraphsExt.jl`, `encoding3dPlantGeomTurtleExt.jl`: VirtualPlantLab's graphs and their turtle + geometry, loaded with PlantGraphs (and PlantGeomTurtle, PlantGeomPrimitives). +- `encoding3d/test/`: the tests, the conformance files included, depending on no platform. +- `ext/encoding3dZMQExt.jl`: the coupling's ZeroMQ transport, loaded with ZMQ. + +## The platform extensions: proposals + +The MTG.jl and VirtualPlantLab extensions are **first proposals for those platforms' teams to take over**. They load +only when the platform's package is loaded (weak dependencies), so encoding3d's requirements do not include them, and +encoding3d's CI does not run their tests. To test them by hand, from the repository's root: + +```bash +julia --project=bindings/julia/encoding3d/test/multiscaletreegraph -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.instantiate()' +julia --project=bindings/julia/encoding3d/test/multiscaletreegraph bindings/julia/encoding3d/test/multiscaletreegraph/runtests.jl +julia --project=bindings/julia/encoding3d/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.instantiate()' +julia --project=bindings/julia/encoding3d/test/plantgraphs bindings/julia/encoding3d/test/plantgraphs/runtests.jl # Julia ≥ 1.12 +``` +- `conformance_run.jl`: the codec's dumps of the conformance files and its written scenes, for + `.github/actions/conformance`. + +Dependencies: JSON.jl and OrderedCollections.jl. Licence: MIT (`encoding3d/LICENSE`), as VirtualPlantLab; the rest of +encoding3d is CeCILL-C. See [the documentation](../../doc/julia.md). diff --git a/bindings/julia/conformance_run.jl b/bindings/julia/conformance_run.jl new file mode 100644 index 0000000..f8eb7a2 --- /dev/null +++ b/bindings/julia/conformance_run.jl @@ -0,0 +1,29 @@ +# The Julia codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump +# it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). +# Then .github/actions/conformance compares the dumps with the expected values and validates the written files. +# +# julia --project=bindings/julia/encoding3d bindings/julia/conformance_run.jl CONFORMANCE OUT + +using encoding3d + +function main(conformance::AbstractString, out::AbstractString) + for (folder, _, files) in walkdir(conformance) + for name in sort(files) + endswith(name, ".glb") || continue + startswith(relpath(folder, conformance), "coupling") && continue # the coupling sessions + relative = relpath(joinpath(folder, name), conformance) + frame = read_frame(joinpath(conformance, relative)) + dump = joinpath(out, "dumps", relative[1:end-length(".glb")] * ".json") + written = joinpath(out, "written", relative) + mkpath(dirname(dump)) + mkpath(dirname(written)) + write_dump(dump, frame) + write_frame(written, frame) # written again from what was read + end + end + index = joinpath(conformance, "growing_sequence", "scene.fspm.json") + isfile(index) && cp(index, joinpath(out, "written", "growing_sequence", "scene.fspm.json"); force = true) + return 0 +end + +main(ARGS[1], ARGS[2]) diff --git a/bindings/julia/coupling_conformance.jl b/bindings/julia/coupling_conformance.jl new file mode 100644 index 0000000..367730f --- /dev/null +++ b/bindings/julia/coupling_conformance.jl @@ -0,0 +1,23 @@ +# encoding3d.jl's participant under test in the coupling conformance sessions (conformance/coupling, specification §9): +# launched by the coordinator as `julia --project=… coupling_conformance.jl SESSION OUT`, it joins with the session's +# subject/declaration.json, writes what it reads from each turn to OUT/turn_.json, and sends +# subject/update_.glb as read and written again by encoding3d.jl. +# +# encoding3d couple-conformance conformance/coupling --participant "julia --project=env bindings/julia/coupling_conformance.jl" +using encoding3d +using JSON +using ZMQ + +function main(session::AbstractString, out::AbstractString) + mkpath(out) + declaration = declaration_from_json(JSON.parsefile(joinpath(session, "subject", "declaration.json"))) + participant = Participant(declaration; timeout = 120) + for turn in turns(participant) + write_dump(joinpath(out, "turn_$(lpad(turn.step, 4, '0')).json"), turn.frame) + update = from_bytes(to_bytes(read_frame(joinpath(session, "subject", "update_$(lpad(turn.step, 4, '0')).glb")))) + send_update(participant, update) + end + return 0 +end + +exit(main(ARGS[1], ARGS[2])) diff --git a/bindings/julia/encoding3d/LICENSE b/bindings/julia/encoding3d/LICENSE new file mode 100644 index 0000000..629f9e7 --- /dev/null +++ b/bindings/julia/encoding3d/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Tristan Gérault and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/julia/encoding3d/Project.toml b/bindings/julia/encoding3d/Project.toml new file mode 100644 index 0000000..0ddac7f --- /dev/null +++ b/bindings/julia/encoding3d/Project.toml @@ -0,0 +1,34 @@ +name = "encoding3d" +uuid = "b7fd6adf-cc0c-448b-9b1a-23349074d866" +authors = ["Tristan Gérault"] +version = "0.1.0" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" + +[weakdeps] +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" +PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" +PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" + +[extensions] +encoding3dMultiScaleTreeGraphExt = "MultiScaleTreeGraph" +encoding3dPlantGeomTurtleExt = ["PlantGraphs", "PlantGeomTurtle", "PlantGeomPrimitives"] +encoding3dPlantGraphsExt = "PlantGraphs" +encoding3dZMQExt = "ZMQ" + +[compat] +Dates = "1" +JSON = "1" +MultiScaleTreeGraph = "0.16" +PlantGeomPrimitives = "1" +PlantGeomTurtle = "1" +PlantGraphs = "1" +ZMQ = "1" +OrderedCollections = "1, 2" +julia = "1.10" + diff --git a/bindings/julia/encoding3d/ext/encoding3dMultiScaleTreeGraphExt.jl b/bindings/julia/encoding3d/ext/encoding3dMultiScaleTreeGraphExt.jl new file mode 100644 index 0000000..d06e751 --- /dev/null +++ b/bindings/julia/encoding3d/ext/encoding3dMultiScaleTreeGraphExt.jl @@ -0,0 +1,391 @@ +""" +A MultiScaleTreeGraph (MTG.jl) as an encoding3d `graph` data structure, and back: + +| encoding3d | MultiScaleTreeGraph | +|---|---| +| one entity set per scale ≥ 1, `rank` the scale | the nodes of that scale | +| `id` | `node_id` | +| `label` | the node's `symbol` | +| `complex` | its first ancestor at a smaller scale (none for the scene) | +| `parent`, `edge_type` | its parent, when at the same scale and linked by `<` (successor) or `+` (branch) | +| relation `bearer` | its parent otherwise, when it is not its complex linked by `/` (e.g. an axis borne by an internode) | +| `sibling_order` | the node's position among its parent's children | +| `mtg_index` (standard name `mtg_index`) | the node's `index` | +| variables | its attributes: missing where a node has none; dates and times as ISO 8601 text with their `format` | +| scalars | the scene's (the scale 0 root's) attributes | + +MTG.jl's scene node and metadata (`symbols`, `scales`, `description`) are kept in the data structure's +`extras["virtualplantlab"]` (with `"model": "MultiScaleTreeGraph"`), so that `to_mtg` rebuilds the same MTG. + +Status: a first proposal for MTG.jl's (VirtualPlantLab, PlantSimEngine) team to take over, not a part of encoding3d: loaded only when the platform's +package is, encoding3d's requirements and CI do not depend on it; tested by hand (bindings/julia/README.md). +""" +module encoding3dMultiScaleTreeGraphExt + +using Dates +using encoding3d +using encoding3d: Columns, Declarations, json_object, check_identifier +using MultiScaleTreeGraph +using OrderedCollections: OrderedDict + +const PLATFORM = "MultiScaleTreeGraph" +const EXTRAS_KEY = "virtualplantlab" # the extras key: VirtualPlantLab's name in the vocabulary +const METADATA_ATTRIBUTES = (:symbols, :scales, :description) +# MTG.jl's node ids start at 1 and size its attribute store (an id of 2^32 allocates 2^32 rows), encoding3d's start at 0 +# and may be sparse (CPlantBox's segments: 2^32 + node id). to_mtg then numbers the nodes 1, 2, ... in the order of +# their ids, each keeping its id in this attribute; from_mtg writes it back as the id. +const ID_ATTRIBUTE = :encoding3d_id +# An entity whose parent (same scale) is in another complex, e.g. the first segment of a lateral root, whose parent +# segment is in the bearing root: MTG.jl takes a node's complex from its ancestors, so the node stays under its complex +# (by '/') and keeps its parent and edge type in these attributes. +const PARENT_ATTRIBUTE = :encoding3d_parent +const EDGE_TYPE_ATTRIBUTE = :encoding3d_edge_type +const OWN_ATTRIBUTES = (ID_ATTRIBUTE, PARENT_ATTRIBUTE, EDGE_TYPE_ATTRIBUTE) +const LINK_NAMES = Dict(:< => "successor", :+ => "branch", :/ => "decomposition") +const LINKS = Dict(name => link for (link, name) in LINK_NAMES) +const INDEX_COLUMN = "mtg_index" +const INDEX_VARIABLE = Variable(unit = "1", standard_name = "mtg_index", kind = "descriptor", role = "state", + description = "the node's index in its MTG code (1 in Internode1)") +"""The node's position among its parent's children, from 0.""" +position_among_siblings(node) = isroot(node) ? 0 : findfirst(child -> child === node, children(parent(node))) - 1 + +parent_node(node) = isroot(node) ? nothing : parent(node) + +"""The nodes of the MTG in depth-first order, children in their order.""" +function preorder(root) + nodes, stack = Any[], Any[root] + while !isempty(stack) + node = pop!(stack) + push!(nodes, node) + append!(stack, reverse(children(node))) + end + return nodes +end + +"""The first ancestor of `node` at a smaller scale than its own, or nothing.""" +function complex_node(node) + ancestor = parent_node(node) + while ancestor !== nothing && scale(ancestor) >= scale(node) + ancestor = parent_node(ancestor) + end + return ancestor +end + +default_set_name(scale_number, symbols) = + length(symbols) == 1 && occursin(r"^[a-zA-Z_][a-zA-Z0-9_]*$", symbols[1]) ? symbols[1] : "Scale$scale_number" + +# ── Attributes as columns ─────────────────────────────────────────────── + + +"""Columns of `nodes`' attributes (missing where a node has none), by name in order of first appearance, and their +declarations (a date's `format` added). `skip`: attributes not written; the MTG's root's metadata (MTG.jl's `symbols`, +`scales`, `description`) neither.""" +function attribute_columns(nodes, set_name, variables, skip = ()) + attribute_of(node, name) = + isroot(node) && name in METADATA_ATTRIBUTES ? nothing : attribute(node, name; default = nothing) + names = Symbol[] + for node in nodes, name in attribute_names(node) + name in names || name in skip || name in OWN_ATTRIBUTES || (isroot(node) && name in METADATA_ATTRIBUTES) || + push!(names, name) + end + columns, declarations = Columns(), Declarations() + for name in names + if string(name) in encoding3d.ENTITY_SET_RESERVED # a reserved name, e.g. `label` + @warn "attribute '$name' of '$set_name' not written: '$name' is a reserved column of encoding3d" + continue + end + values = Any[attribute_of(node, name) for node in nodes] + converted = encoding3d.column_of_values(values) + if converted === nothing + @warn "attribute '$name' of '$set_name' not written: encoding3d holds numbers, booleans, text, dates " * + "and fixed-size numeric vectors" + continue + end + column_name = string(name) + check_identifier(column_name, "attribute of '$set_name'") + column, format = converted + columns[column_name] = column + declared = get(variables, column_name, nothing) + if format !== nothing # dates and times: their format declared + declared = declared === nothing ? Variable(unit = "1", kind = "descriptor") : deepcopy(declared) + declared.format = format + end + declared === nothing || (declarations[column_name] = declared) + end + return columns, declarations +end + +# ── MTG → data structure ──────────────────────────────────────────────── + +""" + from_mtg(mtg; id="plants", scales=Dict(), ranks=Dict(), variables=Dict(), labels=Dict(), + platform="MultiScaleTreeGraph", model=nothing) -> DataStructure + +The MTG whose root is `mtg` as a `graph` data structure `id`. `scales`: the name of each scale's entity set +(scale => name, e.g. `Dict(1 => "Plant", 2 => "Axis", 3 => "Organ")`; by default the scale's symbol when it has +only one, `Scale` otherwise). `ranks`: each scale's rank (by default the scale), e.g. the vocabulary's base scales' +positions. `variables`: the declarations of the attributes (name => `Variable`: unit, standard name, kind, role), +MTG.jl having no units. `labels`: the symbols => the vocabulary's labels (e.g. `Dict("Internode" => "StemInternode")`). A scale 0 root (MTG.jl's `Scene`) is the data structure itself, its +attributes the scalars. +""" +function encoding3d.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,String}(), ranks = Dict{Int,Int}(), + variables = Dict{String,Variable}(), labels = Dict{String,String}(), platform = PLATFORM, + model = nothing) + variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) + fspm_id(node) = attribute(node, ID_ATTRIBUTE; default = node_id(node)) + nodes = preorder(mtg) + scene = scale(mtg) == 0 ? mtg : nothing + extras = json_object() + by_scale = OrderedDict{Int,Vector{Any}}() + for node in nodes + node === scene && continue + scale(node) >= 1 || throw(ArgumentError("node $(node_id(node)): scale $(scale(node)) below a root's")) + push!(get!(by_scale, scale(node), Any[]), node) + end + sort!(by_scale) + set_names = Dict{Int,String}() + for (scale_number, scale_nodes) in by_scale + name = get(scales, scale_number, nothing) + name === nothing && (name = default_set_name(scale_number, unique(string.(symbol.(scale_nodes))))) + check_identifier(name, "the entity set name of scale $scale_number") + set_names[scale_number] = name + end + + entity_sets = EntitySet[] + bearers = OrderedDict{Tuple{String,String},Vector{Tuple{Int64,Int64,String}}}() + for (scale_number, scale_nodes) in by_scale + set_name = set_names[scale_number] + columns = Columns("id" => Int64[fspm_id(node) for node in scale_nodes], + "label" => String[string(symbol(node)) for node in scale_nodes]) + complexes = [complex_node(node) for node in scale_nodes] + if scale_number != first(keys(by_scale)) + columns["complex"] = Int64[complex === nothing || complex === scene ? NO_ID : fspm_id(complex) + for complex in complexes] + end + parents, edge_types = Int64[], String[] + for (node, complex) in zip(scale_nodes, complexes) + if PARENT_ATTRIBUTE in attribute_names(node) # a parent in another complex (to_mtg) + push!(parents, attribute(node, PARENT_ATTRIBUTE)) + push!(edge_types, attribute(node, EDGE_TYPE_ATTRIBUTE)) + continue + end + tree_parent = parent_node(node) + if tree_parent !== nothing && scale(tree_parent) == scale(node) && link(node) in (:<, :+) + push!(parents, fspm_id(tree_parent)) + push!(edge_types, LINK_NAMES[link(node)]) + continue + end + push!(parents, NO_ID) + push!(edge_types, "none") + tree_parent === nothing && continue + tree_parent === complex && link(node) == :/ && continue # its complex, by decomposition + tree_parent === scene && link(node) == :/ && continue # the scene's component + tree_parent === scene && + throw(ArgumentError("node $(fspm_id(node)) is linked to the scene by '$(link(node))', not '/'")) + pair = (set_name, set_names[scale(tree_parent)]) + push!(get!(bearers, pair, Tuple{Int64,Int64,String}[]), + (fspm_id(node), fspm_id(tree_parent), LINK_NAMES[link(node)])) + end + columns["parent"], columns["edge_type"] = parents, edge_types + columns["sibling_order"] = Int64[position_among_siblings(node) for node in scale_nodes] + columns[INDEX_COLUMN] = Int64[index(node) for node in scale_nodes] + attributes, declarations = attribute_columns(scale_nodes, set_name, variables, (Symbol(INDEX_COLUMN),)) + merge!(columns, attributes) + declarations = merge(Declarations(INDEX_COLUMN => INDEX_VARIABLE), declarations) + push!(entity_sets, EntitySet(set_name; columns = columns, variables = declarations, + rank = get(ranks, scale_number, scale_number))) + end + + relations = Relation[] + for ((source_set, target_set), edges) in bearers + name = length(bearers) == 1 ? "bearer" : "bearer_$(source_set)_$(target_set)" + push!(relations, Relation(name; source_set = source_set, target_set = target_set, + columns = Columns("source" => Int64[edge[1] for edge in edges], + "target" => Int64[edge[2] for edge in edges], + "relation" => String[edge[3] for edge in edges]))) + end + + metadata = json_object() # MTG.jl's, on the root + for name in METADATA_ATTRIBUTES + name in attribute_names(mtg) || continue + value = attribute(mtg, name; default = nothing) # nothing kept: e.g. no description + metadata[string(name)] = value isa MultiScaleTreeGraph.ColumnTable ? + json_object(string(column) => getproperty(value, column) for column in names(value)) : value + end + isempty(metadata) || (extras["metadata"] = metadata) + scalars = nothing + if scene !== nothing + extras["scene"] = json_object("id" => fspm_id(scene), "symbol" => string(symbol(scene)), + "index" => index(scene), "link" => string(link(scene))) + columns, declarations = attribute_columns([scene], "scalars", variables) + isempty(columns) || (scalars = Table("scalars"; columns = columns, variables = declarations)) + end + extras = merge(json_object("model" => PLATFORM, # each set's MTG scale (its rank may differ) + "scales" => json_object(set_names[scale_number] => scale_number + for scale_number in keys(by_scale))), extras) + + return DataStructure(id; kind = "graph", entity_sets = entity_sets, relations = relations, scalars = scalars, + platform = platform, model = model, extras = json_object(EXTRAS_KEY => extras), + labels = labels) +end + +# ── Data structure → MTG ──────────────────────────────────────────────── + +"""The attributes of each row of `table`: Dict(column => value), without the rows' missing ones (`missing`, and NaN, +a float column's missing value: a node has no such attribute); dates and times read back from their text by their +declared `format`.""" +function row_attributes(table, skip) + rows = [Dict{Symbol,Any}() for _ in 1:row_count(table)] + for (name, column) in table.columns + name in skip && continue + format = haskey(table.variables, name) ? table.variables[name].format : nothing + for row in 1:row_count(table) + value = encoding3d.value_of(column, row, format) + ismissing(value) || (value isa AbstractFloat && isnan(value)) || (rows[row][Symbol(name)] = value) + end + end + return rows +end + +""" + to_mtg(data_structure) -> Node + +The `graph` data structure as an MTG (its root node). A data structure written by `from_mtg` is rebuilt as it was: +ids, symbols, scales, indices, links, attributes, the order of children. Another one (e.g. from metafspm) becomes an +MTG with one scale per entity set (its rank), each entity's symbol its label (else its entity set's name), its +variables as attributes, under a `Scene` root holding the scalars. +""" +function encoding3d.to_mtg(data_structure::DataStructure) + data_structure.kind == "graph" || + throw(ArgumentError("data structure '$(data_structure.id)' is a $(data_structure.kind), not a graph")) + extras = get(data_structure.extras, EXTRAS_KEY, json_object()) + get(extras, "model", nothing) == PLATFORM || (extras = json_object()) # another VPL model's: not MTG.jl's + entity_sets = [set for set in data_structure.entity_sets if set.rank !== nothing] + sort!(entity_sets; by = set -> set.rank) + mtg_scales = get(extras, "scales", json_object()) # MTG.jl's own: its scales; else the ranks + scale_of(set) = Int(get(mtg_scales, set.name, set.rank)) + + # Every entity: its scale, symbol, index, attributes; its parent in the tree and the link to it + encoding, attributes, tree_parent = Dict{Int64,Any}(), Dict{Int64,Dict{Symbol,Any}}(), Dict{Int64,Tuple{Int64,Symbol}}() + reserved = (encoding3d.ENTITY_SET_RESERVED..., INDEX_COLUMN) + order_of = Dict{Int64,Int64}() # each entity's position among its siblings + for set in entity_sets + ids = set.columns["id"] + labels = get(set.columns, "label", fill(set.name, row_count(set))) + indices = get(set.columns, INDEX_COLUMN, zeros(Int64, row_count(set))) + rows = row_attributes(set, reserved) + for row in 1:row_count(set) + encoding[ids[row]] = (symbol = labels[row], index = indices[row], scale = scale_of(set)) + attributes[ids[row]] = rows[row] + end + haskey(set.columns, "sibling_order") && foreach(((id, position),) -> order_of[id] = position, + zip(ids, set.columns["sibling_order"])) + complexes = get(set.columns, "complex", fill(NO_ID, row_count(set))) + for row in 1:row_count(set) # complex: by decomposition, '/' + complexes[row] == NO_ID || (tree_parent[ids[row]] = (complexes[row], :/)) + end + end + for relation in data_structure.relations # bearers: the link of their label + startswith(relation.name, "bearer") || continue + labels = get(relation.columns, "relation", fill("branch", row_count(relation))) + for (source, target, label) in zip(relation.columns["source"], relation.columns["target"], labels) + tree_parent[source] = (target, LINKS[label]) + end + end + complex_of = Dict{Int64,Int64}() + for set in entity_sets + haskey(set.columns, "complex") || continue + for (id, complex) in zip(set.columns["id"], set.columns["complex"]) + complex == NO_ID || (complex_of[id] = complex) + end + end + for set in entity_sets # parents at the same scale, '<' or '+' + haskey(set.columns, "parent") || continue + edge_types = set.columns["edge_type"] + for (id, parent_id, edge_type) in zip(set.columns["id"], set.columns["parent"], edge_types) + parent_id == NO_ID && continue + if get(complex_of, id, NO_ID) != get(complex_of, parent_id, NO_ID) # in another complex: kept aside + attributes[id][PARENT_ATTRIBUTE] = parent_id + attributes[id][EDGE_TYPE_ATTRIBUTE] = edge_type + else + tree_parent[id] = (parent_id, edge_type == "successor" ? :< : :+) + end + end + end + + # The root: the scene written by from_mtg, or a new scene when there are several roots or scalars + roots = sort([id for id in keys(encoding) if !haskey(tree_parent, id)]) + scene_entry = get(extras, "scene", nothing) + scene_attributes = data_structure.scalars === nothing ? Dict{Symbol,Any}() : + only(row_attributes(data_structure.scalars, ())) + # MTG.jl's ids: the entities' own, unless they start below 1 or are sparse; then 1, 2, ... in their order + all_ids = sort(collect(keys(encoding))) + renumbered = !isempty(all_ids) && (first(all_ids) < 1 || last(all_ids) > 2 * length(all_ids) + 1) + number_of = renumbered ? Dict(id => position for (position, id) in enumerate(all_ids)) : nothing + mtg_id(id) = renumbered ? number_of[id] : Int(id) + renumbered && foreach(id -> attributes[id][ID_ATTRIBUTE] = id, all_ids) + if scene_entry === nothing && length(roots) == 1 && isempty(scene_attributes) + root_id = only(roots) + root_encoding = encoding[root_id] + root = Node(mtg_id(root_id), MutableNodeMTG(:/, Symbol(root_encoding.symbol), root_encoding.index, + root_encoding.scale), attributes[root_id]) + roots = Int64[] + else + scene_entry === nothing && + (scene_entry = json_object("id" => renumbered ? length(all_ids) + 1 : maximum(all_ids; init = 0) + 1, + "symbol" => "Scene", "index" => 0, "link" => "/")) + root = Node(Int(scene_entry["id"]), MutableNodeMTG(Symbol(scene_entry["link"]), Symbol(scene_entry["symbol"]), + Int(scene_entry["index"]), 0), scene_attributes) + root_id = typemin(Int64) # the scene's key among the entities: not an entity's id + end + entity_of = Dict{Int,Int64}(mtg_id(id) => id for id in all_ids) # MTG.jl's id -> the entity's + entity_of[node_id(root)] = root_id + children_of = Dict{Int64,Vector{Int64}}() + for (id, (parent_id, _)) in tree_parent + push!(get!(children_of, parent_id, Int64[]), id) + end + isempty(roots) || (children_of[root_id] = vcat(roots, get(children_of, root_id, Int64[]))) + for list in values(children_of) # the `sibling_order` column, else by id + sort!(list; by = id -> (get(order_of, id, typemax(Int64)), id)) + end + stack = Any[root] + while !isempty(stack) + node = pop!(stack) + created = Any[] + for id in get(children_of, entity_of[node_id(node)], Int64[]) + child_encoding = encoding[id] + child_link = haskey(tree_parent, id) ? tree_parent[id][2] : :/ + push!(created, Node(mtg_id(id), node, MutableNodeMTG(child_link, Symbol(child_encoding.symbol), + child_encoding.index, child_encoding.scale), + attributes[id])) + end + append!(stack, reverse(created)) + end + + metadata = get(extras, "metadata", nothing) + if metadata !== nothing + metadata_values = Pair{Symbol,Any}[] + haskey(metadata, "symbols") && push!(metadata_values, :symbols => String.(metadata["symbols"])) + haskey(metadata, "scales") && push!(metadata_values, :scales => Int.(metadata["scales"])) + if haskey(metadata, "description") && metadata["description"] === nothing + push!(metadata_values, :description => nothing) + elseif haskey(metadata, "description") + description = metadata["description"] + column_names = Symbol.(collect(keys(description))) + columns = AbstractVector[column_values(value) for value in values(description)] + push!(metadata_values, :description => MultiScaleTreeGraph.ColumnTable(column_names, columns)) + end + append!(root, (; metadata_values...)) + elseif scene_entry !== nothing + symbols = unique([(encoding[id].symbol, encoding[id].scale) for id in sort(collect(keys(encoding)))]) + append!(root, (symbols = vcat([string(symbol(root))], [string(item[1]) for item in symbols]), + scales = vcat([0], [item[2] for item in symbols]))) + end + return root +end + +column_values(value::AbstractVector) = + all(item -> item isa AbstractVector, value) ? [String.(item) for item in value] : + all(item -> item isa AbstractString, value) ? String.(value) : collect(value) + +end diff --git a/bindings/julia/encoding3d/ext/encoding3dPlantGeomTurtleExt.jl b/bindings/julia/encoding3d/ext/encoding3dPlantGeomTurtleExt.jl new file mode 100644 index 0000000..1aba40d --- /dev/null +++ b/bindings/julia/encoding3d/ext/encoding3dPlantGeomTurtleExt.jl @@ -0,0 +1,68 @@ +""" +A VirtualPlantLab graph's geometry (PlantGeomTurtle: each node's `feed!` adds triangles to a turtle's mesh) as a mesh of +its data structure, each vertex carrying the row of the node that made it: the node itself when it is an entity, else +(a turtle's movement) its nearest kept ancestor. + +Status: a first proposal for VirtualPlantLab's team to take over, not a part of encoding3d: loaded only when the platform's +package is, encoding3d's requirements and CI do not depend on it; tested by hand (bindings/julia/README.md). +""" +module encoding3dPlantGeomTurtleExt + +using encoding3d +using encoding3d: NO_ID +import PlantGeomPrimitives as PGP +import PlantGeomTurtle as PGT +import PlantGraphs as PG + +static_of(graph::PG.Graph) = PG.static_graph(graph) +static_of(graph::PG.StaticGraph) = graph +graph_data(graph::PG.Graph) = PG.data(graph) +graph_data(::PG.StaticGraph) = nothing + +""" + add_geometry!(data_structure, graph; entity_set="Organ", turtle=Turtle(Float64), name="plant") -> Mesh + +Runs the turtle over the graph as `feed!(turtle, graph)` does (the same order, the same branching states), records +which node made each triangle, and adds the mesh to the data structure (written by `from_graph` from the same graph). +The positions are the turtle's (VPL: metres, z up). The triangles' colours and materials are not written (encoding3d has +no slot for them). +""" +function encoding3d.add_geometry!(data_structure::DataStructure, graph::Union{PG.Graph,PG.StaticGraph}; + entity_set::AbstractString = "Organ", turtle = PGT.Turtle(Float64), + name::AbstractString = "plant") + static = static_of(graph) + set = encoding3d.entity_set(data_structure, entity_set) + row_of = Dict(id => row - 1 for (row, id) in enumerate(set.columns["id"])) + mesh = PGP.Mesh(turtle) + vertex_rows = Int64[] + # As PlantGeomTurtle's feed!(turtle, graph): depth first, the turtle's state saved before each branch + stack = Tuple{PG.GraphNode,Int64}[(PG.getroot(static), NO_ID)] # node, its nearest entity's row + while !isempty(stack) + node, ancestor_row = pop!(stack) + before = length(PGP.vertices(mesh)) + PGT.feed!(turtle, PG.data(node), graph_data(graph)) + own = PG.self_id(node) >= 0 ? get(row_of, PG.self_id(node), nothing) : nothing # SET nodes: id -1 + row = own === nothing ? ancestor_row : own + append!(vertex_rows, fill(row, length(PGP.vertices(mesh)) - before)) + children = collect(PG.children(node, static)) + if length(children) == 1 + push!(stack, (only(children), row)) + elseif length(children) > 1 + for child in children + push!(stack, (PG.GraphNode(PGT.SET(to = PGT.pos(turtle), head = PGT.head(turtle), + up = PGT.up(turtle), arm = PGT.arm(turtle))), row)) + push!(stack, (child, row)) + end + end + end + vertices = PGP.vertices(mesh) + positions = Matrix{Float64}(undef, length(vertices), 3) + for (index, vertex) in enumerate(vertices) + positions[index, :] .= (vertex[1], vertex[2], vertex[3]) + end + added = Mesh(name; positions = positions, feature_rows = Dict(entity_set => vertex_rows)) + push!(data_structure.meshes, added) + return added +end + +end diff --git a/bindings/julia/encoding3d/ext/encoding3dPlantGraphsExt.jl b/bindings/julia/encoding3d/ext/encoding3dPlantGraphsExt.jl new file mode 100644 index 0000000..18d38d1 --- /dev/null +++ b/bindings/julia/encoding3d/ext/encoding3dPlantGraphsExt.jl @@ -0,0 +1,195 @@ +""" +A VirtualPlantLab graph (PlantGraphs.jl: a `Graph` or a `StaticGraph`) as an encoding3d `graph` data structure, and back: + +| encoding3d | PlantGraphs | +|---|---| +| one entity set (default `Organ`, rank 5) | the graph's nodes, but the turtle's movements (PlantGeomTurtle: `T`, `RU`, `F`, `SET`, …) | +| `id`, `label` | the node's id, the name of its type (`Internode`, `Leaf`, …) | +| `parent` | its nearest kept ancestor | +| `edge_type` | `unknown`: PlantGraphs stores no edge types | +| `sibling_order` | the node's position among its kept parent's kept children | +| variables | the fields of each node's type: missing where a type has no such field; dates as ISO 8601 text with their `format` | +| scalars | the graph's data (`data(graph)`), its fields | + +PlantGraphs has one scale. Each type's field names, to rebuild nodes with `to_graph`, are kept in +`extras["virtualplantlab"]` (with `"model": "PlantGraphs"`). + +Status: a first proposal for VirtualPlantLab's team to take over, not a part of encoding3d: loaded only when the platform's +package is, encoding3d's requirements and CI do not depend on it; tested by hand (bindings/julia/README.md). +""" +module encoding3dPlantGraphsExt + +using encoding3d +using encoding3d: Columns, Declarations, json_object, check_identifier, column_of_values +using OrderedCollections: OrderedDict +import PlantGraphs as PG + +const PLATFORM = "VirtualPlantLab" +const EXTRAS_KEY = "virtualplantlab" # the extras key: VirtualPlantLab's name in the vocabulary +const IDENTIFIER = r"^[a-zA-Z_][a-zA-Z0-9_]*$" + +"""Whether a node is kept: the turtle's movements and state nodes (PlantGeomTurtle's types) are not.""" +default_keep(node_data) = nameof(parentmodule(typeof(node_data))) != :PlantGeomTurtle + +label_of(node_data) = string(nameof(typeof(node_data))) + +field_values(object) = object === nothing ? Pair{Symbol,Any}[] : + [name => getfield(object, name) for name in fieldnames(typeof(object))] + +"""Columns of the objects' fields (missing where an object has none), by name in order of first appearance, and their +declarations (a date's `format` added).""" +function field_columns(objects, set_name, variables) + names = Symbol[] + values_of = [Dict{Symbol,Any}(field_values(object)) for object in objects] + for object in objects, (name, _) in field_values(object) + name in names || push!(names, name) + end + columns, declarations = Columns(), Declarations() + for name in names + column_name = string(name) + if !occursin(IDENTIFIER, column_name) || column_name in encoding3d.ENTITY_SET_RESERVED + @warn "field '$name' of '$set_name' not written: not an identifier, or a reserved column name" + continue + end + converted = column_of_values(Any[get(values, name, nothing) for values in values_of]) + if converted === nothing + @warn "field '$name' of '$set_name' not written: encoding3d holds numbers, booleans, text, dates and " * + "fixed-size numeric vectors" + continue + end + column, format = converted + columns[column_name] = column + declared = get(variables, column_name, nothing) + if format !== nothing # dates and times: their format declared + declared = declared === nothing ? Variable(unit = "1", kind = "descriptor") : deepcopy(declared) + declared.format = format + end + declared === nothing || (declarations[column_name] = declared) + end + return columns, declarations +end + +static_of(graph::PG.Graph) = PG.static_graph(graph) +static_of(graph::PG.StaticGraph) = graph +graph_data(graph::PG.Graph) = PG.data(graph) +graph_data(::PG.StaticGraph) = nothing + +"""The graph's kept nodes in depth-first order (children in their order), each with its nearest kept ancestor's id.""" +function kept_nodes(static, keep) + rows = Tuple{PG.GraphNode,Int}[] # node, its kept parent's id (NO_ID: none) + stack = Tuple{PG.GraphNode,Int}[(PG.getroot(static), NO_ID)] + while !isempty(stack) + node, kept_parent = pop!(stack) + parent_of_children = kept_parent + if keep(PG.data(node)) + push!(rows, (node, kept_parent)) + parent_of_children = PG.self_id(node) + end + for child in reverse(collect(PG.children(node, static))) + push!(stack, (child, parent_of_children)) + end + end + return rows +end + +""" + from_graph(graph; id="plant", entity_set="Organ", rank=5, scale=nothing, variables=Dict(), labels=Dict(), + keep=default_keep, platform="VirtualPlantLab", model=nothing) -> DataStructure + +The PlantGraphs graph as a `graph` data structure of one entity set. `scale`: its base scale (e.g. `"Organ"`), or +`"mixed"`. `variables`: the fields' declarations (name => `Variable`). `labels`: the node types' names => the +vocabulary's labels (e.g. `Dict("Internode" => "StemInternode")`). `keep`: which nodes (by their data) are entities; +by default all but the turtle's movements. +""" +function encoding3d.from_graph(graph::Union{PG.Graph,PG.StaticGraph}; id::AbstractString = "plant", + entity_set::AbstractString = "Organ", rank = 5, scale = nothing, + variables = Dict{String,Variable}(), labels = Dict{String,String}(), keep = default_keep, + platform = PLATFORM, model = nothing) + check_identifier(entity_set, "entity set name") + variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) + static = static_of(graph) + rows = kept_nodes(static, keep) + ids = Int64[PG.self_id(node) for (node, _) in rows] + parents = Int64[parent for (_, parent) in rows] + siblings_seen = Dict{Int64,Int64}() # each parent's kept children so far + order = Int64[parent == NO_ID ? 0 : (siblings_seen[parent] = get(siblings_seen, parent, -1) + 1) + for parent in parents] + edge_types = String[parent == NO_ID ? "none" : "unknown" for parent in parents] # PlantGraphs stores none + objects = [PG.data(node) for (node, _) in rows] + columns = Columns("id" => ids, "label" => String[label_of(object) for object in objects], "parent" => parents, + "edge_type" => edge_types, "sibling_order" => order) + fields, declarations = field_columns(objects, entity_set, variables) + merge!(columns, fields) + extras = json_object("model" => "PlantGraphs", + "types" => json_object(label_of(object) => string.(fieldnames(typeof(object))) + for object in unique(object -> typeof(object), objects))) + + scalars = nothing + data = graph_data(graph) + if data !== nothing && !isempty(fieldnames(typeof(data))) + scalar_columns, scalar_declarations = field_columns([data], "scalars", variables) + if !isempty(scalar_columns) + scalars = Table("scalars"; columns = scalar_columns, variables = scalar_declarations) + extras["data_type"] = string(nameof(typeof(data))) + end + end + set = EntitySet(entity_set; columns = columns, variables = declarations, rank = rank, scale = scale) + return DataStructure(id; kind = "graph", entity_sets = [set], scalars = scalars, platform = platform, + model = model, extras = json_object(EXTRAS_KEY => extras), labels = labels) +end + +# ── Data structure → graph ────────────────────────────────────────────── + + +""" + to_graph(data_structure; types, entity_set=nothing) -> (StaticGraph, ids) + +The data structure's entities as a PlantGraphs `StaticGraph`, each built as `types[label](fields...)` from its row, +its type's fields in their order (`types`: label => node type, e.g. `Dict("Internode" => Internode)`). `entity_set`: +the set read (by default the finest one with parents). `ids`: the entity ids => their ids in the graph (PlantGraphs +gives new ids). Wrap the graph as VPL does: `Graph(axiom = static_graph, rules = ..., data = ...)`. +""" +function encoding3d.to_graph(data_structure::DataStructure; types::AbstractDict, entity_set = nothing) + candidates = [set for set in data_structure.entity_sets if haskey(set.columns, "parent")] + set = entity_set === nothing ? + (isempty(candidates) ? throw(ArgumentError("'$(data_structure.id)': no entity set with parents")) : + last(sort(candidates; by = set -> something(set.rank, 0)))) : + encoding3d.entity_set(data_structure, entity_set) + labels = set.columns["label"] + objects = Vector{Any}(undef, row_count(set)) + for row in 1:row_count(set) + haskey(types, labels[row]) || throw(ArgumentError("no node type given for label '$(labels[row])'")) + node_type = types[labels[row]] + arguments = map(fieldnames(node_type)) do field + name = string(field) + format = haskey(set.variables, name) ? set.variables[name].format : nothing + value = haskey(set.columns, name) ? encoding3d.value_of(set.columns[name], row, format) : missing + ismissing(value) && throw(ArgumentError("'$(labels[row])' row $row: no value for field '$name'")) + value + end + objects[row] = node_type(arguments...) + end + ids, parents = set.columns["id"], set.columns["parent"] + children_of = Dict{Int64,Vector{Int}}() + roots = Int[] + for row in 1:row_count(set) + parents[row] == NO_ID ? push!(roots, row) : push!(get!(children_of, parents[row], Int[]), row) + end + if haskey(set.columns, "sibling_order") # children in their order, else in their rows' + foreach(list -> sort!(list; by = row -> set.columns["sibling_order"][row]), values(children_of)) + end + length(roots) == 1 || throw(ArgumentError("'$(set.name)': $(length(roots)) roots, a PlantGraphs graph has one")) + static = PG.StaticGraph(PG.GraphNode(objects[only(roots)])) + graph_ids = Dict{Int64,Int}(ids[only(roots)] => PG.self_id(PG.getroot(static))) + stack = [only(roots)] + while !isempty(stack) + row = pop!(stack) + for child in get(children_of, ids[row], Int[]) + graph_ids[ids[child]] = append!(static, graph_ids[ids[row]], PG.GraphNode(objects[child])) + push!(stack, child) + end + end + return static, graph_ids +end + +end diff --git a/bindings/julia/encoding3d/ext/encoding3dZMQExt.jl b/bindings/julia/encoding3d/ext/encoding3dZMQExt.jl new file mode 100644 index 0000000..1ac3496 --- /dev/null +++ b/bindings/julia/encoding3d/ext/encoding3dZMQExt.jl @@ -0,0 +1,36 @@ +# The coupling's ZeroMQ transport: a participant's DEALER socket, named after it, connected to the coordinator's +# ROUTER socket; a message is its JSON header and, for turns and updates, a partial frame's GLB bytes. +module encoding3dZMQExt + +using encoding3d +using JSON +using ZMQ + +mutable struct ZmqChannel <: encoding3d.CouplingChannel + socket::ZMQ.Socket +end + +function encoding3d.zmq_channel(endpoint::String, participant::String) + socket = ZMQ.Socket(ZMQ.DEALER) + socket.routing_id = participant + socket.linger = 1000 + ZMQ.connect(socket, endpoint) + return ZmqChannel(socket) +end + +function encoding3d.send_message(channel::ZmqChannel, header::AbstractDict, payload = nothing) + parts = payload === nothing ? [JSON.json(header)] : [Vector{UInt8}(codeunits(JSON.json(header))), payload] + ZMQ.send_multipart(channel.socket, parts) + return nothing +end + +function encoding3d.receive_message(channel::ZmqChannel, timeout = nothing) + channel.socket.rcvtimeo = timeout === nothing ? -1 : round(Int, 1000 * timeout) + parts = ZMQ.recv_multipart(channel.socket, Vector{UInt8}) + header = JSON.parse(String(parts[1])) + return header, length(parts) > 1 ? parts[2] : nothing +end + +encoding3d.close_channel(channel::ZmqChannel) = close(channel.socket) + +end diff --git a/bindings/julia/encoding3d/src/codec.jl b/bindings/julia/encoding3d/src/codec.jl new file mode 100644 index 0000000..00432f3 --- /dev/null +++ b/bindings/julia/encoding3d/src/codec.jl @@ -0,0 +1,568 @@ +# Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), +# meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data +# structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, +# standard names, kinds and roles. + +const EXTENSION = "FSPM_scene" +const METADATA = "EXT_structural_metadata" +const FEATURES = "EXT_mesh_features" +const EDGE_TYPE_ENUM = "FSPM_edge_type" +const IDENTIFIER = r"^[a-zA-Z_][a-zA-Z0-9_]*$" +const ARRAY_BUFFER, ELEMENT_ARRAY_BUFFER = 34962, 34963 +const FLOAT, UNSIGNED_INT, UNSIGNED_SHORT, UNSIGNED_BYTE = 5126, 5125, 5123, 5121 +const MAX_FLOAT_FEATURE = 2^24 # feature rows are stored as Float32 vertex attributes +# The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about +# x, column-major). Readers of the tables use the scene's frame, not this node. +const Z_UP_TO_Y_UP = [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0] + +const COMPONENT_TYPES = Dict{DataType,String}(T => uppercase(string(T)) for T in COLUMN_TYPES) +const TYPES_OF_COMPONENTS = Dict{String,DataType}(name => T for (T, name) in COMPONENT_TYPES) +# EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; +Inf +# and -Inf as the largest finite value and the next one after the most negative, declared in the property's extras +# (FSPM_infinity). +no_data_float(::Type{T}) where {T<:AbstractFloat} = -floatmax(T) +infinity_stand_ins(::Type{T}) where {T<:AbstractFloat} = (positive = floatmax(T), negative = nextfloat(-floatmax(T))) + +json_object(pairs...) = OrderedDict{String,Any}(pairs...) + +# ── Writing ────────────────────────────────────────────────────────────── + +""" + to_bytes(frame) -> Vector{UInt8} + +The GLB file of `frame`. +""" +function to_bytes(frame::Frame) + buffers = BufferBuilder() + schema = json_object("id" => "fspm_scene", "classes" => json_object(), "enums" => json_object( + EDGE_TYPE_ENUM => json_object("valueType" => "UINT8", + "values" => [json_object("name" => name, "value" => value - 1) + for (value, name) in enumerate(EDGE_TYPES)]))) + tables, variables = Any[], json_object() + document = json_object("asset" => json_object("version" => "2.0", "generator" => "encoding3d.jl $SPECIFICATION_VERSION"), + "scene" => 0, "scenes" => [json_object("nodes" => [0])], + "nodes" => Any[json_object("name" => "fspm_scene", "matrix" => Z_UP_TO_Y_UP, + "children" => Int[])]) + accessors, meshes = Any[], Any[] + + function add_table(table::AbstractTable, class_id::AbstractString) + check_identifier(class_id, "class") + properties, stored = encode_table(table, class_id, buffers, schema) + schema["classes"][class_id] = json_object("name" => table.name, "properties" => properties) + push!(tables, json_object("name" => table.name, "class" => class_id, "count" => row_count(table), + "properties" => stored)) + declared = json_object(name => declaration_json(table.variables[name]) + for name in variable_names(table) if haskey(table.variables, name)) + isempty(declared) || (variables[class_id] = declared) + return length(tables) - 1 + end + + data_structures = Any[] + for data_structure in frame.data_structures + check_identifier(data_structure.id, "data structure id") + entry = json_object("id" => data_structure.id, "kind" => data_structure.kind) + data_structure.platform === nothing || (entry["platform"] = data_structure.platform) + data_structure.model === nothing || (entry["model"] = data_structure.model) + entry["origin"] = vec(data_structure.origin) # column-major, as glTF matrices + entry["entitySets"] = map(data_structure.entity_sets) do set + set_entry = json_object("name" => set.name, "rank" => set.rank) + set.scale === nothing || (set_entry["scale"] = set.scale) + set_entry["propertyTable"] = add_table(set, "$(data_structure.id)__$(set.name)") + set_entry + end + if !isempty(data_structure.relations) + entry["relations"] = [json_object("name" => relation.name, "sourceSet" => relation.source_set, + "targetSet" => relation.target_set, + "propertyTable" => add_table(relation, "$(data_structure.id)__$(relation.name)")) + for relation in data_structure.relations] + end + data_structure.scalars === nothing || + (entry["scalars"] = add_table(data_structure.scalars, "$(data_structure.id)__scalars")) + data_structure.grid === nothing || (entry["grid"] = grid_json(data_structure.grid)) + isempty(data_structure.labels) || (entry["labels"] = data_structure.labels) + isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) + node = json_object("name" => data_structure.id) + data_structure.origin == identity_matrix() || (node["matrix"] = entry["origin"]) + push!(document["nodes"], node) + push!(document["nodes"][1]["children"], length(document["nodes"]) - 1) + entry["node"] = length(document["nodes"]) - 1 + if !isempty(data_structure.meshes) + node["children"], entry["meshes"] = Int[], Any[] + for mesh in data_structure.meshes + push!(meshes, encode_mesh(mesh, data_structure, entry, buffers, accessors)) + push!(document["nodes"], json_object("name" => mesh.name, "mesh" => length(meshes) - 1)) + push!(node["children"], length(document["nodes"]) - 1) + mesh_entry = json_object("name" => mesh.name, "mesh" => length(meshes) - 1) + if !isempty(mesh.display) + mesh_entry["display"] = [json_object("attribute" => display_attribute(name), "variable" => name, + "entitySet" => get(mesh.display_sources, name, nothing)) + for name in keys(mesh.display)] + end + push!(entry["meshes"], mesh_entry) + end + end + push!(data_structures, entry) + end + + mappings = Any[] + for mapping in frame.mappings + check_identifier(mapping.name, "mapping name") + entry = json_object("name" => mapping.name, + "source" => json_object("dataStructure" => mapping.source[1], "entitySet" => mapping.source[2]), + "target" => json_object("dataStructure" => mapping.target[1], "entitySet" => mapping.target[2])) + mapping.method === nothing || (entry["method"] = mapping.method) + entry["propertyTable"] = add_table(mapping, "mapping__$(mapping.name)") + push!(mappings, entry) + end + + update = nothing + if frame.update !== nothing + check_identifier(frame.update.participant, "participant") + update = json_object("participant" => frame.update.participant) + if !isempty(frame.update.removed) + update["removed"] = Any[json_object("dataStructure" => key[1], "entitySet" => key[2], + "propertyTable" => add_table(Table("$(key[2])__removed"; + columns = Columns("id" => ids)), "$(key[1])__$(key[2])__removed")) + for (key, ids) in frame.update.removed] + end + end + + scene = json_object("version" => SPECIFICATION_VERSION, + "frame" => json_object("time" => Float64(frame.time), "timeUnit" => frame.time_unit, + "step" => frame.step)) + frame.vocabulary_version === nothing || (scene["vocabulary"] = frame.vocabulary_version) + isempty(frame.writer) || (scene["writer"] = frame.writer) + update === nothing || (scene["update"] = update) + scene["dataStructures"] = data_structures + isempty(mappings) || (scene["mappings"] = mappings) + isempty(variables) || (scene["variables"] = variables) + + other_extensions = get(frame.unknown, "extensions", json_object()) + extensions = json_object(METADATA => json_object("schema" => schema, "propertyTables" => tables), + EXTENSION => scene) + merge!(extensions, other_extensions) + document["extensionsUsed"] = vcat([METADATA, EXTENSION], isempty(meshes) ? String[] : [FEATURES], + collect(String, keys(other_extensions))) + document["extensions"] = extensions + haskey(frame.unknown, "extras") && (document["extras"] = frame.unknown["extras"]) + if !isempty(meshes) + document["meshes"], document["accessors"] = meshes, accessors + end + if !isempty(buffers.views) + # the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + append!(buffers.data, zeros(UInt8, padding(length(buffers.data)))) + document["buffers"] = [json_object("byteLength" => length(buffers.data))] + document["bufferViews"] = buffers.views + end + return pack(document, buffers.data) +end + +""" + write_frame(path, frame) + +Write `frame` to the GLB file `path`. +""" +write_frame(path::AbstractString, frame::Frame) = (write(path, to_bytes(frame)); path) + +check_identifier(name, what) = (name isa AbstractString && occursin(IDENTIFIER, name)) || + throw(ArgumentError("$what '$name' must be an identifier (letters, digits, '_'; not starting with a digit)")) + +function declaration_json(variable::Variable) + declaration = json_object("unit" => variable.unit, "role" => variable.role) + variable.standard_name === nothing || (declaration["standardName"] = variable.standard_name) + variable.kind === nothing || (declaration["kind"] = variable.kind) + variable.minimum === nothing || (declaration["minimum"] = variable.minimum) + variable.maximum === nothing || (declaration["maximum"] = variable.maximum) + isempty(variable.references) || (declaration["references"] = variable.references) + isempty(variable.doi) || (declaration["doi"] = variable.doi) + variable.format === nothing || (declaration["format"] = variable.format) + return declaration +end + +"""Row-major values of a column: a vector column's rows one after the other.""" +row_major(column::AbstractVector) = column +row_major(column::AbstractMatrix) = vec(permutedims(column)) + +function encode_table(table::AbstractTable, class_id::AbstractString, buffers::BufferBuilder, schema) + properties, stored = json_object(), json_object() + reserved_names = reserved(table) + for (name, column) in table.columns + check_identifier(name, "column of '$(table.name)'") + description = declaration(table, name).description + property = json_object() + description === nothing || (property["description"] = description) + element = nonmissingtype(eltype(column)) + has_missing = Missing <: eltype(column) && any(ismissing, column) + if name in ENUM_COLUMNS && name in reserved_names + enum_name, codes = enum_codes(name, column, class_id, schema) + property["type"], property["enumType"] = "ENUM", enum_name + stored[name] = json_object("values" => add!(buffers, codes)) + elseif element <: AbstractString + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': text vectors are not supported")) + property["type"] = "STRING" + if has_missing # missing text: noData, a string the column lacks + no_data = free_string(column) + property["noData"] = no_data + column = String[ismissing(value) ? no_data : value for value in column] + end + encoded = [Vector{UInt8}(codeunits(value)) for value in column] + offsets = vcat(0, cumsum(Int64[length(value) for value in encoded])) + offset_type = offsets[end] < 2^32 ? UInt32 : UInt64 + stored[name] = json_object("values" => add!(buffers, reduce(vcat, encoded; init = UInt8[])), + "stringOffsets" => add!(buffers, offset_type.(offsets)), + "stringOffsetType" => uppercase(string(offset_type))) + elseif element === Bool && has_missing # booleans with missing values: UINT8 0, 1, 255 missing + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': boolean vectors are not supported")) + property["type"], property["componentType"] = "SCALAR", "UINT8" + property["noData"] = BOOLEAN_NO_DATA + property["extras"] = json_object("FSPM_boolean" => true) + stored[name] = json_object("values" => add!(buffers, UInt8[ismissing(value) ? BOOLEAN_NO_DATA : UInt8(value) + for value in column])) + elseif element === Bool + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': boolean vectors are not supported")) + property["type"] = "BOOLEAN" + packed = zeros(UInt8, cld(length(column), 8)) + for (row, value) in enumerate(column) + value && (packed[(row-1)÷8+1] |= UInt8(1) << ((row - 1) % 8)) # bit order: little + end + stored[name] = json_object("values" => add!(buffers, packed)) + else + haskey(COMPONENT_TYPES, element) || + throw(ArgumentError("column '$name' of '$(table.name)': type $element is not supported")) + property["type"], property["componentType"] = "SCALAR", COMPONENT_TYPES[element] + if column isa AbstractMatrix + size(column, 2) >= 2 || + throw(ArgumentError("column '$name' of '$(table.name)': vectors have at least 2 components")) + property["array"], property["count"] = true, size(column, 2) + end + data = column + if element <: AbstractFloat + if any(isinf, column) + stand_ins = infinity_stand_ins(element) + any(value -> value == stand_ins.positive || value == stand_ins.negative, column) && + throw(ArgumentError("column '$name' of '$(table.name)' holds both infinities and the values " * + "standing for them ($stand_ins)")) + property["extras"] = json_object("FSPM_infinity" => json_object( + "positive" => Float64(stand_ins.positive), "negative" => Float64(stand_ins.negative))) + data = map(value -> value == Inf ? stand_ins.positive : + value == -Inf ? stand_ins.negative : value, data) + end + if any(isnan, data) + no_data = no_data_float(element) + any(==(no_data), data) && + throw(ArgumentError("column '$name' of '$(table.name)' holds both NaN and $no_data, the " * + "value standing for NaN")) + property["noData"] = Float64(no_data) + data = map(value -> isnan(value) ? no_data : value, data) + end + elseif name in ("complex", "parent") && name in reserved_names + property["noData"] = NO_ID + elseif has_missing # missing integers: noData, a value the column lacks + no_data = free_integer(element, column) + property["noData"] = no_data + data = element[ismissing(value) ? no_data : value for value in column] + end + stored[name] = json_object("values" => add!(buffers, row_major(data))) + end + name == "id" && table isa EntitySet && (property["required"] = true) + properties[name] = property + end + return properties, stored +end + +const BOOLEAN_NO_DATA = 0xff # a boolean column with missing values: UINT8 0, 1, and 255 missing + +"""A value of type `T` the column does not hold: its minimum (signed) or maximum (unsigned) first, then inwards.""" +function free_integer(::Type{T}, column) where {T<:Integer} + present = Set(value for value in column if !ismissing(value)) + first, second = T <: Signed ? (typemin(T), typemax(T)) : (typemax(T), typemin(T)) + for candidate in (first, second) + candidate in present || return candidate + end + candidate = first + while candidate in present # at most length(present) steps + candidate = T <: Signed ? candidate + one(T) : candidate - one(T) + end + return candidate +end + +"""A string the column does not hold: the empty string first, then NUL characters.""" +function free_string(column) + present = Set(value for value in column if !ismissing(value)) + candidate = "" + while candidate in present + candidate *= "\0" + end + return candidate +end + +function enum_codes(name, column, class_id, schema) + if name == "edge_type" + unknown = setdiff(Set(column), Set(EDGE_TYPES)) + isempty(unknown) || + throw(ArgumentError("edge_type of '$class_id': unknown values $(sort(collect(unknown))) " * + "(allowed: $EDGE_TYPES)")) + return EDGE_TYPE_ENUM, UInt8[findfirst(==(value), EDGE_TYPES) - 1 for value in column] + end + names = sort(unique(column)) + enum_name = "$(class_id)__$name" + value_type = length(names) <= 256 ? UInt8 : UInt16 + schema["enums"][enum_name] = json_object("valueType" => uppercase(string(value_type)), + "values" => [json_object("name" => label, "value" => code - 1) + for (code, label) in enumerate(names)]) + code_of = Dict(label => code - 1 for (code, label) in enumerate(names)) + return enum_name, value_type[code_of[value] for value in column] +end + +function grid_json(grid::Grid) + entry = json_object("kind" => grid.kind, "shape" => grid.shape, "origin" => grid.origin, "spacing" => grid.spacing, + "axes" => grid.axes, "periodic" => grid.periodic, "zUp" => grid.z_up, "cells" => grid.cells) + grid.kind == "adaptive" && (entry["maxLevel"] = grid.max_level) + return entry +end + +display_attribute(name::AbstractString) = "_" * uppercase(name) + +function encode_mesh(mesh::Mesh, data_structure::DataStructure, entry, buffers::BufferBuilder, accessors) + function accessor(values, accessor_type, component_type, target; bounds = false) + buffer_view = add!(buffers, row_major(values); target = target) + item = json_object("bufferView" => buffer_view, "componentType" => component_type, + "count" => size(values, 1), "type" => accessor_type) + if bounds + item["min"] = Float64.(vec(minimum(values; dims = 1))) + item["max"] = Float64.(vec(maximum(values; dims = 1))) + end + push!(accessors, item) + return length(accessors) - 1 + end + + positions = Float32.(mesh.positions) + attributes = json_object("POSITION" => accessor(positions, "VEC3", FLOAT, ARRAY_BUFFER; bounds = true)) + tables = Dict(set_entry["name"] => set_entry["propertyTable"] for set_entry in entry["entitySets"]) + feature_ids = Any[] + for (set_index, (set_name, rows)) in enumerate(mesh.feature_rows) + haskey(tables, set_name) || + throw(ArgumentError("mesh '$(mesh.name)': no entity set '$set_name' in '$(data_structure.id)'")) + count = row_count(entity_set(data_structure, set_name)) + count >= MAX_FLOAT_FEATURE && + throw(ArgumentError("mesh '$(mesh.name)': $count entities in '$set_name', feature IDs stop at 2^24")) + any(row -> row < NO_ID || row >= count, rows) && + throw(ArgumentError("mesh '$(mesh.name)': feature rows of '$set_name' outside its $count rows")) + stored = Float32[row == NO_ID ? count : row for row in rows] # null: one past the last row + attributes["_FEATURE_ID_$(set_index - 1)"] = accessor(stored, "SCALAR", FLOAT, ARRAY_BUFFER) + feature = json_object("featureCount" => length(unique(filter(!=(NO_ID), rows))), "attribute" => set_index - 1, + "propertyTable" => tables[set_name], "label" => set_name) + any(==(NO_ID), rows) && (feature["nullFeatureId"] = count) + push!(feature_ids, feature) + end + for (name, values) in mesh.display + length(values) == size(positions, 1) || + throw(ArgumentError("mesh '$(mesh.name)': display copy '$name' has $(length(values)) values, " * + "$(size(positions, 1)) vertices")) + attributes[display_attribute(name)] = accessor(Float32.(values), "SCALAR", FLOAT, ARRAY_BUFFER) + end + primitive = json_object("attributes" => attributes, "mode" => 4) + mesh.indices === nothing || + (primitive["indices"] = accessor(UInt32.(mesh.indices), "SCALAR", UNSIGNED_INT, ELEMENT_ARRAY_BUFFER)) + isempty(feature_ids) || (primitive["extensions"] = json_object(FEATURES => json_object("featureIds" => feature_ids))) + return json_object("name" => mesh.name, "primitives" => [primitive]) +end + +# ── Reading ────────────────────────────────────────────────────────────── + +""" + from_bytes(data) -> Frame + +The frame of a GLB file's bytes. +""" +function from_bytes(data::AbstractVector{UInt8}) + document, binary = unpack(data) + extensions = get(document, "extensions", json_object()) + haskey(extensions, EXTENSION) || throw(ArgumentError("not an encoding3d file: no $EXTENSION extension")) + scene = extensions[EXTENSION] + metadata = get(extensions, METADATA, json_object("schema" => json_object(), "propertyTables" => Any[])) + variables = get(scene, "variables", json_object()) + + function table(index, make) + description = metadata["propertyTables"][index+1] + columns, declarations = decode_table(document, binary, description, metadata["schema"], variables) + return make(String(get(description, "name", description["class"])), columns, declarations) + end + + frame_entry = get(scene, "frame", json_object()) + frame = Frame(time = Float64(get(frame_entry, "time", 0.0)), time_unit = get(frame_entry, "timeUnit", "d"), + step = Int(get(frame_entry, "step", 0)), writer = get(scene, "writer", json_object()), + vocabulary_version = get(scene, "vocabulary", nothing)) + for entry in get(scene, "dataStructures", Any[]) + origin = haskey(entry, "origin") ? reshape(Float64.(entry["origin"]), 4, 4) : identity_matrix() + data_structure = DataStructure(entry["id"]; kind = get(entry, "kind", "graph"), + platform = get(entry, "platform", nothing), model = get(entry, "model", nothing), + extras = get(entry, "extras", json_object()), origin = origin, + labels = get(entry, "labels", json_object())) + for item in get(entry, "entitySets", Any[]) + rank, scale = get(item, "rank", nothing), get(item, "scale", nothing) + push!(data_structure.entity_sets, + table(item["propertyTable"], (name, columns, declarations) -> + EntitySet(name, columns, declarations, rank === nothing ? nothing : Int(rank), scale))) + end + for item in get(entry, "relations", Any[]) + push!(data_structure.relations, + table(item["propertyTable"], (name, columns, declarations) -> + Relation(name, columns, declarations, item["sourceSet"], item["targetSet"]))) + end + haskey(entry, "scalars") && (data_structure.scalars = table(entry["scalars"], Table)) + if haskey(entry, "grid") + grid = entry["grid"] + data_structure.grid = Grid(kind = grid["kind"], shape = Int.(grid["shape"]), + origin = Float64.(grid["origin"]), spacing = Float64.(grid["spacing"]), + axes = String.(grid["axes"]), periodic = Bool.(grid["periodic"]), + z_up = get(grid, "zUp", true), max_level = get(grid, "maxLevel", 0), + cells = grid["cells"]) + end + tables = Dict(item["propertyTable"] => item["name"] for item in get(entry, "entitySets", Any[])) + for mesh_entry in get(entry, "meshes", Any[]) + push!(data_structure.meshes, decode_mesh(document, binary, mesh_entry, tables)) + end + push!(frame.data_structures, data_structure) + end + for entry in get(scene, "mappings", Any[]) + source = (entry["source"]["dataStructure"], entry["source"]["entitySet"]) + target = (entry["target"]["dataStructure"], entry["target"]["entitySet"]) + push!(frame.mappings, table(entry["propertyTable"], (name, columns, declarations) -> + Mapping(name, columns, declarations, source, target, get(entry, "method", nothing)))) + end + if haskey(scene, "update") + entry = scene["update"] + frame.update = Update(entry["participant"]; removed = OrderedDict( + (item["dataStructure"], item["entitySet"]) => Vector{Int64}(table(item["propertyTable"], Table).columns["id"]) + for item in get(entry, "removed", Any[]))) + end + other_extensions = json_object(name => value for (name, value) in extensions + if !(name in (METADATA, EXTENSION, FEATURES))) + isempty(other_extensions) || (frame.unknown["extensions"] = other_extensions) + haskey(document, "extras") && (frame.unknown["extras"] = document["extras"]) + return frame +end + +""" + read_frame(path) -> Frame + +The frame of the GLB file `path`. +""" +read_frame(path::AbstractString) = from_bytes(read(path)) + +function decode_table(document, binary, description, schema, variables) + class_id = description["class"] + properties = schema["classes"][class_id]["properties"] + count = Int(description["count"]) + columns, declarations = Columns(), Declarations() + for (name, stored) in description["properties"] + property = properties[name] + raw = view_bytes(document, binary, stored["values"]) + type = property["type"] + if type == "ENUM" + enum = schema["enums"][property["enumType"]] + codes = read_values(TYPES_OF_COMPONENTS[get(enum, "valueType", "UINT16")], raw, count) + name_of = Dict(Int(item["value"]) => String(item["name"]) for item in enum["values"]) + column = String[name_of[Int(code)] for code in codes] + elseif type == "STRING" + offset_type = TYPES_OF_COMPONENTS[get(stored, "stringOffsetType", "UINT32")] + offsets = read_values(offset_type, view_bytes(document, binary, stored["stringOffsets"]), count + 1) + column = String[String(raw[offsets[row]+1:offsets[row+1]]) for row in 1:count] + if haskey(property, "noData") && any(==(property["noData"]), column) # missing text + column = Union{Missing,String}[value == property["noData"] ? missing : value for value in column] + end + elseif type == "BOOLEAN" + column = Bool[(raw[(row-1)÷8+1] >> ((row - 1) % 8)) & 0x01 == 0x01 for row in 1:count] + elseif type == "SCALAR" + element = TYPES_OF_COMPONENTS[property["componentType"]] + vector = get(property, "array", false) + vector && !haskey(property, "count") && + throw(ArgumentError("'$class_id.$name': variable-length arrays are not part of encoding3d 0.1")) + width = vector ? Int(property["count"]) : 1 + values = read_values(element, raw, count * width) + if element <: AbstractFloat + if haskey(property, "noData") + no_data = element(property["noData"]) + values = map(value -> value == no_data ? element(NaN) : value, values) + end + infinity = get(get(property, "extras", json_object()), "FSPM_infinity", nothing) + if infinity !== nothing + positive, negative = element(infinity["positive"]), element(infinity["negative"]) + values = map(value -> value == positive ? element(Inf) : + value == negative ? element(-Inf) : value, values) + end + end + column = vector ? permutedims(reshape(values, width, count)) : values + if get(get(property, "extras", json_object()), "FSPM_boolean", false) # booleans with missing values + no_data = get(property, "noData", BOOLEAN_NO_DATA) + column = any(==(no_data), column) ? + Union{Missing,Bool}[value == no_data ? missing : value == 1 for value in column] : + Bool[value == 1 for value in column] + elseif element <: Integer && !vector && haskey(property, "noData") && !(name in ("complex", "parent")) && + any(==(property["noData"]), column) # missing integers + column = Union{Missing,element}[value == property["noData"] ? missing : value for value in column] + end + else + throw(ArgumentError("'$class_id.$name': property type $type is not part of encoding3d 0.1")) + end + columns[name] = column + declared = get(get(variables, class_id, json_object()), name, nothing) + if declared !== nothing || haskey(property, "description") + declared = declared === nothing ? json_object() : declared + declarations[name] = Variable(unit = get(declared, "unit", "1"), + standard_name = get(declared, "standardName", nothing), + kind = get(declared, "kind", nothing), role = get(declared, "role", "state"), + description = get(property, "description", nothing), + minimum = get(declared, "minimum", nothing), + maximum = get(declared, "maximum", nothing), + references = String.(get(declared, "references", String[])), + doi = String.(get(declared, "doi", String[])), + format = get(declared, "format", nothing)) + end + end + return columns, declarations +end + +function accessor_values(document, binary, index) + accessor = document["accessors"][index+1] + width = Dict("SCALAR" => 1, "VEC2" => 2, "VEC3" => 3, "VEC4" => 4)[accessor["type"]] + element = Dict(FLOAT => Float32, UNSIGNED_INT => UInt32, UNSIGNED_SHORT => UInt16, + UNSIGNED_BYTE => UInt8)[accessor["componentType"]] + raw = view_bytes(document, binary, accessor["bufferView"]) + buffer_view = document["bufferViews"][accessor["bufferView"]+1] + item_size = sizeof(element) * width + stride = get(buffer_view, "byteStride", item_size) + start = get(accessor, "byteOffset", 0) + count = Int(accessor["count"]) + values = stride == item_size ? read_values(element, raw, count * width, start) : + reduce(vcat, [read_values(element, raw, width, start + item * stride) for item in 0:count-1]; + init = element[]) + return width > 1 ? permutedims(reshape(values, width, count)) : values +end + +function decode_mesh(document, binary, mesh_entry, tables) + primitive = document["meshes"][mesh_entry["mesh"]+1]["primitives"][1] + attributes = primitive["attributes"] + positions = Float64.(accessor_values(document, binary, attributes["POSITION"])) + feature_rows = OrderedDict{String,Vector{Int64}}() + for feature in get(get(get(primitive, "extensions", json_object()), FEATURES, json_object()), "featureIds", Any[]) + rows = Int64.(accessor_values(document, binary, attributes["_FEATURE_ID_$(feature["attribute"])"])) + if haskey(feature, "nullFeatureId") + null = feature["nullFeatureId"] + rows = Int64[row == null ? NO_ID : row for row in rows] + end + feature_rows[tables[feature["propertyTable"]]] = rows + end + display, sources = OrderedDict{String,Vector{Float32}}(), Dict{String,String}() + for item in get(mesh_entry, "display", Any[]) + display[item["variable"]] = accessor_values(document, binary, attributes[item["attribute"]]) + get(item, "entitySet", nothing) === nothing || (sources[item["variable"]] = item["entitySet"]) + end + indices = haskey(primitive, "indices") ? Int64.(accessor_values(document, binary, primitive["indices"])) : nothing + return Mesh(mesh_entry["name"]; positions = positions, feature_rows = feature_rows, indices = indices, + display = display, display_sources = sources) +end diff --git a/bindings/julia/encoding3d/src/coupling.jl b/bindings/julia/encoding3d/src/coupling.jl new file mode 100644 index 0000000..e422b9f --- /dev/null +++ b/bindings/julia/encoding3d/src/coupling.jl @@ -0,0 +1,360 @@ +# A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and +# requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is encoding3d's +# (Python: `encoding3d couple`). Transports: a folder ("files:path", here) and ZeroMQ ("tcp://…", "ipc://…": with ZMQ +# loaded, the package extension encoding3dZMQExt). + +"""The version of the coupling protocol this package speaks.""" +const PROTOCOL_VERSION = "0.3" +const ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT" +const PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT" + +""" + EntitySetSpec(data_structure, name; rank=nothing, rights="follows") + +An entity set a participant works on: its rank (a base scale's position by default) and its rights on its topology, +`"owns"` (adds and removes its entities), `"refines"` (adds and removes the entities it creates, under others'), +`"follows"` (reads). +""" +struct EntitySetSpec + data_structure::String + name::String + rank::Union{Nothing,Int} + rights::String +end + +function EntitySetSpec(data_structure::AbstractString, name::AbstractString; rank = nothing, rights = "follows") + rights in ("owns", "refines", "follows") || throw(ArgumentError("rights '$rights': owns, refines or follows")) + if rank === nothing && name in BASE_SCALES + rank = findfirst(==(name), BASE_SCALES) + end + return EntitySetSpec(String(data_structure), String(name), rank, String(rights)) +end + +""" + VariableSpec(data_structure, entity_set, name, direction; unit="1", standard_name=nothing, kind=nothing, + optional=false) + +A variable a participant provides (`direction` `"provides"`: an output) or requires (`"requires"`: an input). +""" +struct VariableSpec + data_structure::String + entity_set::String + name::String + direction::String + unit::String + standard_name::Union{Nothing,String} + kind::Union{Nothing,String} + optional::Bool +end + +function VariableSpec(data_structure::AbstractString, entity_set::AbstractString, name::AbstractString, + direction::AbstractString; unit = "1", standard_name = nothing, kind = nothing, optional = false) + direction in ("provides", "requires") || throw(ArgumentError("direction '$direction': provides or requires")) + return VariableSpec(String(data_structure), String(entity_set), String(name), String(direction), String(unit), + standard_name, kind, optional) +end + +""" + MappingSpec(name, source, target) + +A mapping a participant provides (its updates carry it, as a frame's `Mapping` of that name): `source` and `target`, +each (data structure, entity set), for links between two data structures. +""" +struct MappingSpec + name::String + source::Tuple{String,String} + target::Tuple{String,String} +end + +""" + Declaration(participant; platform=nothing, entity_sets=EntitySetSpec[], variables=VariableSpec[]) + +A participant's declaration (its name: `ENCODING3D_PARTICIPANT` when empty and launched by the coordinator). +""" +Base.@kwdef mutable struct Declaration + participant::String = "" + platform::Union{Nothing,String} = nothing + entity_sets::Vector{EntitySetSpec} = EntitySetSpec[] + variables::Vector{VariableSpec} = VariableSpec[] + mappings::Vector{MappingSpec} = MappingSpec[] +end + +Declaration(participant::AbstractString; kwargs...) = Declaration(; participant = String(participant), kwargs...) + +function declaration_json(declaration::Declaration) + entity_sets = [json_object("dataStructure" => item.data_structure, "name" => item.name, "rank" => item.rank, + "rights" => item.rights) for item in declaration.entity_sets] + for variable in declaration.variables # an entity set for each one its variables are on, followed + if !any(item -> (item.data_structure, item.name) == (variable.data_structure, variable.entity_set), + declaration.entity_sets) + spec = EntitySetSpec(variable.data_structure, variable.entity_set) + push!(declaration.entity_sets, spec) + push!(entity_sets, json_object("dataStructure" => spec.data_structure, "name" => spec.name, + "rank" => spec.rank, "rights" => spec.rights)) + end + end + variables = Any[] + for variable in declaration.variables + entry = json_object("dataStructure" => variable.data_structure, "entitySet" => variable.entity_set, + "name" => variable.name, "direction" => variable.direction, "unit" => variable.unit) + variable.standard_name === nothing || (entry["standardName"] = variable.standard_name) + variable.kind === nothing || (entry["kind"] = variable.kind) + variable.optional && (entry["optional"] = true) + push!(variables, entry) + end + value = json_object("participant" => declaration.participant, "entitySets" => entity_sets, + "variables" => variables) + declaration.platform === nothing || (value["platform"] = declaration.platform) + isempty(declaration.mappings) || (value["mappings"] = Any[ + json_object("name" => item.name, + "source" => json_object("dataStructure" => item.source[1], "entitySet" => item.source[2]), + "target" => json_object("dataStructure" => item.target[1], "entitySet" => item.target[2])) + for item in declaration.mappings]) + return value +end + +""" + declaration_from_json(value) -> Declaration + +A declaration from its JSON (e.g. a file's, `JSON.parsefile`). +""" +function declaration_from_json(value) + entity_sets = [EntitySetSpec(item["dataStructure"], item["name"]; rank = get(item, "rank", nothing), + rights = get(item, "rights", "follows")) for item in get(value, "entitySets", Any[])] + variables = [VariableSpec(item["dataStructure"], item["entitySet"], item["name"], item["direction"]; + unit = get(item, "unit", "1"), standard_name = get(item, "standardName", nothing), + kind = get(item, "kind", nothing), optional = get(item, "optional", false)) + for item in get(value, "variables", Any[])] + mappings = [MappingSpec(item["name"], (item["source"]["dataStructure"], item["source"]["entitySet"]), + (item["target"]["dataStructure"], item["target"]["entitySet"])) + for item in get(value, "mappings", Any[])] + return Declaration(; participant = String(value["participant"]), platform = get(value, "platform", nothing), + entity_sets, variables, mappings) +end + +""" + Link(receiver, variable, provider, source; entity_set=nothing, source_entity_set=nothing, factor=1.0, + aggregation=nothing, interpolation="hold") + +An input's provider, as a participant may propose it to the coordinator (see the Python `Link`). +""" +Base.@kwdef struct Link + receiver::String + variable::String + provider::String + source::String + entity_set::Union{Nothing,String} = nothing + source_entity_set::Union{Nothing,String} = nothing + factor::Float64 = 1.0 + aggregation::Union{Nothing,String} = nothing + interpolation::String = "hold" + mapping::Union{Nothing,String} = nothing +end + +Link(receiver, variable, provider, source; kwargs...) = + Link(; receiver = String(receiver), variable = String(variable), provider = String(provider), + source = String(source), kwargs...) + +function link_json(link::Link) + value = json_object("receiver" => link.receiver, "variable" => link.variable, "provider" => link.provider, + "source" => link.source) + link.entity_set === nothing || (value["entitySet"] = link.entity_set) + link.source_entity_set === nothing || (value["sourceEntitySet"] = link.source_entity_set) + link.aggregation === nothing || (value["aggregation"] = link.aggregation) + link.factor == 1.0 || (value["factor"] = link.factor) + link.interpolation == "hold" || (value["interpolation"] = link.interpolation) + link.mapping === nothing || (value["mapping"] = link.mapping) + return value +end + +# ── transports ─────────────────────────────────────────────────────────────────── +"""A participant's channel: `send_message(channel, header, payload)`, `receive_message(channel, timeout)`.""" +abstract type CouplingChannel end + +"""A participant's folder queues: `//to_coordinator/` and `to_participant/`.""" +mutable struct FolderChannel <: CouplingChannel + incoming::String + outgoing::String + next_read::Int + next_write::Int + keep::Bool # false: a message's files deleted once read + read_files::Vector{String} # those of the messages read and kept +end + +FolderChannel(incoming, outgoing, next_read, next_write) = + FolderChannel(incoming, outgoing, next_read, next_write, true, String[]) + +function FolderChannel(folder::AbstractString, participant::AbstractString) + incoming = joinpath(folder, participant, "to_participant") + outgoing = joinpath(folder, participant, "to_coordinator") + mkpath(incoming) + mkpath(outgoing) + return FolderChannel(incoming, outgoing, 0, 0) +end + +stem(folder, number) = joinpath(folder, lpad(number, 8, '0')) + +function send_message(channel::FolderChannel, header::AbstractDict, payload = nothing) + path = stem(channel.outgoing, channel.next_write) + header = copy(header) + if payload !== nothing + write(path * ".glb", payload) + header["payload"] = basename(path) * ".glb" + end + write(path * ".json.part", JSON.json(header)) + mv(path * ".json.part", path * ".json"; force = true) + channel.next_write += 1 + return nothing +end + +function receive_message(channel::FolderChannel, timeout = nothing) + path = stem(channel.incoming, channel.next_read) + deadline = timeout === nothing ? nothing : time() + timeout + while !isfile(path * ".json") + deadline !== nothing && time() > deadline && throw(ErrorException("no message in $timeout s")) + sleep(0.005) + end + header = JSON.parse(read(path * ".json", String)) + files = [path * ".json"] + payload = nothing + if haskey(header, "payload") + push!(files, joinpath(channel.incoming, pop!(header, "payload"))) + payload = read(files[end]) + end + channel.next_read += 1 + channel.keep ? append!(channel.read_files, files) : foreach(rm, files) + return header, payload +end + +close_channel(::CouplingChannel) = nothing + +"""As the session says (its welcome's `keepMessages`): keep the messages read, or delete them, those read so far too.""" +keep_messages!(::CouplingChannel, keep::Bool) = nothing +function keep_messages!(channel::FolderChannel, keep::Bool) + keep && return nothing + channel.keep = false + foreach(path -> isfile(path) && rm(path), channel.read_files) + empty!(channel.read_files) + return nothing +end + +""" + zmq_channel(endpoint, participant) -> CouplingChannel + +A participant's ZeroMQ channel (a DEALER socket named after it). Defined when ZMQ is loaded (`encoding3dZMQExt`). +""" +function zmq_channel end + +function participant_channel(endpoint::AbstractString, participant::AbstractString) + startswith(endpoint, "files:") && return FolderChannel(endpoint[length("files:")+1:end], participant) + hasmethod(zmq_channel, Tuple{String,String}) || + throw(ArgumentError("the endpoint $endpoint needs ZeroMQ: `using ZMQ` (or use a folder, \"files:path\")")) + return zmq_channel(String(endpoint), String(participant)) +end + +# ── the participant ────────────────────────────────────────────────────────────── +""" + Turn + +A turn: `step` (0: the initial state, phase `"init"`), the time it starts from and the time it reaches (`until`), and +`frame`, the participant's inputs (its entity sets, their entities, its inputs; in `frame.update.removed` the entities +removed since its last turn). +""" +struct Turn + step::Int + phase::String + time::Float64 + until::Float64 + frame::Frame +end + +""" + Participant + +A participant in a session (`Participant(declaration; …)` joins it). `id_offset`: the ids of the entities it creates are in +`[id_offset, id_offset + id_count)`. +""" +mutable struct Participant + channel::CouplingChannel + declaration::Declaration + id_offset::Int64 + id_count::Int64 + session::Any + couplings::Vector{Any} + step::Int + stopped::Union{Nothing,String} + timeout::Union{Nothing,Float64} +end + +""" + Participant(declaration; endpoint=ENV["ENCODING3D_ENDPOINT"], links=Link[], timeout=nothing) -> Participant + +Join the session at `endpoint`: send the declaration and the links it proposes, wait for the welcome. +""" +function Participant(declaration::Declaration; endpoint = get(ENV, ENDPOINT_VARIABLE, ""), links = Link[], + timeout = nothing) + isempty(endpoint) && throw(ArgumentError("no endpoint: give one, or set $ENDPOINT_VARIABLE")) + isempty(declaration.participant) && (declaration.participant = get(ENV, PARTICIPANT_VARIABLE, "")) + channel = participant_channel(endpoint, declaration.participant) + send_message(channel, json_object("type" => "hello", "protocol" => PROTOCOL_VERSION, + "declaration" => declaration_json(declaration), + "links" => Any[link_json(link) for link in links])) + header, _ = receive_message(channel, timeout) + header["type"] == "stop" && error("the coordinator refused $(declaration.participant): $(get(header, "reason", ""))") + header["type"] == "welcome" || error("expected a welcome, got $(header["type"])") + keep_messages!(channel, get(header["session"], "keepMessages", true)) + return Participant(channel, declaration, Int64(header["idOffset"]), Int64(header["idCount"]), header["session"], + collect(get(header, "couplings", Any[])), -1, nothing, timeout === nothing ? nothing : + Float64(timeout)) +end + +""" + next_turn(participant) -> Union{Turn,Nothing} + +The next turn, or `nothing` when the coordinator stopped the session (`participant.stopped`: its reason). +""" +function next_turn(participant::Participant) + header, payload = receive_message(participant.channel, participant.timeout) + if header["type"] == "stop" + participant.stopped = String(get(header, "reason", "")) + close_channel(participant.channel) + return nothing + end + header["type"] == "turn" || error("expected a turn, got $(header["type"])") + participant.step = Int(header["step"]) + return Turn(Int(header["step"]), String(header["phase"]), Float64(header["time"]), Float64(header["until"]), + from_bytes(payload)) +end + +"""Iterate a participant's turns: `for turn in turns(participant) … end`.""" +struct Turns + participant::Participant +end +turns(participant::Participant) = Turns(participant) +Base.IteratorSize(::Type{Turns}) = Base.SizeUnknown() +Base.eltype(::Type{Turns}) = Turn +function Base.iterate(turns::Turns, state = nothing) + turn = next_turn(turns.participant) + return turn === nothing ? nothing : (turn, nothing) +end + +""" + send_update(participant, frame) + +Send the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the +entities it adds or changes); in `frame.update.removed`, the entities it removed. +""" +function send_update(participant::Participant, frame::Frame) + frame.update === nothing && (frame.update = Update(participant.declaration.participant)) + frame.update.participant = participant.declaration.participant + send_message(participant.channel, json_object("type" => "update", "step" => participant.step), to_bytes(frame)) + return nothing +end + +""" + fail(participant, message) + +Tell the coordinator this participant cannot go on (the session stops). +""" +fail(participant::Participant, message::AbstractString) = + send_message(participant.channel, json_object("type" => "error", "message" => String(message))) diff --git a/bindings/julia/encoding3d/src/dump.jl b/bindings/julia/encoding3d/src/dump.jl new file mode 100644 index 0000000..834a159 --- /dev/null +++ b/bindings/julia/encoding3d/src/dump.jl @@ -0,0 +1,108 @@ +# A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): +# every table's columns as lists, NaN as null, ±Inf as "Infinity" / "-Infinity", positions as their Float32 values. + +json_value(value::AbstractFloat) = + isnan(value) ? nothing : isinf(value) ? (value > 0 ? "Infinity" : "-Infinity") : Float64(value) +json_value(::Missing) = nothing +json_value(value) = value + +json_column(column::AbstractVector) = [json_value(value) for value in column] +json_column(column::AbstractMatrix) = [[json_value(value) for value in row] for row in eachrow(column)] + +"""The name of a column's type in the expected values: numpy's (`float64`, `int64`, `uint8`, `bool`...), or +`string`.""" +function dtype_name(column::AbstractArray) + element = nonmissingtype(eltype(column)) + return element <: AbstractString ? "string" : lowercase(string(element)) +end + +function table_dict(table::AbstractTable) + entry = json_object("name" => table.name, "count" => row_count(table), + "columns" => json_object(name => json_object("dtype" => dtype_name(column), + "values" => json_column(column)) + for (name, column) in table.columns)) + declarations = json_object() + for (name, variable) in table.variables + declared = json_object("unit" => variable.unit, "role" => variable.role) + variable.standard_name === nothing || (declared["standard_name"] = variable.standard_name) + variable.kind === nothing || (declared["kind"] = variable.kind) + variable.description === nothing || (declared["description"] = variable.description) + variable.minimum === nothing || (declared["minimum"] = variable.minimum) + variable.maximum === nothing || (declared["maximum"] = variable.maximum) + isempty(variable.references) || (declared["references"] = variable.references) + isempty(variable.doi) || (declared["doi"] = variable.doi) + variable.format === nothing || (declared["format"] = variable.format) + declarations[name] = declared + end + isempty(declarations) || (entry["variables"] = declarations) + return entry +end + +""" + frame_to_dict(frame) -> OrderedDict + +`frame` as plain JSON values, in the layout of the conformance files' expected values. +""" +function frame_to_dict(frame::Frame) + data_structures = Any[] + for data_structure in frame.data_structures + entry = json_object("id" => data_structure.id, "kind" => data_structure.kind, + "origin" => [collect(row) for row in eachrow(data_structure.origin)], + "entity_sets" => [merge!(table_dict(set), json_object("rank" => set.rank), + set.scale === nothing ? json_object() : + json_object("scale" => set.scale)) + for set in data_structure.entity_sets]) + data_structure.platform === nothing || (entry["platform"] = data_structure.platform) + data_structure.model === nothing || (entry["model"] = data_structure.model) + isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) + isempty(data_structure.labels) || (entry["labels"] = data_structure.labels) + if !isempty(data_structure.relations) + entry["relations"] = [merge!(table_dict(relation), json_object("source_set" => relation.source_set, + "target_set" => relation.target_set)) + for relation in data_structure.relations] + end + data_structure.scalars === nothing || (entry["scalars"] = table_dict(data_structure.scalars)) + if data_structure.grid !== nothing + grid = data_structure.grid + entry["grid"] = json_object("kind" => grid.kind, "shape" => grid.shape, "origin" => grid.origin, + "spacing" => grid.spacing, "axes" => grid.axes, "periodic" => grid.periodic, + "z_up" => grid.z_up, "max_level" => grid.max_level, "cells" => grid.cells) + end + if !isempty(data_structure.meshes) + entry["meshes"] = map(data_structure.meshes) do mesh + item = json_object("name" => mesh.name, + "positions_float32" => json_column(Float64.(Float32.(mesh.positions))), + "feature_rows" => json_object(name => rows for (name, rows) in mesh.feature_rows)) + mesh.indices === nothing || (item["indices"] = mesh.indices) + isempty(mesh.display) || (item["display"] = json_object(name => json_column(Float64.(values)) + for (name, values) in mesh.display)) + item + end + end + push!(data_structures, entry) + end + mappings = [merge!(table_dict(mapping), json_object("source" => collect(mapping.source), + "target" => collect(mapping.target), + "method" => mapping.method)) + for mapping in frame.mappings] + out = json_object("time" => frame.time, "time_unit" => frame.time_unit, "step" => frame.step, + "data_structures" => data_structures, "mappings" => mappings) + if frame.update !== nothing # a partial frame: its sender and removed entities, sorted + out["update"] = json_object("participant" => frame.update.participant, + "removed" => Any[json_object("data_structure" => key[1], "entity_set" => key[2], "ids" => ids) + for (key, ids) in sort(collect(frame.update.removed); by = first)]) + end + return out +end + +""" + write_dump(path, frame) + +Write `frame` as JSON in the layout of the conformance files' expected values, for `encoding3d compare`. +""" +function write_dump(path::AbstractString, frame::Frame) + open(path, "w") do io + JSON.json(io, frame_to_dict(frame)) + end + return path +end diff --git a/bindings/julia/encoding3d/src/encoding3d.jl b/bindings/julia/encoding3d/src/encoding3d.jl new file mode 100644 index 0000000..01dc7b0 --- /dev/null +++ b/bindings/julia/encoding3d/src/encoding3d.jl @@ -0,0 +1,86 @@ +""" + encoding3d + +Reading and writing encoding3d files in Julia: whole functional-structural plant model scenes (multiscale plant graphs, +their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 +binary (GLB) file per time step. The specification is `spec/specification.md` in the encoding3d repository. + + read_frame(path) -> Frame write_frame(path, frame) + from_bytes(data) -> Frame to_bytes(frame) -> Vector{UInt8} + read_index(path), read_sequence(path) write_sequence(directory, frames) + frame_to_dict(frame), write_dump(path, frame) the layout of the conformance files' expected values + +With MultiScaleTreeGraph loaded, `from_mtg` and `to_mtg` convert an MTG to and from a data structure. With +VirtualPlantLab's PlantGraphs loaded, `from_graph` and `to_graph` convert its graphs; with PlantGeomTurtle too, +`add_geometry!` adds a graph's turtle geometry as a mesh. + +A platform takes part in a coupling session (specification §11) with `Participant(declaration)`, then answers each of +`turns(participant)` with `send_update(participant, frame)`; over a folder ("files:path"), or ZeroMQ with ZMQ loaded. +""" +module encoding3d + +using Dates +using JSON +using OrderedCollections: OrderedDict + +"""The version of the specification this package reads and writes.""" +const SPECIFICATION_VERSION = "0.3" + +include("glb.jl") +include("model.jl") +include("codec.jl") +include("sequence.jl") +include("dump.jl") +include("values.jl") +include("coupling.jl") + +export Frame, DataStructure, EntitySet, Relation, Table, Mapping, Grid, Mesh, Variable, NO_ID, EDGE_TYPES +export read_frame, write_frame, from_bytes, to_bytes, read_index, read_sequence, write_sequence, write_index +export frame_to_dict, write_dump, row_count, variable_names, entity_ids, entity_set, data_structure +export from_mtg, to_mtg, from_graph, to_graph, add_geometry! +export Update, Declaration, EntitySetSpec, VariableSpec, MappingSpec, Link, declaration_from_json, Participant, Turn, next_turn, turns, + send_update, fail + +""" + from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) + -> DataStructure + +A MultiScaleTreeGraph (its root node) as a `graph` data structure. Defined when MultiScaleTreeGraph is loaded +(the package extension `encoding3dMultiScaleTreeGraphExt`). +""" +function from_mtg end + +""" + to_mtg(data_structure) -> Node + +A `graph` data structure as a MultiScaleTreeGraph (its root node). Defined when MultiScaleTreeGraph is loaded (the +package extension `encoding3dMultiScaleTreeGraphExt`). +""" +function to_mtg end + +""" + from_graph(graph; id="plant", entity_set="Organ", rank=5, variables=Dict(), keep, platform="VirtualPlantLab", + model=nothing) -> DataStructure + +A VirtualPlantLab graph (PlantGraphs: `Graph` or `StaticGraph`) as a `graph` data structure of one entity set. +Defined when PlantGraphs is loaded (the package extension `encoding3dPlantGraphsExt`). +""" +function from_graph end + +""" + to_graph(data_structure; types, entity_set=nothing) -> (StaticGraph, ids) + +A data structure's entities as a PlantGraphs `StaticGraph`, each node built from its row by `types[label]`. Defined +when PlantGraphs is loaded (the package extension `encoding3dPlantGraphsExt`). +""" +function to_graph end + +""" + add_geometry!(data_structure, graph; entity_set="Organ", turtle=Turtle(Float64), name="plant") -> Mesh + +Adds a VirtualPlantLab graph's turtle geometry to its data structure as a mesh, each vertex carrying its node's row. +Defined when PlantGraphs, PlantGeomTurtle and PlantGeomPrimitives are loaded (`encoding3dPlantGeomTurtleExt`). +""" +function add_geometry! end + +end diff --git a/bindings/julia/encoding3d/src/glb.jl b/bindings/julia/encoding3d/src/glb.jl new file mode 100644 index 0000000..7eafbf4 --- /dev/null +++ b/bindings/julia/encoding3d/src/glb.jl @@ -0,0 +1,117 @@ +# The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary +# chunk padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. + +const GLB_MAGIC = 0x46546C67 # "glTF" +const GLB_VERSION = 0x00000002 +const JSON_CHUNK = 0x4E4F534A # "JSON" +const BINARY_CHUNK = 0x004E4942 # "BIN\0" +const ALIGNMENT = 8 + +padding(length::Integer) = mod(-length, ALIGNMENT) + +function write_word!(io::IO, value::UInt32) + write(io, htol(value)) +end + +""" + pack(document, binary) -> Vector{UInt8} + +The bytes of a GLB file from its JSON `document` (any value JSON.jl writes) and its `binary` buffer. +""" +function pack(document, binary::AbstractVector{UInt8}) + text = Vector{UInt8}(codeunits(JSON.json(document))) + append!(text, fill(UInt8(' '), padding(length(text)))) + binary = vcat(binary, zeros(UInt8, padding(length(binary)))) + total = 12 + 8 + length(text) + (isempty(binary) ? 0 : 8 + length(binary)) + io = IOBuffer(sizehint = total) + write_word!(io, GLB_MAGIC) + write_word!(io, GLB_VERSION) + write_word!(io, UInt32(total)) + write_word!(io, UInt32(length(text))) + write_word!(io, JSON_CHUNK) + write(io, text) + if !isempty(binary) + write_word!(io, UInt32(length(binary))) + write_word!(io, BINARY_CHUNK) + write(io, binary) + end + return take!(io) +end + +read_word(data::AbstractVector{UInt8}, offset::Integer) = ltoh(reinterpret(UInt32, data[offset+1:offset+4])[1]) + +""" + unpack(data) -> (document, binary) + +The JSON document (a `JSON.Object`, keys in their order) and the binary buffer of a GLB file's bytes. +""" +function unpack(data::AbstractVector{UInt8}) + length(data) < 20 && throw(ArgumentError("not a GLB file: too short")) + read_word(data, 0) == GLB_MAGIC || throw(ArgumentError("not a GLB file: wrong magic")) + version = read_word(data, 4) + version == GLB_VERSION || throw(ArgumentError("GLB version $(Int(version)), 2 expected")) + total = read_word(data, 8) + total == length(data) || + throw(ArgumentError("GLB header length $(Int(total)), the file holds $(length(data)) bytes")) + offset, document, binary = 12, nothing, UInt8[] + while offset < total + chunk_length, chunk_type = read_word(data, offset), read_word(data, offset + 4) + chunk = view(data, offset+9:offset+8+Int(chunk_length)) + if chunk_type == JSON_CHUNK + document = JSON.parse(String(copy(chunk))) + elseif chunk_type == BINARY_CHUNK + binary = Vector{UInt8}(chunk) + end + offset += 8 + Int(chunk_length) + end + document === nothing && throw(ArgumentError("GLB file without a JSON chunk")) + return document, binary +end + +""" + BufferBuilder() + +Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. +""" +mutable struct BufferBuilder + data::Vector{UInt8} + views::Vector{OrderedDict{String,Any}} +end +BufferBuilder() = BufferBuilder(UInt8[], OrderedDict{String,Any}[]) + +little_endian_bytes(values::AbstractArray{T}) where {T<:Union{Number,Bool}} = + Vector{UInt8}(reinterpret(UInt8, htol.(vec(collect(values))))) + +""" + add!(buffers, data; target=nothing) -> Int + +Append `data` (bytes, or an array of numbers written little-endian) to the buffer; the 0-based index of its buffer +view. +""" +function add!(buffers::BufferBuilder, data::AbstractVector{UInt8}; target = nothing) + append!(buffers.data, zeros(UInt8, padding(length(buffers.data)))) + buffer_view = OrderedDict{String,Any}("buffer" => 0, "byteOffset" => length(buffers.data), + "byteLength" => length(data)) + target === nothing || (buffer_view["target"] = target) + append!(buffers.data, data) + push!(buffers.views, buffer_view) + return length(buffers.views) - 1 +end +add!(buffers::BufferBuilder, data::AbstractArray; target = nothing) = + add!(buffers, little_endian_bytes(data); target = target) + +function view_bytes(document, binary::Vector{UInt8}, index::Integer) + buffer_view = document["bufferViews"][index+1] + get(buffer_view, "buffer", 0) == 0 || + throw(ArgumentError("encoding3d files keep every buffer view in the GLB binary chunk")) + start = get(buffer_view, "byteOffset", 0) + return view(binary, start+1:start+buffer_view["byteLength"]) +end + +"""The `count` values of type `T` stored little-endian in `bytes` from byte `offset` (0-based).""" +function read_values(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer, offset::Integer = 0) where {T} + size = sizeof(T) + length(bytes) >= offset + count * size || + throw(ArgumentError("a buffer view holds $(length(bytes)) bytes, $(offset + count * size) needed")) + return ltoh.(collect(reinterpret(T, bytes[offset+1:offset+count*size]))) +end diff --git a/bindings/julia/encoding3d/src/model.jl b/bindings/julia/encoding3d/src/model.jl new file mode 100644 index 0000000..aabc1da --- /dev/null +++ b/bindings/julia/encoding3d/src/model.jl @@ -0,0 +1,341 @@ +# The encoding3d data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +# +# Frame one time step of a scene +# DataStructure one plant population's graph, or one environment model's grid +# EntitySet the entities of one scale (or a grid's cells): a table of columns +# Relation an edge table between entity sets (cell adjacency, a branch's bearer, ...) +# Table the data structure's scalars: a table of one row +# Grid a regular or adaptive grid's layout +# Mesh triangles, each vertex carrying the row of its entity in one or several entity sets +# Mapping source entity ↔ target entity weights between two data structures +# +# A column is a Vector (one value per row: numbers, Bool, String) or a Matrix (one row per entity, one column per +# component: a fixed-size vector per entity). Missing values: NaN in float columns; `missing` elsewhere (a +# Vector{Union{Missing,T}} of integers, booleans or text). + +"""The id of no entity: the `complex` or `parent` of an entity that has none, the feature row of a vertex with none.""" +const NO_ID = -1 +"""The values of `edge_type`: the root of its scale, MTG `<`, MTG `+`.""" +const EDGE_TYPES = ("none", "successor", "branch", "unknown") +const ENUM_COLUMNS = ("label", "edge_type", "relation") +const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "sibling_order") +"""The vocabulary's base scales: an entity set's `scale` is one of them, or `"mixed"`.""" +const BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", + "Connection") +"""The formats of a text variable holding ISO 8601 dates or times.""" +const FORMATS = ("date", "date-time", "time") +const DATA_STRUCTURE_KINDS = ("graph", "regular_grid", "adaptive_grid", "table") +const ROLES = ("state", "input", "parameter", "derived", "setting") +const COLUMN_TYPES = (Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64) + +""" + Variable(; unit="1", standard_name=nothing, kind=nothing, role="state", description=nothing, + minimum=nothing, maximum=nothing, references=String[], doi=String[]) + +The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind +(`extensive`, `intensive`, ...: how it maps between scales and grids), its role and its provenance; the `format` of +a text column holding dates or times (`"date"`, `"date-time"`, `"time"`). +""" +Base.@kwdef mutable struct Variable + unit::String = "1" + standard_name::Union{Nothing,String} = nothing + kind::Union{Nothing,String} = nothing + role::String = "state" + description::Union{Nothing,String} = nothing + minimum::Union{Nothing,Real} = nothing + maximum::Union{Nothing,Real} = nothing + references::Vector{String} = String[] + doi::Vector{String} = String[] + format::Union{Nothing,String} = nothing +end + +function Base.:(==)(left::Variable, right::Variable) + all(isequal(getfield(left, name), getfield(right, name)) for name in fieldnames(Variable)) +end + +const Columns = OrderedDict{String,AbstractArray} +const Declarations = OrderedDict{String,Variable} +const JSONObject = AbstractDict{String,Any} + +"""The tables of a frame: `Table` (scalars), `EntitySet`, `Relation`, `Mapping`. Each has `name`, `columns` and +`variables`.""" +abstract type AbstractTable end + +""" + Table(name; columns=Columns(), variables=Declarations()) + +Columns of equal length and the declarations of its variables: a data structure's scalars (one row). +""" +mutable struct Table <: AbstractTable + name::String + columns::Columns + variables::Declarations +end + +""" + EntitySet(name; columns, variables, rank=nothing) + +The entities of one scale of a graph (`rank`: the scale's rank, coarser first), or a grid's cells (`rank` nothing). +Reserved columns: `id` (Int64, stable across frames, never reused), `label` (String), `complex` (the id of the +containing entity at a coarser scale, `NO_ID` if none), `parent` (the predecessor's id at the same scale, `NO_ID` if +none), `edge_type` (`"none"`, `"successor"`, `"branch"`, `"unknown"`), `sibling_order` (optional: its position among its tree +parent's children); an adaptive grid's cells: `level` and `index`. `scale`: its base scale (`BASE_SCALES`), or +`"mixed"`. +""" +mutable struct EntitySet <: AbstractTable + name::String + columns::Columns + variables::Declarations + rank::Union{Nothing,Int} + scale::Union{Nothing,String} +end + +""" + Relation(name; columns, variables, source_set, target_set) + +An edge table: `source` and `target` entity ids (in the entity sets `source_set` and `target_set`), an optional `id` +per edge, an optional `relation` label per edge, and edge variables. +""" +mutable struct Relation <: AbstractTable + name::String + columns::Columns + variables::Declarations + source_set::String + target_set::String +end + +""" + Mapping(name; columns, variables, source, target, method=nothing) + +Weights between the entities of two data structures: `source` and `target` ids, `weight`. `source` and `target`: +(data structure id, entity set name). +""" +mutable struct Mapping <: AbstractTable + name::String + columns::Columns + variables::Declarations + source::Tuple{String,String} + target::Tuple{String,String} + method::Union{Nothing,String} +end + +# The columns whose meaning the specification fixes, per kind of table +const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order") +const RELATION_RESERVED = ("id", "source", "target", "relation") +const MAPPING_RESERVED = ("source", "target", "weight") +reserved(::Table) = () +reserved(::EntitySet) = ENTITY_SET_RESERVED +reserved(::Relation) = RELATION_RESERVED +reserved(::Mapping) = MAPPING_RESERVED + +as_column(name::AbstractString, values::AbstractVector{<:AbstractString}) = Vector{String}(values) +as_column(name::AbstractString, values::AbstractVector{Bool}) = Vector{Bool}(values) +as_column(name::AbstractString, values::AbstractVector{T}) where {T<:Number} = Vector{T}(values) +as_column(name::AbstractString, values::AbstractMatrix{T}) where {T<:Number} = Matrix{T}(values) +as_column(name::AbstractString, values::Number) = [values] +as_column(name::AbstractString, values::AbstractString) = [String(values)] +function as_column(name::AbstractString, values::AbstractVector) + isempty(values) && return Float64[] + present = [value for value in values if !ismissing(value)] + if all(value -> value isa AbstractString, present) + return length(present) == length(values) ? Vector{String}(values) : + Union{Missing,String}[ismissing(value) ? missing : String(value) for value in values] + end + promoted = isempty(present) ? Float64[] : [present...] # the narrowest common type + element = eltype(promoted) + element <: Union{Number,Bool} || + throw(ArgumentError("column '$name': numbers, booleans or strings per row (got $element)")) + length(present) == length(values) && return promoted + element <: AbstractFloat && return element[ismissing(value) ? NaN : value for value in values] # missing: NaN + return Union{Missing,element}[value for value in values] +end +as_column(name::AbstractString, values) = + throw(ArgumentError("column '$name': a vector, or a matrix of one row per entity (got $(typeof(values)))")) + +function prepare_columns(table_name::AbstractString, columns, reserved_names) + prepared = Columns() + for (name, values) in pairs(columns) + name = String(name) + column = as_column(name, values) + if name in ID_COLUMNS && name in reserved_names && eltype(column) <: Integer && !(eltype(column) <: Bool) + column = convert(Array{Int64}, column) # ids are Int64, whatever the platform's integers + end + name in reserved_names && Missing <: eltype(column) && + throw(ArgumentError("table '$table_name': the reserved column '$name' has no missing values")) + prepared[name] = column + end + counts = Dict(name => size(column, 1) for (name, column) in prepared) + length(Set(values(counts))) > 1 && + throw(ArgumentError("table '$table_name': columns of different lengths $counts")) + return prepared +end + +prepare_variables(variables) = Declarations(String(name) => variable for (name, variable) in pairs(variables)) + +Table(name::AbstractString; columns = Columns(), variables = Declarations()) = + Table(String(name), prepare_columns(name, columns, ()), prepare_variables(variables)) +EntitySet(name::AbstractString; columns = Columns(), variables = Declarations(), rank = nothing, scale = nothing) = + EntitySet(String(name), prepare_columns(name, columns, ENTITY_SET_RESERVED), prepare_variables(variables), rank, + scale) +Relation(name::AbstractString; columns = Columns(), variables = Declarations(), source_set = "", target_set = "") = + Relation(String(name), prepare_columns(name, columns, RELATION_RESERVED), prepare_variables(variables), + String(source_set), String(target_set)) +Mapping(name::AbstractString; columns = Columns(), variables = Declarations(), source = ("", ""), + target = ("", ""), method = nothing) = + Mapping(String(name), prepare_columns(name, columns, MAPPING_RESERVED), prepare_variables(variables), + (String(source[1]), String(source[2])), (String(target[1]), String(target[2])), method) + +"""The number of rows of a table.""" +row_count(table::AbstractTable) = isempty(table.columns) ? 0 : size(first(values(table.columns)), 1) + +"""The names of a table's variable columns (its columns but the reserved ones).""" +variable_names(table::AbstractTable) = [name for name in keys(table.columns) if !(name in reserved(table))] + +"""The declaration of a column (an undeclared one: unit "1", role "state").""" +declaration(table::AbstractTable, name::AbstractString) = get(table.variables, name, Variable()) + +"""The ids of an entity set's entities: its `id` column, or a grid's row numbers (from 0) when it has none.""" +entity_ids(entity_set::EntitySet) = + haskey(entity_set.columns, "id") ? entity_set.columns["id"] : collect(Int64, 0:row_count(entity_set)-1) + +""" + Grid(; kind="regular", shape, origin, spacing, axes=["x","y","z"], periodic, z_up=true, max_level=0, + cells="Cell") + +A grid's layout. Regular: `shape` cells of size `spacing` from `origin`, along `axes`, `periodic` per axis, z +pointing up or down (`z_up`). Adaptive: the same for its base level, refined up to `max_level`, its leaf cells (an +entity set with `level` and `index` columns) explicit. `cells`: the name of the cells' entity set. +""" +Base.@kwdef mutable struct Grid + kind::String = "regular" + shape::Vector{Int} = Int[] + origin::Vector{Float64} = Float64[] + spacing::Vector{Float64} = Float64[] + axes::Vector{String} = ["x", "y", "z"] + periodic::Vector{Bool} = Bool[] + z_up::Bool = true + max_level::Int = 0 + cells::String = "Cell" +end + +""" + Mesh(name; positions, feature_rows=OrderedDict(), indices=nothing, display=OrderedDict(), + display_sources=Dict()) + +Triangles: `positions` (V×3, in the data structure's frame), three consecutive vertices per triangle, or `indices` +(3T, 0-based) into them; `feature_rows`: entity set name => the row (0-based) of each vertex's entity in that set, +`NO_ID` for none; `display`: variable name => its Float32 copy per vertex, from the entity set `display_sources[name]`. +""" +mutable struct Mesh + name::String + positions::Matrix{Float64} + feature_rows::OrderedDict{String,Vector{Int64}} + indices::Union{Nothing,Vector{Int64}} + display::OrderedDict{String,Vector{Float32}} + display_sources::Dict{String,String} +end + +function Mesh(name::AbstractString; positions, feature_rows = OrderedDict{String,Vector{Int64}}(), + indices = nothing, display = OrderedDict{String,Vector{Float32}}(), + display_sources = Dict{String,String}()) + positions = Matrix{Float64}(positions isa AbstractVector ? permutedims(reshape(positions, 3, :)) : positions) + size(positions, 2) == 3 || throw(ArgumentError("mesh '$name': positions are V×3")) + rows = OrderedDict{String,Vector{Int64}}(String(set) => Vector{Int64}(vec(values)) for (set, values) in pairs(feature_rows)) + for (set, values) in rows + length(values) == size(positions, 1) || + throw(ArgumentError("mesh '$name': $(length(values)) feature rows for '$set', $(size(positions, 1)) vertices")) + end + Mesh(String(name), positions, rows, indices === nothing ? nothing : Vector{Int64}(vec(indices)), + OrderedDict{String,Vector{Float32}}(String(variable) => Vector{Float32}(vec(values)) + for (variable, values) in pairs(display)), + Dict{String,String}(String(variable) => String(set) for (variable, set) in pairs(display_sources))) +end + +""" + DataStructure(id; kind="graph", entity_sets=EntitySet[], relations=Relation[], scalars=nothing, grid=nothing, + meshes=Mesh[], origin=identity, platform=nothing, model=nothing, extras=OrderedDict()) + +One plant population's graph, or one environment model's grid. `origin`: its frame in the scene (a 4×4 matrix, +`origin[row, column]`, or 16 numbers row by row). `extras`: JSON values for the writing platform (e.g. what it needs +to rebuild its own data structure); other readers keep them without interpreting them. `labels`: the platform's +labels => the vocabulary's. +""" +mutable struct DataStructure + id::String + kind::String + entity_sets::Vector{EntitySet} + relations::Vector{Relation} + scalars::Union{Nothing,Table} + grid::Union{Nothing,Grid} + meshes::Vector{Mesh} + origin::Matrix{Float64} + platform::Union{Nothing,String} + model::Union{Nothing,String} + extras::AbstractDict{String,Any} + labels::AbstractDict{String,String} +end + +identity_matrix() = Matrix{Float64}([i == j for i in 1:4, j in 1:4]) + +function DataStructure(id::AbstractString; kind = "graph", entity_sets = EntitySet[], relations = Relation[], + scalars = nothing, grid = nothing, meshes = Mesh[], origin = identity_matrix(), + platform = nothing, model = nothing, extras = OrderedDict{String,Any}(), + labels = OrderedDict{String,String}()) + origin = Matrix{Float64}(origin isa AbstractVector ? permutedims(reshape(origin, 4, 4)) : origin) + size(origin) == (4, 4) || throw(ArgumentError("data structure '$id': the origin is a 4×4 matrix")) + DataStructure(String(id), String(kind), collect(EntitySet, entity_sets), collect(Relation, relations), scalars, + grid, collect(Mesh, meshes), origin, platform, model, extras, + OrderedDict{String,String}(String(key) => String(value) for (key, value) in pairs(labels))) +end + +"""The entity set `name` of a data structure.""" +function entity_set(data_structure::DataStructure, name::AbstractString) + for candidate in data_structure.entity_sets + candidate.name == name && return candidate + end + throw(KeyError("data structure '$(data_structure.id)' has no entity set '$name'")) +end + +""" + Update(participant; removed=OrderedDict()) + +What makes a frame partial: a coupling session's message (specification §11). `participant`: its sender +(`"coordinator"` for a turn's inputs). Its tables hold some rows and columns only. `removed`: (data structure id, +entity set name) => the ids of the entities removed. +""" +mutable struct Update + participant::String + removed::OrderedDict{Tuple{String,String},Vector{Int64}} +end + +Update(participant::AbstractString; removed = OrderedDict{Tuple{String,String},Vector{Int64}}()) = + Update(String(participant), OrderedDict{Tuple{String,String},Vector{Int64}}( + (String(key[1]), String(key[2])) => Vector{Int64}(ids) for (key, ids) in removed)) + +""" + Frame(; data_structures=DataStructure[], mappings=Mapping[], time=0.0, time_unit="d", step=0, + writer=OrderedDict(), vocabulary_version=nothing, update=nothing, unknown=OrderedDict()) + +A scene at one time step (`time` in `time_unit`, `step`), or a parameter set. `update`: set for a partial frame +(`Update`), a coupling session's message. `unknown`: the content of a read file this package does not interpret +(other root extensions, the root's `extras`), written back as it was. +""" +Base.@kwdef mutable struct Frame + data_structures::Vector{DataStructure} = DataStructure[] + mappings::Vector{Mapping} = Mapping[] + time::Float64 = 0.0 + time_unit::String = "d" + step::Int = 0 + writer::AbstractDict{String,Any} = OrderedDict{String,Any}() + vocabulary_version::Union{Nothing,String} = nothing + update::Union{Nothing,Update} = nothing + unknown::AbstractDict{String,Any} = OrderedDict{String,Any}() +end + +"""The data structure `id` of a frame.""" +function data_structure(frame::Frame, id::AbstractString) + for candidate in frame.data_structures + candidate.id == id && return candidate + end + throw(KeyError("the frame has no data structure '$id'")) +end diff --git a/bindings/julia/encoding3d/src/sequence.jl b/bindings/julia/encoding3d/src/sequence.jl new file mode 100644 index 0000000..691e9b7 --- /dev/null +++ b/bindings/julia/encoding3d/src/sequence.jl @@ -0,0 +1,63 @@ +# Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. + +const INDEX_FILE = "scene.fspm.json" + +""" + write_index(path, entries; time_unit="d") + +Write the index of a sequence: `entries`, one `(time, step, file)` per frame (a NamedTuple or a dictionary), in time +order. +""" +function write_index(path::AbstractString, entries; time_unit::AbstractString = "d") + field(entry, name) = entry isa NamedTuple ? getproperty(entry, name) : entry[string(name)] + frames = [json_object("time" => Float64(field(entry, :time)), "step" => Int(field(entry, :step)), + "file" => String(field(entry, :file))) for entry in entries] + open(path, "w") do io + JSON.json(io, json_object("fspmIndex" => SPECIFICATION_VERSION, "timeUnit" => time_unit, "frames" => frames); + pretty = 1) + write(io, "\n") + end + return path +end + +""" + write_sequence(directory, frames; name="frame") -> String + +Write `frames` to `directory`, one GLB each (`name_000000.glb`, ...), and their index; the index's path. +""" +function write_sequence(directory::AbstractString, frames; name::AbstractString = "frame") + mkpath(directory) + entries, time_units = NamedTuple[], Set{String}() + for (number, frame) in enumerate(frames) + file = string(name, "_", lpad(number - 1, 6, '0'), ".glb") + write_frame(joinpath(directory, file), frame) + push!(entries, (time = frame.time, step = frame.step, file = file)) + push!(time_units, frame.time_unit) + end + length(time_units) > 1 && + throw(ArgumentError("the frames' times are in several units: $(sort(collect(time_units)))")) + path = joinpath(directory, INDEX_FILE) + write_index(path, entries; time_unit = isempty(time_units) ? "d" : first(time_units)) + return path +end + +""" + read_index(path) -> JSON.Object + +The index of a sequence: its frames' times, steps and files, each frame with the absolute `"path"` of its file. +""" +function read_index(path::AbstractString) + index = JSON.parse(read(path, String)) + directory = dirname(abspath(path)) + for entry in get(index, "frames", Any[]) + entry["path"] = joinpath(directory, entry["file"]) + end + return index +end + +""" + read_sequence(path) + +The frames of the sequence whose index is `path`, read one after the other as they are iterated. +""" +read_sequence(path::AbstractString) = (read_frame(entry["path"]) for entry in read_index(path)["frames"]) diff --git a/bindings/julia/encoding3d/src/values.jl b/bindings/julia/encoding3d/src/values.jl new file mode 100644 index 0000000..2ccabea --- /dev/null +++ b/bindings/julia/encoding3d/src/values.jl @@ -0,0 +1,56 @@ +# Values of a platform's objects (attributes, fields) as columns: what the adapters share. + +const TextType = Union{Date,DateTime,Time} +"""The format (a variable's `format`) of each type written as ISO 8601 text.""" +const FORMAT_OF = Dict{DataType,String}(Date => "date", DateTime => "date-time", Time => "time") +"""Each format's type, to read its text back.""" +const TYPE_OF_FORMAT = Dict{String,DataType}(format => type for (type, format) in FORMAT_OF) + +""" + column_of_values(values) -> (column, format) or nothing + +A column of `values`, `nothing` where an object has no such value: missing (NaN in a float column, `missing` +otherwise). `format`: the variable's format when the values are dates or times written as ISO 8601 text, else +nothing. Nothing for values encoding3d cannot hold. +""" +function column_of_values(values::AbstractVector) + present = [value for value in values if value !== nothing] + complete = length(present) == length(values) + column_of(type) = complete ? type[value for value in values] : + Union{Missing,type}[value === nothing ? missing : value for value in values] + format = nothing + if all(value -> value isa Bool, present) + column = column_of(Bool) + elseif all(value -> value isa Integer && !(value isa Bool), present) + column = column_of(Int64) + elseif all(value -> value isa Real && !(value isa Bool), present) + column = Float64[value === nothing ? NaN : value for value in values] + elseif all(value -> value isa Union{AbstractString,Symbol}, present) + column = complete ? String[string(value) for value in values] : + Union{Missing,String}[value === nothing ? missing : string(value) for value in values] + elseif !isempty(present) && all(value -> value isa TextType, present) && length(Set(typeof.(present))) == 1 + format = FORMAT_OF[typeof(first(present))] + column = complete ? String[string(value) for value in values] : + Union{Missing,String}[value === nothing ? missing : string(value) for value in values] + elseif all(value -> value isa AbstractVector{<:Real}, present) && length(Set(length.(present))) == 1 && + length(first(present)) >= 2 + width = length(first(present)) + integer = complete && all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) + column = integer ? zeros(Int64, length(values), width) : fill(NaN, length(values), width) + for (row, value) in enumerate(values) + value === nothing || (column[row, :] .= value) # a missing vector: NaN + end + else + return nothing + end + return column, format +end + +"""A column's value at `row` as a platform reads it back: `missing` stays missing, ISO 8601 text of a `format` its +date or time, a vector row a Vector.""" +function value_of(column::AbstractVector, row, format) + value = column[row] + ismissing(value) && return missing + return format === nothing ? value : TYPE_OF_FORMAT[format](value) +end +value_of(column::AbstractMatrix, row, format) = collect(column[row, :]) diff --git a/bindings/julia/encoding3d/test/Project.toml b/bindings/julia/encoding3d/test/Project.toml new file mode 100644 index 0000000..386a2f2 --- /dev/null +++ b/bindings/julia/encoding3d/test/Project.toml @@ -0,0 +1,7 @@ +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +JSON = "1" diff --git a/bindings/julia/encoding3d/test/coupling/function.jl b/bindings/julia/encoding3d/test/coupling/function.jl new file mode 100644 index 0000000..033bd0f --- /dev/null +++ b/bindings/julia/encoding3d/test/coupling/function.jl @@ -0,0 +1,45 @@ +# A Julia participant for the coupling tests (bindings/python/test/test_coupling.py launches it with the Python +# coordinator): the "function" of the two-scale session. It refines each organ it is sent into two segments and +# provides each segment's sugar, 1e-6 mol per m of its length (its input). +using encoding3d +using ZMQ + +declaration = Declaration("function"; platform = "julia-test", + entity_sets = [EntitySetSpec("plant", "Organ"), EntitySetSpec("plant", "SubOrgan"; rights = "refines")], + variables = [VariableSpec("plant", "SubOrgan", "segment_length", "requires"; unit = "m", standard_name = "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides"; unit = "mol", kind = "extensive")]) +participant = Participant(declaration; timeout = 60) + +segments = Dict{Int64,Tuple{Int64,Int64}}() # segment id => (its organ, its parent) +order = Int64[] # the segments in creation order +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + removed = get(turn.frame.update.removed, ("plant", "SubOrgan"), Int64[]) + for segment in removed + delete!(segments, segment) + end + filter!(segment -> haskey(segments, segment), order) + lengths = Dict{Int64,Float64}() + if any(set -> set.name == "SubOrgan", plant.entity_sets) + known = entity_set(plant, "SubOrgan") + for (identifier, length) in zip(known.columns["id"], known.columns["segment_length"]) + lengths[identifier] = isnan(length) ? 0.0 : length + end + end + with_segments = Set(first(value) for value in values(segments)) + for organ in entity_set(plant, "Organ").columns["id"] + organ in with_segments && continue + start = participant.id_offset + length(segments) + 2 * turn.step * 1000 + segments[start] = (organ, NO_ID) + segments[start+1] = (organ, start) + push!(order, start, start + 1) + end + columns = encoding3d.Columns( + "id" => order, + "complex" => [segments[segment][1] for segment in order], + "parent" => [segments[segment][2] for segment in order], + "edge_type" => [segments[segment][2] == NO_ID ? "none" : "successor" for segment in order], + "sugar" => [1e-6 * get(lengths, segment, 0.0) for segment in order]) + send_update(participant, Frame(data_structures = [DataStructure("plant"; + entity_sets = [EntitySet("SubOrgan"; columns = columns, rank = 6)])])) +end diff --git a/bindings/julia/encoding3d/test/multiscaletreegraph/Project.toml b/bindings/julia/encoding3d/test/multiscaletreegraph/Project.toml new file mode 100644 index 0000000..3692d1f --- /dev/null +++ b/bindings/julia/encoding3d/test/multiscaletreegraph/Project.toml @@ -0,0 +1,10 @@ +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[compat] +JSON = "1" +MultiScaleTreeGraph = "0.16" diff --git a/bindings/julia/encoding3d/test/multiscaletreegraph/runtests.jl b/bindings/julia/encoding3d/test/multiscaletreegraph/runtests.jl new file mode 100644 index 0000000..c53958c --- /dev/null +++ b/bindings/julia/encoding3d/test/multiscaletreegraph/runtests.jl @@ -0,0 +1,177 @@ +# encoding3d's MultiScaleTreeGraph.jl extension (an MTG as a data structure, and back): a proposal for MTG.jl's users +# (VirtualPlantLab, PlantSimEngine), tested by hand in its own project, so that encoding3d's tests and CI depend on no +# platform: +# +# julia --project=bindings/julia/encoding3d/test/multiscaletreegraph -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.instantiate()' +# julia --project=bindings/julia/encoding3d/test/multiscaletreegraph bindings/julia/encoding3d/test/multiscaletreegraph/runtests.jl +using Dates +using encoding3d +using JSON +using MultiScaleTreeGraph +using Test + +const CONFORMANCE = normpath(joinpath(@__DIR__, "..", "..", "..", "..", "..", "conformance")) +const MTG_FILES = joinpath(pkgdir(MultiScaleTreeGraph), "test", "files") + +conformance_files() = sort([relpath(joinpath(folder, name), CONFORMANCE) + for (folder, _, files) in walkdir(CONFORMANCE) for name in files + if endswith(name, ".glb") && !startswith(relpath(folder, CONFORMANCE), "coupling")]) + +"""The differences between two JSON values, as `encoding3d compare` finds them: objects by key, arrays in order, +numbers exactly (1 and 1.0 equal).""" +function differences(actual, expected, path = "\$") + if expected isa AbstractDict + actual isa AbstractDict || return ["$path: an object expected"] + found = ["$path: missing key '$key'" for key in keys(expected) if !haskey(actual, key)] + append!(found, ["$path: unexpected key '$key'" for key in keys(actual) if !haskey(expected, key)]) + for key in keys(expected) + haskey(actual, key) && append!(found, differences(actual[key], expected[key], "$path.$key")) + end + return found + elseif expected isa AbstractVector + actual isa AbstractVector || return ["$path: an array expected"] + length(actual) == length(expected) || return ["$path: $(length(actual)) items, $(length(expected)) expected"] + return reduce(vcat, [differences(left, right, "$path[$index]") + for (index, (left, right)) in enumerate(zip(actual, expected))]; init = String[]) + elseif expected isa Number && !(expected isa Bool) + actual isa Number && !(actual isa Bool) && Float64(actual) == Float64(expected) && return String[] + return ["$path: $actual, $expected expected"] + end + isequal(actual, expected) && typeof(actual) == typeof(expected) && return String[] + return ["$path: $(repr(actual)), $(repr(expected)) expected"] +end + +as_json(frame) = JSON.parse(JSON.json(frame_to_dict(frame))) +expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] * ".expected.json"), String)) + + +@testset "encoding3d and MultiScaleTreeGraph" begin + @testset "MultiScaleTreeGraph: $name, exported, written, read and rebuilt as it was" for name in + ("simple_plant.mtg", "palm.mtg", "simple_plant-P1U1.mtg") + mtg = read_mtg(joinpath(MTG_FILES, name)) + variables = Dict("Length" => Variable(unit = "m", standard_name = "length", kind = "extensive")) + data_structure_of_mtg = from_mtg(mtg; id = "plant", variables = variables) + frame = from_bytes(to_bytes(Frame(data_structures = [data_structure_of_mtg]))) + rebuilt = to_mtg(frame.data_structures[1]) + nodes, rebuilt_nodes = traverse(mtg, node -> node), traverse(rebuilt, node -> node) + @test length(nodes) == length(rebuilt_nodes) + for (node, rebuilt_node) in zip(nodes, rebuilt_nodes) + @test node_id(node) == node_id(rebuilt_node) + @test (isroot(node) ? nothing : node_id(parent(node))) == + (isroot(rebuilt_node) ? nothing : node_id(parent(rebuilt_node))) + @test node_mtg(node) == node_mtg(rebuilt_node) + @test [node_id(child) for child in children(node)] == [node_id(child) for child in children(rebuilt_node)] + attributes = Dict(pairs(node_attributes(node))) + rebuilt_attributes = Dict(pairs(node_attributes(rebuilt_node))) + if isroot(node) + for key in (:symbols, :scales) + @test string.(attributes[key]) == string.(rebuilt_attributes[key]) + end + description, rebuilt_description = attributes[:description], rebuilt_attributes[:description] + if description === nothing + @test rebuilt_description === nothing + else + @test names(description) == names(rebuilt_description) + @test all(isequal(getproperty(description, column), getproperty(rebuilt_description, column)) + for column in names(description)) + end + filter!(item -> !(item.first in (:symbols, :scales, :description)), attributes) + filter!(item -> !(item.first in (:symbols, :scales, :description)), rebuilt_attributes) + end + @test isequal(attributes, rebuilt_attributes) + end + end + + @testset "MultiScaleTreeGraph: missing attributes, dates, the order of children, MTG.jl's extras" begin + mtg = read_mtg(joinpath(MTG_FILES, "simple_plant.mtg")) + written = from_mtg(mtg; scales = Dict(3 => "Organ"), ranks = Dict(3 => 5)) + organs = entity_set(written, "Organ") + @test organs.rank == 5 # the vocabulary's position + @test isequal(organs.columns["isAlive"], [missing, false, true, true]) # Internode, Leaf, Internode, Leaf + @test organs.variables["dateDeath"].format == "date" + @test isequal(organs.columns["dateDeath"], [missing, "2022-08-24", missing, missing]) + @test organs.columns["sibling_order"] == [0, 0, 1, 0] # the leaf before the next internode + @test organs.variables["mtg_index"].standard_name == "mtg_index" + @test Set(keys(written.extras["virtualplantlab"])) == Set(["model", "scales", "metadata", "scene"]) + rebuilt = to_mtg(from_bytes(to_bytes(Frame(data_structures = [written]))).data_structures[1]) + @test scale(get_node(rebuilt, 4)) == 3 # its MTG scale, not its rank + @test attribute(get_node(rebuilt, 5), :dateDeath) == Date(2022, 8, 24) + @test !(:isAlive in attribute_names(get_node(rebuilt, 4))) # missing: no attribute + end + + @testset "MultiScaleTreeGraph: the bearer of a branch at another scale" begin + scene = Node(1, MutableNodeMTG(:/, :Scene, 0, 0), Dict{Symbol,Any}()) + plant = Node(2, scene, MutableNodeMTG(:/, :Plant, 1, 1), Dict{Symbol,Any}()) + axis = Node(3, plant, MutableNodeMTG(:/, :Axis, 1, 2), Dict{Symbol,Any}()) + internode = Node(4, axis, MutableNodeMTG(:/, :Internode, 1, 3), Dict{Symbol,Any}(:length => 0.1)) + branch = Node(5, internode, MutableNodeMTG(:+, :Axis, 2, 2), Dict{Symbol,Any}()) # borne by an internode + Node(6, branch, MutableNodeMTG(:/, :Internode, 1, 3), Dict{Symbol,Any}(:length => 0.2)) + written = from_mtg(scene; scales = Dict(1 => "Plant", 2 => "Axis", 3 => "SubOrgan")) + @test only(written.relations).name == "bearer" + @test only(written.relations).columns["source"] == [5] && only(written.relations).columns["target"] == [4] + axes = entity_set(written, "Axis") + @test axes.columns["parent"] == [-1, -1] && axes.columns["complex"] == [2, 2] + rebuilt = to_mtg(from_bytes(to_bytes(Frame(data_structures = [written]))).data_structures[1]) + rebuilt_branch = get_node(rebuilt, 5) + @test node_id(parent(rebuilt_branch)) == 4 && link(rebuilt_branch) == :+ + @test attribute(get_node(rebuilt, 6), :length) == 0.2 + end + + @testset "MultiScaleTreeGraph: ids from 0 and sparse (MTG.jl's: from 1, dense), numbered and written back" begin + plants = DataStructure("plant"; entity_sets = [ + EntitySet("Plant"; columns = Dict("id" => [0]), rank = 1), + EntitySet("Organ"; rank = 5, columns = Dict("id" => [1, 2], "complex" => [0, 0], "parent" => [-1, 1], + "edge_type" => ["none", "branch"], "length" => [1.0, 2.0])), + EntitySet("SubOrgan"; rank = 6, columns = Dict("id" => [2^32 + 5, 2^32 + 9], "complex" => [1, 2], + "parent" => [-1, 2^32 + 5], "edge_type" => ["none", "branch"]))]) + mtg = to_mtg(plants) + nodes = traverse(mtg, node -> node) + @test sort(node_id.(nodes)) == 1:5 # dense, from 1 + @test [attribute(node, :encoding3d_id) for node in nodes] == [0, 1, 2, 2^32 + 9, 2^32 + 5] + lateral = only(node for node in nodes if attribute(node, :encoding3d_id) == 2^32 + 9) + @test attribute(parent(lateral), :encoding3d_id) == 2 && link(lateral) == :/ # under its own organ + @test attribute(lateral, :encoding3d_parent) == 2^32 + 5 && attribute(lateral, :encoding3d_edge_type) == "branch" + again = from_mtg(mtg; scales = Dict(1 => "Plant", 5 => "Organ", 6 => "SubOrgan")) + for name in ("Plant", "Organ", "SubOrgan") # the same rows, in the MTG's order + written, read = entity_set(plants, name), entity_set(again, name) + order = sortperm(read.columns["id"]) + for column in ("id", "complex", "parent", "edge_type", "length") + haskey(written.columns, column) || continue + @test read.columns[column][order] == written.columns[column][sortperm(written.columns["id"])] + end + end + @test !haskey(entity_set(again, "SubOrgan").columns, "encoding3d_id") + end + + @testset "MultiScaleTreeGraph: another platform's graph (the conformance root system) as an MTG" begin + root_system = read_frame(joinpath(CONFORMANCE, "root_system.glb")).data_structures[1] + mtg = to_mtg(root_system) + nodes = traverse(mtg, node -> node) + @test length(nodes) == sum(row_count(set) for set in root_system.entity_sets) + 1 # + the scene + entity_id(node) = attribute(node, :encoding3d_id; default = node_id(node)) # ids 10 to 1000: renumbered + node_of = Dict(entity_id(node) => node for node in nodes if !isroot(node)) + segments = entity_set(root_system, "SubOrgan") + complex_of = Dict(zip(segments.columns["id"], segments.columns["complex"])) + for (id, parent_id) in zip(segments.columns["id"], segments.columns["parent"]) + node = node_of[id] + @test entity_id(parent(node)) == complex_of[id] || entity_id(parent(node)) == parent_id + parent_id == NO_ID && continue + if complex_of[parent_id] == complex_of[id] + @test entity_id(parent(node)) == parent_id && link(node) == :< # along its axis + else + @test attribute(node, :encoding3d_parent) == parent_id && link(node) == :/ # a lateral's first segment + end + end + @test attribute(node_of[segments.columns["id"][1]], :length) == segments.columns["length"][1] + again = from_mtg(mtg; scales = Dict(1 => "Plant", 2 => "Axis", 6 => "SubOrgan")) # and back: the same graph + for set in root_system.entity_sets + read = entity_set(again, set.name) + order, written_order = sortperm(read.columns["id"]), sortperm(set.columns["id"]) + for column in ("id", "complex", "parent", "edge_type", "length") + haskey(set.columns, column) || continue + @test isequal(read.columns[column][order], set.columns[column][written_order]) + end + end + end + +end diff --git a/bindings/julia/encoding3d/test/plantgraphs/Project.toml b/bindings/julia/encoding3d/test/plantgraphs/Project.toml new file mode 100644 index 0000000..d09ebb2 --- /dev/null +++ b/bindings/julia/encoding3d/test/plantgraphs/Project.toml @@ -0,0 +1,12 @@ +[deps] +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" +PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" +PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +PlantGeomPrimitives = "1" +PlantGeomTurtle = "1" +PlantGraphs = "1" +julia = "1.12" diff --git a/bindings/julia/encoding3d/test/plantgraphs/runtests.jl b/bindings/julia/encoding3d/test/plantgraphs/runtests.jl new file mode 100644 index 0000000..b2514b3 --- /dev/null +++ b/bindings/julia/encoding3d/test/plantgraphs/runtests.jl @@ -0,0 +1,80 @@ +# encoding3d.jl's VirtualPlantLab extensions (PlantGraphs, PlantGeomTurtle; Julia 1.12 or later, as PlantGraphs): a +# proposal for VirtualPlantLab, tested by hand in its own project, so that encoding3d's tests and CI depend on no platform: +# +# julia --project=bindings/julia/encoding3d/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.instantiate()' +# julia --project=bindings/julia/encoding3d/test/plantgraphs bindings/julia/encoding3d/test/plantgraphs/runtests.jl +using encoding3d +using PlantGeomPrimitives +using PlantGeomTurtle +using PlantGraphs +using Test + +# A small VPL plant: internodes as cylinders, leaves as ellipses, the turtle's rotations between them +struct Internode <: PlantGraphs.Node + length::Float64 + width::Float64 +end +struct Leaf <: PlantGraphs.Node + length::Float64 + width::Float64 + age::Int +end +struct Meristem <: PlantGraphs.Node end +struct Weather + temperature::Float64 +end + +PlantGeomTurtle.feed!(turtle::Turtle, node::Internode, data) = + HollowCylinder!(turtle; length = node.length, width = node.width, height = node.width, move = true) +PlantGeomTurtle.feed!(turtle::Turtle, node::Leaf, data) = + Ellipse!(turtle; length = node.length, width = node.width, move = false) + +plant() = Graph(axiom = Internode(0.1, 0.01) + (RU(30.0) + Leaf(0.05, 0.02, 3), RU(-30.0) + Leaf(0.04, 0.02, 1)) + + Internode(0.08, 0.008) + Meristem(), + data = Weather(20.5)) + +@testset "encoding3d and VirtualPlantLab" begin + @testset "A graph as a data structure, written, read and rebuilt" begin + graph = plant() + written = from_graph(graph; id = "plant", + variables = Dict("length" => Variable(unit = "m", standard_name = "length", kind = "extensive"))) + organs = entity_set(written, "Organ") + @test organs.columns["label"] == ["Internode", "Leaf", "Leaf", "Internode", "Meristem"] # no turtle moves + ids = organs.columns["id"] + @test organs.columns["parent"] == [-1, ids[1], ids[1], ids[1], ids[4]] # the leaves on the first internode + @test organs.columns["edge_type"] == ["none", "unknown", "unknown", "unknown", "unknown"] # none stored + @test organs.columns["sibling_order"] == [0, 0, 1, 2, 0] # the leaves, then the internode + @test isequal(organs.columns["length"], [0.1, 0.05, 0.04, 0.08, NaN]) # NaN: the meristem has none + @test isequal(organs.columns["age"], [missing, 3, 1, missing, missing]) # missing where no field + @test written.extras["virtualplantlab"]["model"] == "PlantGraphs" + @test written.scalars.columns["temperature"] == [20.5] + + frame = from_bytes(to_bytes(Frame(data_structures = [written]))) + read_back = frame.data_structures[1] + static, graph_ids = to_graph(read_back; types = Dict("Internode" => Internode, "Leaf" => Leaf, + "Meristem" => Meristem)) + again = from_graph(static; keep = node -> true) + again_organs = entity_set(again, "Organ") + @test again_organs.columns["label"] == organs.columns["label"] + @test isequal(again_organs.columns["length"], organs.columns["length"]) + @test isequal(again_organs.columns["age"], organs.columns["age"]) + @test again_organs.columns["edge_type"] == organs.columns["edge_type"] + @test again_organs.columns["parent"] == [-1; [graph_ids[parent] for parent in organs.columns["parent"][2:end]]] + end + + @testset "Its turtle geometry, each triangle with its node" begin + graph = plant() + written = from_graph(graph) + mesh = add_geometry!(written, graph) + reference = PlantGeomPrimitives.Mesh(graph) # VPL's own mesh (both packages export a Mesh) + @test size(mesh.positions, 1) == length(vertices(reference)) + @test all(mesh.positions[index, :] == collect(vertices(reference)[index]) for index in 1:length(vertices(reference))) + rows = mesh.feature_rows["Organ"] + labels = entity_set(written, "Organ").columns["label"] + @test all(row != NO_ID for row in rows) + @test Set(labels[row + 1] for row in rows) == Set(["Internode", "Leaf"]) # meristems have no geometry + path = joinpath(mktempdir(), "vpl_plant.glb") + write_frame(path, Frame(data_structures = [written])) + @test read_frame(path).data_structures[1].meshes[1].feature_rows == mesh.feature_rows + end +end diff --git a/bindings/julia/encoding3d/test/runtests.jl b/bindings/julia/encoding3d/test/runtests.jl new file mode 100644 index 0000000..0cc37eb --- /dev/null +++ b/bindings/julia/encoding3d/test/runtests.jl @@ -0,0 +1,167 @@ +using Dates +using encoding3d +using JSON +using Test + +const CONFORMANCE = normpath(joinpath(@__DIR__, "..", "..", "..", "..", "conformance")) + +conformance_files() = sort([relpath(joinpath(folder, name), CONFORMANCE) + for (folder, _, files) in walkdir(CONFORMANCE) for name in files + if endswith(name, ".glb") && !startswith(relpath(folder, CONFORMANCE), "coupling")]) + +"""The differences between two JSON values, as `encoding3d compare` finds them: objects by key, arrays in order, +numbers exactly (1 and 1.0 equal).""" +function differences(actual, expected, path = "\$") + if expected isa AbstractDict + actual isa AbstractDict || return ["$path: an object expected"] + found = ["$path: missing key '$key'" for key in keys(expected) if !haskey(actual, key)] + append!(found, ["$path: unexpected key '$key'" for key in keys(actual) if !haskey(expected, key)]) + for key in keys(expected) + haskey(actual, key) && append!(found, differences(actual[key], expected[key], "$path.$key")) + end + return found + elseif expected isa AbstractVector + actual isa AbstractVector || return ["$path: an array expected"] + length(actual) == length(expected) || return ["$path: $(length(actual)) items, $(length(expected)) expected"] + return reduce(vcat, [differences(left, right, "$path[$index]") + for (index, (left, right)) in enumerate(zip(actual, expected))]; init = String[]) + elseif expected isa Number && !(expected isa Bool) + actual isa Number && !(actual isa Bool) && Float64(actual) == Float64(expected) && return String[] + return ["$path: $actual, $expected expected"] + end + isequal(actual, expected) && typeof(actual) == typeof(expected) && return String[] + return ["$path: $(repr(actual)), $(repr(expected)) expected"] +end + +as_json(frame) = JSON.parse(JSON.json(frame_to_dict(frame))) +expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] * ".expected.json"), String)) + +@testset "encoding3d" begin + @testset "GLB container: the conformance files' JSON and binary chunks repacked as they were" begin + for relative in conformance_files() + data = read(joinpath(CONFORMANCE, relative)) + document, binary = encoding3d.unpack(data) + again, binary_again = encoding3d.unpack(encoding3d.pack(document, binary)) + @test binary_again == binary + @test isempty(differences(again, document)) + @test length(binary) % 8 == 0 + end + @test_throws ArgumentError encoding3d.unpack(UInt8[1, 2, 3]) + end + + @testset "Reading: every conformance file as its expected values ($relative)" for relative in conformance_files() + frame = read_frame(joinpath(CONFORMANCE, relative)) + @test isempty(differences(as_json(frame), expected_of(relative))) + end + + @testset "Writing: what is written reads back as the expected values ($relative)" for relative in conformance_files() + frame = from_bytes(to_bytes(read_frame(joinpath(CONFORMANCE, relative)))) + @test isempty(differences(as_json(frame), expected_of(relative))) + end + + @testset "Columns of every type, NaN, infinities, vectors, text" begin + columns = encoding3d.Columns("id" => Int32[0, 1, 2], "label" => ["Leaf", "Internode", "Leaf"], + "parent" => [-1, 0, 1], "edge_type" => ["none", "successor", "branch"], + "with_nan" => [1.0, NaN, -2.5], "with_infinities" => Float32[Inf, 1, -Inf], + "flag" => [true, false, true], "name" => ["é", "", "a b"], + "position" => [1.0 2.0 3.0; 4.0 5.0 6.0; 7.0 8.0 9.0]) + for T in encoding3d.COLUMN_TYPES + columns["column_$(lowercase(string(T)))"] = T[1, 2, 3] + end + frame = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = columns, rank = 1, + variables = Dict("with_nan" => Variable(unit = "m", standard_name = "length", kind = "extensive", + description = "a length", minimum = 0, + references = ["a reference"])))], + origin = [1.0 0 0 0.5; 0 1 0 0; 0 0 1 -0.1; 0 0 0 1], extras = Dict{String,Any}("platform" => [1, 2]))], + time = 2.5, time_unit = "h", step = 3) + read_back = from_bytes(to_bytes(frame)) + organ = entity_set(data_structure(read_back, "plants"), "Organ") + @test eltype(organ.columns["id"]) == Int64 # ids are Int64 + @test isequal(organ.columns["with_nan"], [1.0, NaN, -2.5]) + @test organ.columns["with_infinities"] == Float32[Inf, 1, -Inf] + @test organ.columns["flag"] == [true, false, true] + @test organ.columns["name"] == ["é", "", "a b"] + @test organ.columns["position"] == [1.0 2.0 3.0; 4.0 5.0 6.0; 7.0 8.0 9.0] + @test organ.columns["label"] == ["Leaf", "Internode", "Leaf"] + for T in encoding3d.COLUMN_TYPES + @test organ.columns["column_$(lowercase(string(T)))"] == T[1, 2, 3] + @test eltype(organ.columns["column_$(lowercase(string(T)))"]) == T + end + @test organ.variables["with_nan"] == frame.data_structures[1].entity_sets[1].variables["with_nan"] + @test data_structure(read_back, "plants").origin == frame.data_structures[1].origin + @test data_structure(read_back, "plants").extras["platform"] == [1, 2] + @test (read_back.time, read_back.time_unit, read_back.step) == (2.5, "h", 3) + end + + @testset "Meshes: positions, feature rows (none included), indices, display copies" begin + set = EntitySet("SubOrgan"; columns = Dict("id" => [10, 11]), rank = 1) + mesh = Mesh("roots"; positions = [0.0 0 0; 1 0 0; 0 1 0; 0 0 1], feature_rows = Dict("SubOrgan" => [0, 1, 1, -1]), + indices = [0, 1, 2, 0, 2, 3], display = Dict("radius" => Float32[1, 2, 2, 0]), + display_sources = Dict("radius" => "SubOrgan")) + frame = Frame(data_structures = [DataStructure("roots"; entity_sets = [set], meshes = [mesh])]) + read_back = from_bytes(to_bytes(frame)).data_structures[1].meshes[1] + @test read_back.positions == mesh.positions + @test read_back.feature_rows == mesh.feature_rows + @test read_back.indices == mesh.indices + @test read_back.display == mesh.display && read_back.display_sources == mesh.display_sources + end + + @testset "What the specification forbids is refused" begin + bad_column = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0], "1st" => [1.0]))])]) + @test_throws ArgumentError to_bytes(bad_column) + @test_throws ArgumentError to_bytes(Frame(data_structures = [DataStructure("bad id")])) + bad_edge = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0], "parent" => [-1], "edge_type" => ["root"]))])]) + @test_throws ArgumentError to_bytes(bad_edge) + both = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0, 1], "x" => [NaN, -floatmax(Float64)]))])]) + @test_throws ArgumentError to_bytes(both) + @test_throws ArgumentError EntitySet("Organ"; columns = Dict("id" => [0, 1], "x" => [1.0])) + end + + @testset "Sequences: frames and their index" begin + folder = mktempdir() + frames = [Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Plant"; columns = Dict("id" => [0], "biomass" => [Float64(step)]), rank = 1)])], + time = step / 24, time_unit = "d", step = step) for step in 0:2] + index = write_sequence(folder, frames) + entries = read_index(index)["frames"] + @test [entry["step"] for entry in entries] == [0, 1, 2] + @test [frame.data_structures[1].entity_sets[1].columns["biomass"][1] for frame in read_sequence(index)] == + [0.0, 1.0, 2.0] + read_back = read_index(joinpath(CONFORMANCE, "growing_sequence", "scene.fspm.json")) + @test all(isfile(entry["path"]) for entry in read_back["frames"]) + end + + @testset "coupling" begin + # a partial frame: its sender and removed entities kept + frame = Frame(data_structures = [DataStructure("plant"; entity_sets = [ + EntitySet("SubOrgan"; rank = 6, columns = encoding3d.Columns( + "id" => Int64[10, 11], "complex" => Int64[3, 3], "parent" => Int64[NO_ID, 10], + "edge_type" => ["none", "successor"], "sugar" => [1.0, 2.0]))])], + update = Update("vpl"; removed = Dict(("plant", "SubOrgan") => Int64[5]))) + again = from_bytes(to_bytes(frame)) + @test again.update.participant == "vpl" + @test again.update.removed[("plant", "SubOrgan")] == [5] + @test entity_set(data_structure(again, "plant"), "SubOrgan").columns["complex"] == [3, 3] + # a declaration as JSON: an entity set followed for each variable's + declaration = Declaration("function"; entity_sets = [EntitySetSpec("plant", "SubOrgan"; rights = "refines")], + variables = [VariableSpec("plant", "Organ", "length", "requires"; unit = "m", + standard_name = "length")]) + value = encoding3d.declaration_json(declaration) + @test [item["name"] for item in value["entitySets"]] == ["SubOrgan", "Organ"] + @test value["entitySets"][1]["rank"] == 6 && value["entitySets"][2]["rights"] == "follows" + @test value["variables"][1]["standardName"] == "length" + # the folder transport: a participant's queues, a message and its payload + folder = mktempdir() + participant = encoding3d.FolderChannel(folder, "function") + encoding3d.send_message(participant, Dict("type" => "update", "step" => 0), to_bytes(frame)) + coordinator = encoding3d.FolderChannel(joinpath(folder, "function", "to_coordinator"), mktempdir(), 0, 0) + header, payload = encoding3d.receive_message(coordinator, 5) + @test header["type"] == "update" && from_bytes(payload).update.participant == "vpl" + @test isfile(joinpath(folder, "function", "to_coordinator", "00000000.glb")) + @test_throws ErrorException encoding3d.receive_message(coordinator, 0.05) # nothing more: a timeout + end +end diff --git a/bindings/python/conformance_run.py b/bindings/python/conformance_run.py index 16e4904..0ccd22f 100644 --- a/bindings/python/conformance_run.py +++ b/bindings/python/conformance_run.py @@ -10,24 +10,26 @@ import shutil import sys -from openalea import fspm_tf -from openalea.fspm_tf.dump import frame_to_dict +from openalea import encoding3d +from openalea.encoding3d.dump import frame_to_dict def main(conformance: str, out: str) -> int: for folder, _, files in os.walk(conformance): + if os.path.relpath(folder, conformance).split(os.sep)[0] == "coupling": + continue # the coupling sessions: their own check for name in sorted(files): if not name.endswith(".glb"): continue relative = os.path.relpath(os.path.join(folder, name), conformance) - frame = fspm_tf.read(os.path.join(conformance, relative)) + frame = encoding3d.read(os.path.join(conformance, relative)) dump = os.path.join(out, "dumps", relative[:-len(".glb")] + ".json") written = os.path.join(out, "written", relative) for path in (dump, written): os.makedirs(os.path.dirname(path), exist_ok=True) with open(dump, "w", encoding="utf-8") as file: json.dump(frame_to_dict(frame), file, indent=1) - fspm_tf.write(written, frame) # written again from what was read + encoding3d.write(written, frame) # written again from what was read index = os.path.join(conformance, "growing_sequence", "scene.fspm.json") if os.path.exists(index): shutil.copy(index, os.path.join(out, "written", "growing_sequence", "scene.fspm.json")) diff --git a/bindings/python/src/openalea/encoding3d/__init__.py b/bindings/python/src/openalea/encoding3d/__init__.py new file mode 100644 index 0000000..7b111c7 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/__init__.py @@ -0,0 +1,15 @@ +""" +encoding3d: an exchange format for whole FSPM scenes. This package is its Python reference implementation: the codec +(openalea.encoding3d.codec), the validator (openalea.encoding3d.validate), the display copies (openalea.encoding3d.display) and the vocabulary +(openalea.encoding3d.vocabulary). See the specification in the repository's spec/ folder. +""" +SPECIFICATION_VERSION = "0.3" + +from openalea.encoding3d.model import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, # noqa: E402 + Table, Update, Variable) +from openalea.encoding3d.codec import (from_bytes, read, read_index, read_sequence, to_bytes, write, # noqa: E402 + write_index, write_sequence) + +__all__ = ["SPECIFICATION_VERSION", "DataStructure", "EntitySet", "Frame", "Grid", "Mapping", "Mesh", "NO_ID", + "Relation", "Table", "Update", "Variable", "from_bytes", "read", "read_index", "read_sequence", "to_bytes", "write", + "write_index", "write_sequence"] diff --git a/bindings/python/src/openalea/fspm_tf/_spec.py b/bindings/python/src/openalea/encoding3d/_spec.py similarity index 58% rename from bindings/python/src/openalea/fspm_tf/_spec.py rename to bindings/python/src/openalea/encoding3d/_spec.py index b27f8dd..e6caa2e 100644 --- a/bindings/python/src/openalea/fspm_tf/_spec.py +++ b/bindings/python/src/openalea/encoding3d/_spec.py @@ -1,6 +1,6 @@ """ -Where the specification's files are: the spec/ folder of the fspm-TF repository, shipped with the installed package as -openalea.fspm_tf.spec (pyproject's package-dir), or found in a clone from bindings/python/src (e.g. ParaView's plugin, +Where the specification's files are: the spec/ folder of the encoding3d repository, shipped with the installed package as +openalea.encoding3d.spec (pyproject's package-dir), or found in a clone from bindings/python/src (e.g. ParaView's plugin, which puts only that folder on its path). """ import os @@ -9,9 +9,9 @@ def path(*parts: str) -> str: """The path of a file of the specification, e.g. path("schema", "index.schema.json").""" try: - from openalea.fspm_tf import spec + from openalea.encoding3d import spec base = list(spec.__path__)[0] except ImportError: - here = os.path.dirname(os.path.abspath(__file__)) # bindings/python/src/openalea/fspm_tf + here = os.path.dirname(os.path.abspath(__file__)) # bindings/python/src/openalea/encoding3d base = os.path.join(here, os.pardir, os.pardir, os.pardir, os.pardir, os.pardir, "spec") return os.path.join(os.path.normpath(base), *parts) diff --git a/bindings/python/src/openalea/encoding3d/cli.py b/bindings/python/src/openalea/encoding3d/cli.py new file mode 100644 index 0000000..7107d21 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/cli.py @@ -0,0 +1,207 @@ +""" +The encoding3d command line:: + + encoding3d validate FILE... check .glb frames or sequences (scene.fspm.json); exit 1 on errors + encoding3d info FILE what a frame holds + encoding3d display FILE NAME... [-o OUT] add display copies of variables (in place, or to OUT) + encoding3d display FILE --remove [-o OUT] remove them + encoding3d compare ACTUAL EXPECTED a codec's JSON dump (or a .glb) against expected values; exit 1 if they + differ + encoding3d compare-all DUMPS CONFORMANCE every conformance file's expected values against the dump at the same + path (x.expected.json: DUMPS/x.json); exit 1 if any differ + encoding3d couple --steps N --interval DT [--participant NAME=COMMAND]... [--expect NAME]... + [--replay NAME=INDEX]... [--links LINKS.json] [--stages A,B;C] [--endpoint E] [--record DIR] + run a coupling session (specification §11): launch the participants' + commands, wait for the expected ones, replay sequences; exit 1 if it + stops on an error + encoding3d couple-conformance SESSIONS --participant COMMAND [--out DIR] [--transport zmq|files] + run the coupling conformance sessions (conformance/coupling) with + COMMAND SESSION OUT as the participant under test; exit 1 if what it + read, or the session's record, differs from the expected values +""" +import argparse +import sys + +from openalea.encoding3d import read, write +from openalea.encoding3d.display import add_display, remove_display +from openalea.encoding3d.validate import validate_file + + +def _validate(arguments) -> int: + failed = False + for path in arguments.files: + report = validate_file(path) + print(report) + failed = failed or not report.ok + return 1 if failed else 0 + + +def _info(arguments) -> int: + frame = read(arguments.file) + print(f"{arguments.file}: time {frame.time} {frame.time_unit}, step {frame.step}, writer {frame.writer or '-'}") + for data_structure in frame.data_structures: + print(f" {data_structure.id} ({data_structure.kind}{', ' + data_structure.model if data_structure.model else ''})") + for entity_set in data_structure.entity_sets: + rank = "" if entity_set.rank is None else f" rank {entity_set.rank}" + print(f" {entity_set.name}{rank}: {entity_set.count} entities, variables " + f"{', '.join(entity_set.variable_names()) or '-'}") + for relation in data_structure.relations: + print(f" relation {relation.name} ({relation.source_set} -> {relation.target_set}): {relation.count}") + if data_structure.scalars is not None: + print(f" scalars: {', '.join(data_structure.scalars.columns)}") + for mesh in data_structure.meshes: + print(f" mesh {mesh.name}: {len(mesh.positions)} vertices, entity sets {', '.join(mesh.feature_rows)}" + f"{', display ' + ', '.join(mesh.display) if mesh.display else ''}") + for mapping in frame.mappings: + print(f" mapping {mapping.name}: {mapping.source[0]}/{mapping.source[1]} -> " + f"{mapping.target[0]}/{mapping.target[1]}, {mapping.count} pairs") + return 0 + + +def _display(arguments) -> int: + frame = read(arguments.file) + if arguments.remove: + remove_display(frame, arguments.names or None) + else: + if not arguments.names: + print("give the names of the variables to copy, or --remove", file=sys.stderr) + return 2 + add_display(frame, arguments.names, entity_set=arguments.entity_set) + write(arguments.output or arguments.file, frame) + return 0 + + +def _print_differences(name: str, differences: list, shown: int = 20) -> None: + print(f"{name}: {'equal' if not differences else f'{len(differences)} difference(s)'}") + for line in differences[:shown]: + print(f" {line}") + if len(differences) > shown: + print(f" ... {len(differences) - shown} more") + + +def _compare(arguments) -> int: + from openalea.encoding3d.compare import compare_files + differences = compare_files(arguments.actual, arguments.expected) + _print_differences(arguments.actual, differences) + return 1 if differences else 0 + + +def _compare_all(arguments) -> int: + from openalea.encoding3d.compare import compare_all + results = compare_all(arguments.dumps, arguments.conformance) + for name, differences in results.items(): + _print_differences(name, differences) + failed = sum(1 for differences in results.values() if differences) + print(f"{len(results) - failed} of {len(results)} conformance files read as expected") + return 1 if failed or not results else 0 + + +def _pairs(values, what: str) -> dict: + pairs = {} + for value in values or []: + name, separator, rest = value.partition("=") + if not separator or not name or not rest: + raise SystemExit(f"--{what} {value}: NAME=VALUE expected") + pairs[name] = rest + return pairs + + +def _couple(arguments) -> int: + import json + from openalea.encoding3d.coupling import Coordinator, CouplingError + links = [] + if arguments.links: + with open(arguments.links, encoding="utf-8") as file: + links = json.load(file) + stages = None if not arguments.stages else [stage.split(",") for stage in arguments.stages.split(";")] + coordinator = Coordinator(arguments.endpoint, steps=arguments.steps, interval=arguments.interval, + time_unit=arguments.time_unit, start=arguments.start, links=links, stages=stages, + expect=arguments.expect or [], replays=_pairs(arguments.replay, "replay"), + record=arguments.record, timeout=arguments.timeout, + keep_messages=arguments.keep_messages) + print(f"coordinator at {coordinator.endpoint}", flush=True) + for name, command in _pairs(arguments.participant, "participant").items(): + coordinator.launch(name, command) + try: + coordinator.run() + except CouplingError as error: + print(f"the session stopped: {error}", file=sys.stderr) + return 1 + for coupling in coordinator.couplings: + source = "-" if coupling.source is None else f"{coupling.provider}.{coupling.source.name}" + print(f" {coupling.receiver}.{coupling.variable.name} <- {source} ({coupling.how}" + f"{', ' + coupling.aggregation if coupling.aggregation else ''}, factor {coupling.factor:g})") + print(f"{arguments.steps} steps, stages {coordinator.stages}" + + (f", recorded in {arguments.record}" if arguments.record else "")) + return 0 + + +def _couple_conformance(arguments) -> int: + import tempfile + from openalea.encoding3d.coupling.conformance import TRANSPORTS, run_all + out = arguments.out or tempfile.mkdtemp(prefix="encoding3d_sessions_") + transports = [arguments.transport] if arguments.transport else list(TRANSPORTS) + results = run_all(arguments.sessions, arguments.participant, out, transports) + for (name, transport), differences in results.items(): + _print_differences(f"{name} ({transport})", differences) + failed = sum(1 for differences in results.values() if differences) + print(f"{len(results) - failed} of {len(results)} sessions as expected (outputs in {out})") + return 1 if failed or not results else 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(prog="encoding3d", description="encoding3d files: validate, inspect, display; couple platforms") + commands = parser.add_subparsers(dest="command", required=True) + validate = commands.add_parser("validate", help="check frames (.glb) or sequences (scene.fspm.json)") + validate.add_argument("files", nargs="+") + validate.set_defaults(run=_validate) + info = commands.add_parser("info", help="what a frame holds") + info.add_argument("file") + info.set_defaults(run=_info) + display = commands.add_parser("display", help="add or remove display copies of variables") + display.add_argument("file") + display.add_argument("names", nargs="*") + display.add_argument("--entity-set", default=None, help="the entity set the variables come from") + display.add_argument("--remove", action="store_true", help="remove display copies (all, or the names given)") + display.add_argument("-o", "--output", default=None, help="write to this file instead of in place") + display.set_defaults(run=_display) + compare = commands.add_parser("compare", help="a codec's JSON dump (or a .glb) against expected values") + compare.add_argument("actual") + compare.add_argument("expected") + compare.set_defaults(run=_compare) + compare_all = commands.add_parser("compare-all", help="dumps of every conformance file against their expected " + "values") + compare_all.add_argument("dumps", help="the folder of the codec's dumps (x.json for x.expected.json)") + compare_all.add_argument("conformance", help="encoding3d's conformance folder") + compare_all.set_defaults(run=_compare_all) + couple = commands.add_parser("couple", help="run a coupling session between platforms") + couple.add_argument("--steps", type=int, required=True) + couple.add_argument("--interval", type=float, required=True, help="the exchange interval") + couple.add_argument("--time-unit", default="d") + couple.add_argument("--start", type=float, default=0.) + couple.add_argument("--participant", action="append", help="NAME=COMMAND: a participant to launch") + couple.add_argument("--expect", action="append", help="NAME: a participant started separately") + couple.add_argument("--replay", action="append", help="NAME=INDEX: a recorded sequence as a participant") + couple.add_argument("--links", help="a JSON file: the session's links") + couple.add_argument("--stages", help="the stages, e.g. 'growth;function,light' (default: owners, refiners, " + "others)") + couple.add_argument("--endpoint", default="tcp://127.0.0.1:*", help="a ZeroMQ endpoint, or files:FOLDER") + couple.add_argument("--record", help="a folder: the session recorded as a sequence") + couple.add_argument("--timeout", type=float, default=600., help="seconds to wait for a message") + couple.add_argument("--keep-messages", action="store_true", help="over a folder, keep every message (else each " + "is deleted once read)") + couple.set_defaults(run=_couple) + sessions = commands.add_parser("couple-conformance", help="a participant's side against the coupling " + "conformance sessions") + sessions.add_argument("sessions", help="the sessions' folder (conformance/coupling)") + sessions.add_argument("--participant", required=True, help="the command of the participant under test; it is " + "run as COMMAND SESSION OUT") + sessions.add_argument("--out", help="where the outputs go (default: a new temporary folder)") + sessions.add_argument("--transport", choices=["zmq", "files"], help="one transport only (default: both)") + sessions.set_defaults(run=_couple_conformance) + arguments = parser.parse_args(argv) + return arguments.run(arguments) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/encoding3d/codec.py similarity index 68% rename from bindings/python/src/openalea/fspm_tf/codec.py rename to bindings/python/src/openalea/encoding3d/codec.py index 84ef88b..adde183 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/encoding3d/codec.py @@ -1,5 +1,5 @@ """ -Reading and writing fspm-TF frames as GLB files, and sequences of frames with their index:: +Reading and writing encoding3d frames as GLB files, and sequences of frames with their index:: write(path, frame) a Frame to a .glb file to_bytes(frame) -> bytes read(path) -> Frame a .glb file to a Frame from_bytes(data) -> Frame @@ -10,16 +10,18 @@ IDs, and the FSPM_scene root extension holds what these two cannot: data structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, standard names, kinds and roles. """ +import itertools import json import os import re import numpy as np -from openalea.fspm_tf import SPECIFICATION_VERSION -from openalea.fspm_tf.glb import BufferBuilder, pack, unpack, view_bytes -from openalea.fspm_tf.model import (DataStructure, EDGE_TYPES, ENUM_COLUMNS, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, - Relation, Table, Variable) +from openalea.encoding3d import SPECIFICATION_VERSION +from openalea.encoding3d.glb import BufferBuilder, pack, unpack, view_bytes +from openalea.encoding3d.model import (DataStructure, EDGE_TYPES, ENUM_COLUMNS, EntitySet, Frame, Grid, ID_COLUMNS, + LazyColumns, LazyList, Mapping, Mesh, NO_ID, Relation, Table, Update, Variable, + _as_column) EXTENSION = "FSPM_scene" METADATA = "EXT_structural_metadata" @@ -56,7 +58,7 @@ def to_bytes(frame: Frame) -> bytes: EDGE_TYPE_ENUM: {"valueType": "UINT8", "values": [{"name": name, "value": value} for value, name in enumerate(EDGE_TYPES)]}}} tables, variables = [], {} - document = {"asset": {"version": "2.0", "generator": f"fspm-tf {SPECIFICATION_VERSION}"}, + document = {"asset": {"version": "2.0", "generator": f"encoding3d {SPECIFICATION_VERSION}"}, "scene": 0, "scenes": [{"nodes": [0]}], "nodes": [{"name": "fspm_scene", "matrix": Z_UP_TO_Y_UP, "children": []}]} accessors, meshes = [], [] @@ -80,10 +82,13 @@ def add_table(table: Table, class_id: str) -> int: if getattr(data_structure, key) is not None: entry[key] = getattr(data_structure, key) entry["origin"] = data_structure.origin.T.reshape(-1).tolist() # column-major, as glTF matrices - entry["entitySets"] = [ - {"name": entity_set.name, "rank": entity_set.rank, - "propertyTable": add_table(entity_set, f"{data_structure.id}__{entity_set.name}")} - for entity_set in data_structure.entity_sets] + entry["entitySets"] = [] + for entity_set in data_structure.entity_sets: + set_entry = {"name": entity_set.name, "rank": entity_set.rank} + if entity_set.scale is not None: + set_entry["scale"] = entity_set.scale + set_entry["propertyTable"] = add_table(entity_set, f"{data_structure.id}__{entity_set.name}") + entry["entitySets"].append(set_entry) if data_structure.relations: entry["relations"] = [ {"name": relation.name, "sourceSet": relation.source_set, "targetSet": relation.target_set, @@ -93,6 +98,8 @@ def add_table(table: Table, class_id: str) -> int: entry["scalars"] = add_table(data_structure.scalars, f"{data_structure.id}__scalars") if data_structure.grid is not None: entry["grid"] = _grid_json(data_structure.grid) + if data_structure.labels: + entry["labels"] = dict(data_structure.labels) if data_structure.extras: entry["extras"] = data_structure.extras node = {"name": data_structure.id} @@ -124,12 +131,25 @@ def add_table(table: Table, class_id: str) -> int: **({"method": mapping.method} if mapping.method else {}), "propertyTable": add_table(mapping, f"mapping__{mapping.name}")}) + update = None + if frame.update is not None: + _check_identifier(frame.update.participant, "participant") + update = {"participant": frame.update.participant} + if frame.update.removed: + update["removed"] = [ + {"dataStructure": data_structure_id, "entitySet": set_name, + "propertyTable": add_table(Table(f"{set_name}__removed", columns={"id": ids}), + f"{data_structure_id}__{set_name}__removed")} + for (data_structure_id, set_name), ids in frame.update.removed.items()] + scene = {"version": SPECIFICATION_VERSION, "frame": {"time": float(frame.time), "timeUnit": frame.time_unit, "step": int(frame.step)}} if frame.vocabulary_version: scene["vocabulary"] = frame.vocabulary_version if frame.writer: scene["writer"] = dict(frame.writer) + if update is not None: + scene["update"] = update scene["dataStructures"] = data_structures if mappings: scene["mappings"] = mappings @@ -147,6 +167,8 @@ def add_table(table: Table, class_id: str) -> int: document["meshes"], document["accessors"] = meshes, accessors binary = buffers.binary() if buffers.views: + # the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + binary += b"\0" * (-len(binary) % 8) document["buffers"] = [{"byteLength": len(binary)}] document["bufferViews"] = buffers.views return pack(document, binary) @@ -172,6 +194,8 @@ def _declaration_json(variable: Variable) -> dict: for key in ("references", "doi"): if getattr(variable, key): declaration[key] = list(getattr(variable, key)) + if variable.format is not None: + declaration["format"] = variable.format return declaration @@ -190,6 +214,10 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d values[name] = {"values": buffers.add(codes)} elif column.dtype == object: prop["type"] = "STRING" + if any(value is None for value in column): # missing text: noData, a string the column lacks + no_data = _free_string(column) + prop["noData"] = no_data + column = np.array([no_data if value is None else value for value in column], dtype=object) encoded = [value.encode("utf-8") for value in column] offsets = np.concatenate([[0], np.cumsum([len(value) for value in encoded])]) offset_type = "UINT32" if offsets[-1] < 2 ** 32 else "UINT64" @@ -199,8 +227,15 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d elif column.dtype == bool: if column.ndim != 1: raise ValueError(f"column '{name}' of '{table.name}': boolean vectors are not supported") - prop["type"] = "BOOLEAN" - values[name] = {"values": buffers.add(np.packbits(column, bitorder="little"))} + if np.ma.isMaskedArray(column): # booleans with missing values: UINT8 0, 1, 255 + prop.update(type="SCALAR", componentType="UINT8", noData=BOOLEAN_NO_DATA, + extras={"FSPM_boolean": True}) + stored = np.where(np.ma.getmaskarray(column), BOOLEAN_NO_DATA, + np.asarray(column.data, dtype=np.uint8)).astype(np.uint8) + values[name] = {"values": buffers.add(stored)} + else: + prop["type"] = "BOOLEAN" + values[name] = {"values": buffers.add(np.packbits(column, bitorder="little"))} else: if column.dtype not in COMPONENT_TYPES: raise TypeError(f"column '{name}' of '{table.name}': dtype {column.dtype} is not supported") @@ -229,26 +264,61 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d data = np.where(np.isnan(column), no_data, column).astype(column.dtype) elif name in ("complex", "parent") and name in table.reserved: prop["noData"] = NO_ID - values[name] = {"values": buffers.add(data)} + elif np.ma.isMaskedArray(column): # missing integers: noData, a value the column lacks + no_data = _free_integer(column) + prop["noData"] = no_data + data = np.where(np.ma.getmaskarray(column), no_data, np.asarray(column.data)).astype(column.dtype) + values[name] = {"values": buffers.add(np.asarray(data))} if name == "id" and name in table.reserved and table.table_kind == "entity_set": prop["required"] = True properties[name] = prop return properties, values +BOOLEAN_NO_DATA = 255 # a boolean column with missing values: UINT8 0, 1, and 255 missing + + +def _free_integer(column: np.ndarray) -> int: + """A value of the column's integer type that it does not hold: its minimum (signed) or maximum (unsigned) first.""" + info = np.iinfo(column.dtype) + present = set(np.asarray(column.compressed() if np.ma.isMaskedArray(column) else column).reshape(-1).tolist()) + first, second = (info.min, info.max) if info.min < 0 else (info.max, info.min) + for candidate in (first, second): + if candidate not in present: + return int(candidate) + step = 1 if info.min < 0 else -1 # then inwards from the first: at most len(present) tries + for candidate in itertools.count(first + step, step): + if candidate not in present: + return int(candidate) + + +def _free_string(column: np.ndarray) -> str: + """A string the column does not hold: the empty string first, then NUL characters.""" + present = {value for value in column if value is not None} + candidate = "" + while candidate in present: + candidate += "\0" + return candidate + + def _enum_codes(name: str, column: np.ndarray, class_id: str, schema: dict) -> tuple: + values = np.asarray(column, dtype=object).tolist() + distinct = set(values) # hashed: a few names, however many rows + if None in distinct: + raise ValueError(f"{name} of '{class_id}': an enum column has no missing values (give each row a {name})") + names = sorted(distinct) + code_of = {label: code for code, label in enumerate(names)} + codes = np.fromiter(map(code_of.__getitem__, values), dtype=np.int64, count=len(values)) if name == "edge_type": - unknown = set(column) - set(EDGE_TYPES) + unknown = set(names) - set(EDGE_TYPES) if unknown: raise ValueError(f"edge_type of '{class_id}': unknown values {sorted(unknown)} (allowed: {EDGE_TYPES})") - return EDGE_TYPE_ENUM, np.array([EDGE_TYPES.index(value) for value in column], dtype=np.uint8) - names = sorted(set(column)) + return EDGE_TYPE_ENUM, np.array([EDGE_TYPES.index(value) for value in names], dtype=np.uint8)[codes] enum_id = f"{class_id}__{name}" value_type = "UINT8" if len(names) <= 256 else "UINT16" schema["enums"][enum_id] = {"valueType": value_type, "values": [{"name": label, "value": code} for code, label in enumerate(names)]} - code_of = {label: code for code, label in enumerate(names)} - return enum_id, np.array([code_of[value] for value in column], dtype=value_type.lower()) + return enum_id, codes.astype(value_type.lower()) def _grid_json(grid: Grid) -> dict: @@ -314,16 +384,20 @@ def accessor(array, accessor_type, component_type, target, with_bounds=False): def from_bytes(data: bytes) -> Frame: """The Frame of a GLB file's bytes.""" document, binary = unpack(data) + return _frame_of(document, binary, lazy=False) + + +def _frame_of(document: dict, binary, lazy: bool) -> Frame: extensions = document.get("extensions", {}) if EXTENSION not in extensions: - raise ValueError(f"not an fspm-TF file: no {EXTENSION} extension") + raise ValueError(f"not an encoding3d file: no {EXTENSION} extension") scene, metadata = extensions[EXTENSION], extensions.get(METADATA, {"schema": {}, "propertyTables": []}) variables = scene.get("variables", {}) def table(index: int, kind, **fields): description = metadata["propertyTables"][index] class_id = description["class"] - columns, declarations = _decode_table(document, binary, description, metadata["schema"], variables) + columns, declarations = _decode_table(document, binary, description, metadata["schema"], variables, lazy) return kind(name=description.get("name", class_id), columns=columns, variables=declarations, **fields) frame_info = scene.get("frame", {}) @@ -333,9 +407,10 @@ def table(index: int, kind, **fields): for entry in scene.get("dataStructures", []): data_structure = DataStructure( id=entry["id"], kind=entry.get("kind", "graph"), platform=entry.get("platform"), - model=entry.get("model"), extras=entry.get("extras", {}), + model=entry.get("model"), extras=entry.get("extras", {}), labels=entry.get("labels", {}), origin=np.asarray(entry.get("origin", np.eye(4).reshape(-1)), dtype=float).reshape(4, 4).T) - data_structure.entity_sets = [table(item["propertyTable"], EntitySet, rank=item.get("rank")) + data_structure.entity_sets = [table(item["propertyTable"], EntitySet, rank=item.get("rank"), + scale=item.get("scale")) for item in entry.get("entitySets", [])] data_structure.relations = [table(item["propertyTable"], Relation, source_set=item["sourceSet"], target_set=item["targetSet"]) for item in entry.get("relations", [])] @@ -348,14 +423,21 @@ def table(index: int, kind, **fields): periodic=tuple(grid["periodic"]), z_up=grid.get("zUp", True), max_level=grid.get("maxLevel", 0), cells=grid["cells"]) tables = {item["propertyTable"]: item["name"] for item in entry.get("entitySets", [])} - for mesh_entry in entry.get("meshes", []): - data_structure.meshes.append(_decode_mesh(document, binary, mesh_entry, tables, data_structure)) + meshes = [(lambda mesh_entry=mesh_entry, data_structure=data_structure, tables=tables: + _decode_mesh(document, binary, mesh_entry, tables, data_structure)) + for mesh_entry in entry.get("meshes", [])] + data_structure.meshes = LazyList(meshes) if lazy else [mesh() for mesh in meshes] frame.data_structures.append(data_structure) for entry in scene.get("mappings", []): frame.mappings.append(table(entry["propertyTable"], Mapping, source=(entry["source"]["dataStructure"], entry["source"]["entitySet"]), target=(entry["target"]["dataStructure"], entry["target"]["entitySet"]), method=entry.get("method"))) + if "update" in scene: + update = scene["update"] + frame.update = Update(participant=update["participant"], removed={ + (item["dataStructure"], item["entitySet"]): np.asarray(table(item["propertyTable"], Table).columns["id"]) + for item in update.get("removed", [])}) unknown_extensions = {name: value for name, value in extensions.items() if name not in (METADATA, EXTENSION, FEATURES)} if unknown_extensions: @@ -365,50 +447,50 @@ def table(index: int, kind, **fields): return frame -def read(path: str) -> Frame: - """The Frame of the GLB file *path*.""" +def read(path: str, lazy: bool = False) -> Frame: + """The Frame of the GLB file *path*. *lazy*: the file is mapped in memory and only its JSON parsed; each column + (LazyColumns) and each mesh (LazyList) is decoded when first read, so that a reader using a few columns of a large + frame decodes only those. A lazy frame holds the file open until frame.close() (or the end of a with block).""" + if not lazy: + with open(path, "rb") as file: + return from_bytes(file.read()) + import mmap with open(path, "rb") as file: - return from_bytes(file.read()) + mapped = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) + document, binary = unpack(memoryview(mapped)) + frame = _frame_of(document, binary, lazy=True) + frame._source = (mapped, binary) + return frame -def _decode_table(document: dict, binary: bytes, description: dict, schema: dict, variables: dict) -> tuple: +def _decode_table(document: dict, binary: bytes, description: dict, schema: dict, variables: dict, + lazy: bool = False) -> tuple: + """(columns, declarations) of a property table; *lazy*: the columns a LazyColumns, each decoded when first read.""" class_id = description["class"] - properties = schema["classes"][class_id]["properties"] count = description["count"] - columns, declarations = {}, {} - for name, stored in description["properties"].items(): + if lazy: + columns = LazyColumns({name: (lambda name=name, stored=stored: _checked( + name, _decode_column(document, binary, schema, class_id, name, stored, count))) + for name, stored in description["properties"].items()}, count) + else: + columns = {name: _decode_column(document, binary, schema, class_id, name, stored, count) + for name, stored in description["properties"].items()} + return columns, _decode_declarations(schema, class_id, description, variables) + + +def _checked(name: str, column): + """A column decoded lazily, as a table checks it (ids int64).""" + column = _as_column(name, column) + if name in ID_COLUMNS and column.dtype.kind in "iu" and not np.ma.isMaskedArray(column): + column = column.astype(np.int64) + return column + + +def _decode_declarations(schema: dict, class_id: str, description: dict, variables: dict) -> dict: + properties = schema["classes"][class_id]["properties"] + declarations = {} + for name in description["properties"]: prop = properties[name] - raw = view_bytes(document, binary, stored["values"]) - if prop["type"] == "ENUM": - enum = schema["enums"][prop["enumType"]] - codes = np.frombuffer(raw, dtype=enum.get("valueType", "UINT16").lower(), count=count) - name_of = {item["value"]: item["name"] for item in enum["values"]} - column = np.array([name_of[int(code)] for code in codes], dtype=object) - elif prop["type"] == "STRING": - offset_type = stored.get("stringOffsetType", "UINT32").lower() - offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, - count=count + 1) - column = np.array([raw[offsets[i]:offsets[i + 1]].decode("utf-8") for i in range(count)], dtype=object) - elif prop["type"] == "BOOLEAN": - column = np.unpackbits(np.frombuffer(raw, dtype=np.uint8), bitorder="little", count=count).astype(bool) - elif prop["type"] == "SCALAR": - dtype = DTYPES[prop["componentType"]] - width = prop.get("count", 1) if prop.get("array") else 1 - if prop.get("array") and "count" not in prop: - raise ValueError(f"'{class_id}.{name}': variable-length arrays are not part of fspm-TF 0.1") - column = np.frombuffer(raw, dtype=dtype.newbyteorder("<"), count=count * width).astype(dtype) - if width > 1: - column = column.reshape(count, width) - if dtype.kind == "f" and "noData" in prop: - column = np.where(column == np.asarray(prop["noData"], dtype=dtype), np.nan, column).astype(dtype) - infinity = prop.get("extras", {}).get("FSPM_infinity") - if dtype.kind == "f" and infinity: - column = np.where(column == np.asarray(infinity["positive"], dtype=dtype), np.inf, - np.where(column == np.asarray(infinity["negative"], dtype=dtype), -np.inf, column) - ).astype(dtype) - else: - raise ValueError(f"'{class_id}.{name}': property type {prop['type']} is not part of fspm-TF 0.1") - columns[name] = column declared = variables.get(class_id, {}).get(name) if declared is not None or "description" in prop: declared = declared or {} @@ -416,8 +498,57 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict unit=declared.get("unit", "1"), standard_name=declared.get("standardName"), kind=declared.get("kind"), role=declared.get("role", "state"), description=prop.get("description"), minimum=declared.get("minimum"), maximum=declared.get("maximum"), - references=tuple(declared.get("references", ())), doi=tuple(declared.get("doi", ()))) - return columns, declarations + references=tuple(declared.get("references", ())), doi=tuple(declared.get("doi", ())), + format=declared.get("format")) + return declarations + + +def _decode_column(document: dict, binary, schema: dict, class_id: str, name: str, stored: dict, count: int): + """One column of a property table, decoded from the binary chunk (a copy, not a view of it).""" + prop = schema["classes"][class_id]["properties"][name] + raw = view_bytes(document, binary, stored["values"]) + if prop["type"] == "ENUM": + enum = schema["enums"][prop["enumType"]] + codes = np.frombuffer(raw, dtype=enum.get("valueType", "UINT16").lower(), count=count) + values = np.array([item["value"] for item in enum["values"]], dtype=np.int64) + names = np.empty(int(values.max()) + 1 if values.size else 0, dtype=object) + names[values] = [item["name"] for item in enum["values"]] + return names[codes.astype(np.int64)] if count else np.zeros(0, dtype=object) + if prop["type"] == "STRING": + offset_type = stored.get("stringOffsetType", "UINT32").lower() + offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, + count=count + 1) + raw = bytes(raw) + column = np.array([raw[offsets[i]:offsets[i + 1]].decode("utf-8") for i in range(count)], dtype=object) + if "noData" in prop: # missing text + column = np.array([None if value == prop["noData"] else value for value in column], dtype=object) + return column + if prop["type"] == "BOOLEAN": + return np.unpackbits(np.frombuffer(raw, dtype=np.uint8), bitorder="little", count=count).astype(bool) + if prop["type"] != "SCALAR": + raise ValueError(f"'{class_id}.{name}': property type {prop['type']} is not part of encoding3d") + dtype = DTYPES[prop["componentType"]] + width = prop.get("count", 1) if prop.get("array") else 1 + if prop.get("array") and "count" not in prop: + raise ValueError(f"'{class_id}.{name}': variable-length arrays are not part of encoding3d") + column = np.frombuffer(raw, dtype=dtype.newbyteorder("<"), count=count * width).astype(dtype) + if width > 1: + column = column.reshape(count, width) + if dtype.kind == "f" and "noData" in prop: + column = np.where(column == np.asarray(prop["noData"], dtype=dtype), np.nan, column).astype(dtype) + infinity = prop.get("extras", {}).get("FSPM_infinity") + if dtype.kind == "f" and infinity: + column = np.where(column == np.asarray(infinity["positive"], dtype=dtype), np.inf, + np.where(column == np.asarray(infinity["negative"], dtype=dtype), -np.inf, column) + ).astype(dtype) + if prop.get("extras", {}).get("FSPM_boolean"): # booleans with missing values + missing = column == prop.get("noData", BOOLEAN_NO_DATA) + return np.ma.MaskedArray(column == 1, mask=missing) if missing.any() else column == 1 + if dtype.kind in "iu" and "noData" in prop and name not in ("complex", "parent"): # missing integers + missing = column == prop["noData"] + if missing.any(): + return np.ma.MaskedArray(column, mask=missing) + return column def _accessor_array(document: dict, binary: bytes, index: int) -> np.ndarray: @@ -451,7 +582,7 @@ def _decode_mesh(document: dict, binary: bytes, mesh_entry: dict, tables: dict, feature_rows[set_name] = rows display, sources = {}, {} for item in mesh_entry.get("display", []): - display[item["variable"]] = _accessor_array(document, binary, attributes[item["attribute"]]) + display[item["variable"]] = _accessor_array(document, binary, attributes[item["attribute"]]).copy() if item.get("entitySet") is not None: sources[item["variable"]] = item["entitySet"] indices = (_accessor_array(document, binary, primitive["indices"]).astype(np.int64) diff --git a/bindings/python/src/openalea/fspm_tf/compare.py b/bindings/python/src/openalea/encoding3d/compare.py similarity index 96% rename from bindings/python/src/openalea/fspm_tf/compare.py rename to bindings/python/src/openalea/encoding3d/compare.py index e904afc..1dc0c07 100644 --- a/bindings/python/src/openalea/fspm_tf/compare.py +++ b/bindings/python/src/openalea/encoding3d/compare.py @@ -1,6 +1,6 @@ """ Comparing what a codec reads with the conformance files' expected values (``*.expected.json``), whatever the codec's -language: the codec dumps each GLB it reads as JSON (the layout of fspm_tf.dump), and this compares the dumps:: +language: the codec dumps each GLB it reads as JSON (the layout of encoding3d.dump), and this compares the dumps:: compare(actual, expected) -> [differences] two JSON values: objects by key, arrays in order, numbers exactly (1 and 1.0 equal), null, "Infinity", "-Infinity", text @@ -12,7 +12,7 @@ import math import os -from openalea.fspm_tf.dump import frame_to_dict +from openalea.encoding3d.dump import frame_to_dict def _number(value) -> bool: @@ -51,7 +51,7 @@ def compare(actual, expected, path: str = "$") -> list: def load(path: str): """A JSON dump, or a GLB read by this package and dumped (to check this package itself).""" if path.endswith(".glb"): - from openalea.fspm_tf import read + from openalea.encoding3d import read return json.loads(json.dumps(frame_to_dict(read(path)))) with open(path, encoding="utf-8") as file: return json.load(file) diff --git a/bindings/python/src/openalea/encoding3d/coupling/__init__.py b/bindings/python/src/openalea/encoding3d/coupling/__init__.py new file mode 100644 index 0000000..3cf82bb --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/__init__.py @@ -0,0 +1,18 @@ +""" +Coupling platforms through encoding3d (specification §11): a session between participants, each a platform's model +in its own process (or a recorded sequence), coordinated by a coordinator that holds the shared structure. + +- messages: the declarations, links and errors (Declaration, EntitySetSpec, VariableSpec, Link, CouplingError); +- resolution: each input's provider and the stages (resolve, stages); +- store: the session's structure, the updates merged into it and the inputs read from it (SessionStore); +- channels: the transports, ZeroMQ and a folder; +- participant: a platform's side (Participant, Turn); +- coordinator: the coordinator (Coordinator), launching participants and recording the session; +- replay: a recorded sequence as a participant (Replay). +""" +from openalea.encoding3d.coupling.messages import (ID_COUNT, PROTOCOL_VERSION, CouplingError, Declaration, # noqa: F401 + EntitySetSpec, Link, MappingSpec, VariableSpec) +from openalea.encoding3d.coupling.resolution import Coupling, resolve, stages # noqa: F401 +from openalea.encoding3d.coupling.participant import Participant, Turn # noqa: F401 +from openalea.encoding3d.coupling.coordinator import Coordinator, python_command # noqa: F401 +from openalea.encoding3d.coupling.replay import Replay # noqa: F401 diff --git a/bindings/python/src/openalea/encoding3d/coupling/channels.py b/bindings/python/src/openalea/encoding3d/coupling/channels.py new file mode 100644 index 0000000..95737b2 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/channels.py @@ -0,0 +1,216 @@ +""" +The session's transports. A message is a JSON header and, for turns and updates, a payload: a partial frame's GLB +bytes. Both transports carry the same messages: + +- ZeroMQ (endpoint "tcp://host:port" or "ipc://path"): the coordinator binds a ROUTER socket, each participant + connects a DEALER socket whose identity is its name; a message is two parts (header, payload) or one (header). +- a folder (endpoint "files:path"): each participant has a folder // with two message queues, + to_coordinator/ and to_participant/; message n is n.json (its header) and n.glb (its payload), the payload written + first and the header renamed into place last (a header present: a complete message). Nothing else is needed than + files, so any platform can take part. Each message is deleted by its reader once read, unless the session keeps + them (keepMessages: the folder is then the session's log; a record, Coordinator(record=), is the usual one). + +Both sides poll; every wait has a timeout (seconds, None: none). +""" +import json +import os +import time + +POLL = 0.005 # seconds between two looks at a folder queue +ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT" # what a launched participant connects to +PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT" # its name + + +class ChannelTimeout(TimeoutError): + pass + + +def _header(data: bytes) -> dict: + return json.loads(data.decode("utf-8")) + + +def _bytes(header: dict) -> bytes: + return json.dumps(header, separators=(",", ":")).encode("utf-8") + + +# ── ZeroMQ ─────────────────────────────────────────────────────────────────────── +class ZmqCoordinatorChannel: + def __init__(self, endpoint: str = "tcp://127.0.0.1:*"): + import zmq + self._zmq = zmq + self.context = zmq.Context.instance() + self.socket = self.context.socket(zmq.ROUTER) + self.socket.setsockopt(zmq.LINGER, 1000) + self.socket.bind(endpoint) + self.endpoint = self.socket.getsockopt_string(zmq.LAST_ENDPOINT) + + def receive(self, timeout=None): + """(participant, header, payload or None).""" + if not self.socket.poll(None if timeout is None else int(timeout * 1000)): + raise ChannelTimeout(f"no message in {timeout} s") + parts = self.socket.recv_multipart() + return parts[0].decode("utf-8"), _header(parts[1]), parts[2] if len(parts) > 2 else None + + def send(self, participant: str, header: dict, payload: bytes = None) -> None: + parts = [participant.encode("utf-8"), _bytes(header)] + ([payload] if payload is not None else []) + self.socket.send_multipart(parts) + + def close(self) -> None: + self.socket.close() + + +class ZmqParticipantChannel: + def __init__(self, endpoint: str, participant: str): + import zmq + self.context = zmq.Context.instance() + self.socket = self.context.socket(zmq.DEALER) + self.socket.setsockopt(zmq.IDENTITY, participant.encode("utf-8")) + self.socket.setsockopt(zmq.LINGER, 1000) + self.socket.connect(endpoint) + + def receive(self, timeout=None): + """(header, payload or None).""" + if not self.socket.poll(None if timeout is None else int(timeout * 1000)): + raise ChannelTimeout(f"no message in {timeout} s") + parts = self.socket.recv_multipart() + return _header(parts[0]), parts[1] if len(parts) > 1 else None + + def send(self, header: dict, payload: bytes = None) -> None: + self.socket.send_multipart([_bytes(header)] + ([payload] if payload is not None else [])) + + def close(self) -> None: + self.socket.close() + + +# ── a folder ───────────────────────────────────────────────────────────────────── +class _Queue: + """A folder of numbered messages, written by one side and read by the other.""" + + def __init__(self, folder: str): + self.folder = folder + os.makedirs(folder, exist_ok=True) + self.next_read = 0 + self.next_write = 0 + self.keep = True # False: a message's files deleted once read + self.read_files = [] # those of the messages read and kept + + def discard_read(self) -> None: + """Delete the files of the messages read so far, and from now on each message once read.""" + self.keep = False + for path in self.read_files: + if os.path.exists(path): + os.remove(path) + self.read_files = [] + + def write(self, header: dict, payload: bytes = None) -> None: + stem = os.path.join(self.folder, f"{self.next_write:08d}") + if payload is not None: + with open(stem + ".glb", "wb") as file: + file.write(payload) + header = dict(header, payload=os.path.basename(stem) + ".glb") + with open(stem + ".json.part", "wb") as file: + file.write(_bytes(header)) + os.replace(stem + ".json.part", stem + ".json") + self.next_write += 1 + + def read(self): + """The next message (header, payload or None), or None if not there yet.""" + stem = os.path.join(self.folder, f"{self.next_read:08d}") + if not os.path.exists(stem + ".json"): + return None + with open(stem + ".json", "rb") as file: + header = _header(file.read()) + payload, files = None, [stem + ".json"] + if "payload" in header: + files.append(os.path.join(self.folder, header.pop("payload"))) + with open(files[-1], "rb") as file: + payload = file.read() + self.next_read += 1 + if self.keep: + self.read_files += files + else: + for path in files: + os.remove(path) + return header, payload + + +class FolderCoordinatorChannel: + def __init__(self, folder: str, keep: bool = False): + self.folder = os.path.abspath(folder) + os.makedirs(self.folder, exist_ok=True) + for name in os.listdir(self.folder): # a session's folder is the log of that session only + if os.path.exists(os.path.join(self.folder, name, "to_participant", f"{0:08d}.json")): + raise ValueError(f"{self.folder} holds a previous session ({name}): use a new folder") + self.endpoint = "files:" + self.folder + self.keep = keep + self.incoming, self.outgoing = {}, {} + + def _discover(self) -> None: + for name in sorted(os.listdir(self.folder)): + if name not in self.incoming and os.path.isdir(os.path.join(self.folder, name, "to_coordinator")): + self.incoming[name] = _Queue(os.path.join(self.folder, name, "to_coordinator")) + if not self.keep: + self.incoming[name].discard_read() + self.outgoing[name] = _Queue(os.path.join(self.folder, name, "to_participant")) + + def receive(self, timeout=None): + deadline = None if timeout is None else time.monotonic() + timeout + while True: + self._discover() + for name, queue in self.incoming.items(): + message = queue.read() + if message is not None: + return (name,) + message + if deadline is not None and time.monotonic() > deadline: + raise ChannelTimeout(f"no message in {timeout} s") + time.sleep(POLL) + + def send(self, participant: str, header: dict, payload: bytes = None) -> None: + self._discover() + self.outgoing[participant].write(header, payload) + + def close(self) -> None: + pass + + +class FolderParticipantChannel: + def __init__(self, folder: str, participant: str): + base = os.path.join(folder, participant) + self.incoming = _Queue(os.path.join(base, "to_participant")) + self.outgoing = _Queue(os.path.join(base, "to_coordinator")) + + def receive(self, timeout=None): + deadline = None if timeout is None else time.monotonic() + timeout + while True: + message = self.incoming.read() + if message is not None: + return message + if deadline is not None and time.monotonic() > deadline: + raise ChannelTimeout(f"no message in {timeout} s") + time.sleep(POLL) + + def send(self, header: dict, payload: bytes = None) -> None: + self.outgoing.write(header, payload) + + def keep_messages(self, keep: bool) -> None: + """As the session says (its welcome): keep the messages read, or delete them (those read so far too).""" + if not keep: + self.incoming.discard_read() + + def close(self) -> None: + pass + + +def coordinator_channel(endpoint: str, keep_messages: bool = False): + """The coordinator's side of *endpoint*: "files:path", or a ZeroMQ endpoint to bind ("tcp://127.0.0.1:*": a + free port). *keep_messages*: the folder transport keeps every message (the session's log).""" + if endpoint.startswith("files:"): + return FolderCoordinatorChannel(endpoint[len("files:"):], keep_messages) + return ZmqCoordinatorChannel(endpoint) + + +def participant_channel(endpoint: str, participant: str): + """A participant's side of *endpoint*.""" + if endpoint.startswith("files:"): + return FolderParticipantChannel(endpoint[len("files:"):], participant) + return ZmqParticipantChannel(endpoint, participant) diff --git a/bindings/python/src/openalea/encoding3d/coupling/conformance.py b/bindings/python/src/openalea/encoding3d/coupling/conformance.py new file mode 100644 index 0000000..db3474f --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/conformance.py @@ -0,0 +1,119 @@ +""" +Conformance sessions (specification §9): how a codec proves its participant's side of a coupling session. + +A session's folder (conformance/coupling//) holds: + + session.json the coordinator's settings: steps, interval, timeUnit, links, stages + peer/scene.fspm.json, frames the other participant, a recorded sequence the coordinator replays ("peer") + subject/declaration.json the participant under test's declaration ("subject") + subject/update_.glb what it sends at each step (0: its initial state) + expected/turn_.json what it must read from each turn (frame_to_dict) + expected/record/frame_.json the session's record, once its updates merged + +The participant under test is a small program of the codec's language, run as + + SESSION OUT + +that joins the session with subject/declaration.json, and at each turn writes what it read (its dump, the +conformance files' layout) to OUT/turn_.json and sends subject/update_.glb as read and written again by +its codec, until the coordinator stops it. `encoding3d couple-conformance SESSIONS --participant COMMAND` runs every +session over ZeroMQ and over a folder and compares the dumps; this module's `subject` is Python's program +(`python -m openalea.encoding3d.coupling.conformance subject SESSION OUT`). +""" +import json +import os +import sys + +from openalea.encoding3d.codec import from_bytes, read, read_index, to_bytes +from openalea.encoding3d.compare import compare +from openalea.encoding3d.coupling.coordinator import Coordinator +from openalea.encoding3d.coupling.messages import Declaration, Link +from openalea.encoding3d.coupling.participant import Participant +from openalea.encoding3d.dump import frame_to_dict + +TRANSPORTS = ("zmq", "files") + + +def subject(session: str, out: str) -> int: + """The participant under test, in Python.""" + os.makedirs(out, exist_ok=True) + with open(os.path.join(session, "subject", "declaration.json"), encoding="utf-8") as file: + declaration = Declaration.from_json(json.load(file)) + participant = Participant.connect(declaration, timeout=120) + for turn in participant.turns(): + with open(os.path.join(out, f"turn_{turn.step:04d}.json"), "w", encoding="utf-8") as file: + json.dump(frame_to_dict(turn.frame), file, indent=1) + participant.send(from_bytes(to_bytes(read(os.path.join(session, "subject", + f"update_{turn.step:04d}.glb"))))) + return 0 + + +def _settings(session: str) -> dict: + with open(os.path.join(session, "session.json"), encoding="utf-8") as file: + return json.load(file) + + +def run_session(session: str, command: str, out: str, transport: str = "zmq") -> list: + """Run one session with *command* as its subject; the differences between what it read (and the record) and + the expected values, as lines (empty: it conforms).""" + settings = _settings(session) + endpoint = "tcp://127.0.0.1:*" if transport == "zmq" else "files:" + os.path.join(out, "messages") + coordinator = Coordinator(endpoint, steps=settings["steps"], interval=settings["interval"], + time_unit=settings.get("timeUnit", "d"), start=settings.get("start", 0.), + links=[Link.from_json(link) for link in settings.get("links", [])], + stages=settings.get("stages"), + replays={"peer": os.path.join(session, "peer", "scene.fspm.json")}, + record=os.path.join(out, "record"), timeout=settings.get("timeout", 120)) + turns = os.path.join(out, "turns") + coordinator.launch("subject", f"{command} {session} {turns}") + coordinator.run() + differences = [] + if transport == "files": # each message deleted by its reader once read + left = [os.path.join(root, name) for root, _, names in os.walk(os.path.join(out, "messages")) + for name in names] + if left: + differences.append(f"{len(left)} message file(s) left in the folder, e.g. " + f"{os.path.relpath(left[0], out)}: a reader deletes each message once read") + expected_folder = os.path.join(session, "expected") + for name in sorted(os.listdir(expected_folder)): + if name.startswith("turn_") and name.endswith(".json"): + differences += _compare(os.path.join(turns, name), os.path.join(expected_folder, name)) + for entry in read_index(os.path.join(out, "record", "scene.fspm.json"))["frames"]: + expected = os.path.join(expected_folder, "record", entry["file"].replace(".glb", ".json")) + differences += _compare_values(frame_to_dict(read(entry["path"])), expected, entry["file"]) + return differences + + +def _compare(actual_path: str, expected_path: str) -> list: + if not os.path.exists(actual_path): + return [f"{os.path.basename(actual_path)}: not written"] + with open(actual_path, encoding="utf-8") as file: + actual = json.load(file) + return _compare_values(actual, expected_path, os.path.basename(actual_path)) + + +def _compare_values(actual, expected_path: str, name: str) -> list: + with open(expected_path, encoding="utf-8") as file: + expected = json.load(file) + actual = json.loads(json.dumps(actual)) # the same JSON types as the expected file's + return [f"{name}: {line}" for line in compare(actual, expected)] + + +def run_all(sessions: str, command: str, out: str, transports=TRANSPORTS) -> dict: + """Every session of the folder *sessions*, over each transport: {(session, transport): differences}.""" + results = {} + for name in sorted(os.listdir(sessions)): + session = os.path.join(sessions, name) + if not os.path.exists(os.path.join(session, "session.json")): + continue + for transport in transports: + results[(name, transport)] = run_session(os.path.abspath(session), command, + os.path.join(out, name, transport), transport) + return results + + +if __name__ == "__main__": + if len(sys.argv) == 4 and sys.argv[1] == "subject": + sys.exit(subject(sys.argv[2], sys.argv[3])) + print("usage: python -m openalea.encoding3d.coupling.conformance subject SESSION OUT", file=sys.stderr) + sys.exit(2) diff --git a/bindings/python/src/openalea/encoding3d/coupling/coordinator.py b/bindings/python/src/openalea/encoding3d/coupling/coordinator.py new file mode 100644 index 0000000..d39c259 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/coordinator.py @@ -0,0 +1,203 @@ +""" +The coordinator: it runs a session between participants it launches (commands) or that connect to it, and replayed +sequences. + +1. Each participant sends its hello (its declaration, and links it proposes); the session's links are those given to + the coordinator and those proposed, the given ones first. +2. Resolution (resolution.resolve, resolution.stages): each input's provider, the stages. Each participant is + welcomed with its id range and its couplings, or the session stops with the reason. +3. Initial state: stage by stage, each participant is sent a turn of step 0 (the entities so far and its inputs) and + answers with its initial outputs and entities. +4. Steps 1 … steps: stage by stage, the participants of a stage are sent their turn (time t → t + interval: their + inputs, the entities added or removed since their last turn), and their updates are merged once all of them + answered (so within a stage they see the same state, and the next stage sees theirs). +5. Stop. A participant's error, or its update the store refuses, stops the session: every participant is told why. + +*record*: a folder where the session is written as a sequence of full frames (every entity set and output), after the +initial state and each step. *keep_messages*: over a folder, every message kept (else each deleted once read). +""" +import os +import subprocess +import sys +import time as clock + +from openalea.encoding3d.codec import from_bytes, to_bytes, write, write_index +from openalea.encoding3d.coupling.channels import (ChannelTimeout, ENDPOINT_VARIABLE, PARTICIPANT_VARIABLE, + coordinator_channel) +from openalea.encoding3d.coupling.messages import (ID_COUNT, PROTOCOL_VERSION, CouplingError, Declaration, Link) +from openalea.encoding3d.coupling.replay import Replay +from openalea.encoding3d.coupling.resolution import ranks, resolve, stages +from openalea.encoding3d.coupling.store import SessionStore + + +class Coordinator: + def __init__(self, endpoint: str = "tcp://127.0.0.1:*", steps: int = 1, interval: float = 1., + time_unit: str = "d", start: float = 0., links=(), stages=None, expect=(), replays=None, + record: str = None, timeout: float = 600., keep_messages: bool = False): + self.channel = coordinator_channel(endpoint, keep_messages) + self.keep_messages = keep_messages + self.endpoint = self.channel.endpoint + self.steps, self.interval, self.time_unit, self.start = steps, interval, time_unit, start + self.links = [link if isinstance(link, Link) else Link.from_json(link) for link in links] + self.given_stages = stages + self.expect = list(expect) + self.replays = {name: Replay(name, path) for name, path in (replays or {}).items()} + self.record, self.timeout = record, timeout + self.processes = {} + self.declarations, self.offsets, self.removed_seen = {}, {}, {} + self.couplings, self.stages, self.store = [], [], None + self.recorded = [] + + # ── launching ───────────────────────────────────────────────────────────────── + def launch(self, name: str, command, environment=None, cwd=None) -> None: + """Start participant *name*: *command* (a list, or a string run by the shell) with ENCODING3D_ENDPOINT and + ENCODING3D_PARTICIPANT set; the session waits for its hello.""" + environment = dict(os.environ if environment is None else environment) + environment[ENDPOINT_VARIABLE] = self.endpoint + environment[PARTICIPANT_VARIABLE] = name + self.processes[name] = subprocess.Popen(command, env=environment, cwd=cwd, shell=isinstance(command, str)) + self.expect.append(name) + + def _check_processes(self) -> None: + for name, process in self.processes.items(): + code = process.poll() + if code not in (None, 0) or (code == 0 and name not in self.finished): + raise CouplingError(f"participant {name} exited with code {code} during the session") + + def _receive(self): + """The next message, watching the launched participants and the timeout.""" + deadline = None if self.timeout is None else clock.monotonic() + self.timeout + while True: + try: + return self.channel.receive(0.2) + except ChannelTimeout: + self._check_processes() + if deadline is not None and clock.monotonic() > deadline: + raise CouplingError(f"no message in {self.timeout} s") from None + + # ── the session ────────────────────────────────────────────────────────────── + def run(self) -> SessionStore: + """Run the session; the session's store at its end.""" + self.finished = set() + try: + self._join() + self._run_steps() + except Exception as error: + self._stop(f"{type(error).__name__}: {error}") + self._wait_processes() + raise + self._stop("end of the session") + self._wait_processes() + if self.record and self.recorded: + write_index(os.path.join(self.record, "scene.fspm.json"), self.recorded, time_unit=self.time_unit) + return self.store + + def _join(self) -> None: + pending = set(self.expect) + proposed = [] + for name, replay in self.replays.items(): + self.declarations[name] = replay.declaration + while pending: + participant, header, _ = self._receive() + if header.get("type") == "error": + raise CouplingError(f"{participant}: {header.get('message')}") + if header.get("type") != "hello": + raise CouplingError(f"{participant}: expected its hello, got {header.get('type')}") + if header.get("protocol") != PROTOCOL_VERSION: + raise CouplingError(f"{participant}: protocol {header.get('protocol')}, the coordinator's is " + f"{PROTOCOL_VERSION}") + declaration = Declaration.from_json(header["declaration"]) + if declaration.participant != participant: + raise CouplingError(f"{participant} declares itself as {declaration.participant}") + if participant in self.declarations: + raise CouplingError(f"two participants named {participant}") + if participant not in pending: + raise CouplingError(f"{participant} is not expected (expected: {sorted(self.expect)})") + self.declarations[participant] = declaration + proposed += [Link.from_json(link) for link in header.get("links", [])] + pending.discard(participant) + given = {(link.receiver, link.variable, link.entity_set) for link in self.links} + links = self.links + [link for link in proposed + if (link.receiver, link.variable, link.entity_set) not in given] + self.couplings = resolve(self.declarations, links) + self.stages = stages(self.declarations, self.given_stages) + self.store = SessionStore(ranks(self.declarations)) + order = [name for name in self.expect if name in self.declarations] + list(self.replays) + for position, name in enumerate(order): # deterministic: the expected order, then replays + self.offsets[name] = (position + 1) * ID_COUNT + self.removed_seen[name] = 0 + for name, replay in self.replays.items(): + replay.id_offset = self.offsets[name] + replay.linear = {(coupling.source.data_structure, coupling.source.entity_set, coupling.source.name) + for coupling in self.couplings + if coupling.provider == name and coupling.interpolation == "linear"} + replay.needed = {(coupling.source.data_structure, coupling.source.entity_set, coupling.source.name) + for coupling in self.couplings if coupling.provider == name} + session = {"start": self.start, "interval": self.interval, "timeUnit": self.time_unit, "steps": self.steps, + "stages": self.stages, "participants": list(self.declarations), + "keepMessages": self.keep_messages} + for name in self.declarations: + if name in self.replays: + continue + self.channel.send(name, {"type": "welcome", "idOffset": self.offsets[name], "idCount": ID_COUNT, + "session": session, + "couplings": [coupling.to_json() for coupling in self.couplings + if coupling.receiver == name]}) + + def _run_steps(self) -> None: + for step in range(self.steps + 1): + time = self.start + max(step - 1, 0) * self.interval + until = self.start + step * self.interval + for stage in self.stages: + updates = {} + live = [name for name in stage if name not in self.replays] + for name in live: + frame = self.store.inputs(name, self.declarations[name], self.couplings, self.removed_seen[name], + time, self.time_unit, step) + self.removed_seen[name] = len(self.store.removed_log) + self.channel.send(name, {"type": "turn", "step": step, "phase": "init" if step == 0 else "step", + "time": time, "until": until}, to_bytes(frame)) + for name in stage: + if name in self.replays: + updates[name] = self.replays[name].update(until, self.time_unit) + waiting = set(live) + while waiting: + participant, header, payload = self._receive() + if header.get("type") == "error": + raise CouplingError(f"{participant}: {header.get('message')}") + if header.get("type") != "update" or participant not in waiting: + raise CouplingError(f"{participant}: unexpected {header.get('type')} at step {step}") + if header.get("step") != step: + raise CouplingError(f"{participant}: update of step {header.get('step')} at step {step}") + updates[participant] = from_bytes(payload) + waiting.discard(participant) + for name in stage: # merged in the stage's order, once all answered + self.store.merge(name, self.declarations[name], updates[name], self.offsets[name]) + if self.record: + os.makedirs(self.record, exist_ok=True) + path = f"frame_{step:05d}.glb" + write(os.path.join(self.record, path), self.store.frame(until, self.time_unit, step)) + self.recorded.append({"time": until, "step": step, "file": path}) + + def _stop(self, reason: str) -> None: + for name in self.declarations: + if name in self.replays: + continue + try: + self.channel.send(name, {"type": "stop", "reason": reason}) + except Exception: # a participant already gone + pass + self.finished = set(self.declarations) + + def _wait_processes(self) -> None: + for name, process in self.processes.items(): + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + self.channel.close() + + +def python_command(script: str, *arguments) -> list: + """The command running a Python *script* with this interpreter.""" + return [sys.executable, script, *map(str, arguments)] diff --git a/bindings/python/src/openalea/encoding3d/coupling/messages.py b/bindings/python/src/openalea/encoding3d/coupling/messages.py new file mode 100644 index 0000000..7a7029f --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/messages.py @@ -0,0 +1,210 @@ +""" +A coupling session's messages (specification §11), as JSON headers, some with a partial frame (GLB bytes) as payload. + +Participant to coordinator:: + + hello {"type": "hello", "protocol": "0.3", "declaration": {...}, "links": [...]} + update {"type": "update", "step": k} + its outputs (a partial frame) + error {"type": "error", "message": "..."} + +Coordinator to participant:: + + welcome {"type": "welcome", "idOffset": ..., "idCount": ..., "session": {...}, "couplings": [...]} + turn {"type": "turn", "step": k, "phase": "init" | "step", "time": t, "until": t'} + its inputs + stop {"type": "stop", "reason": "..."} + +A declaration says what a participant works on (its entity sets and its rights on their topology), what it provides +(its outputs) and requires (its inputs). Links say which output feeds which input; the coordinator completes them by +standard name, then by name. +""" +from dataclasses import dataclass, field +from typing import Optional + +from openalea.encoding3d.model import BASE_SCALES + +PROTOCOL_VERSION = "0.3" +RIGHTS = ("owns", "refines", "follows") +AGGREGATIONS = ("sum", "mean", "max", "min", "copy") +INTERPOLATIONS = ("hold", "linear") +ID_COUNT = 2 ** 32 # the ids a participant may create: [offset, offset + ID_COUNT) + + +class CouplingError(RuntimeError): + """A coupling that cannot run: a declaration, link or update the protocol refuses, or a participant's error.""" + + +@dataclass +class EntitySetSpec: + """An entity set a participant works on, with its rank (coarser first: a base scale's position by default) and + its rights on its topology: "owns" (adds and removes its entities), "refines" (adds and removes the entities it + creates, under others'), "follows" (reads).""" + data_structure: str + name: str + rank: Optional[int] = None + rights: str = "follows" + + def __post_init__(self): + if self.rank is None and self.name in BASE_SCALES: + self.rank = BASE_SCALES.index(self.name) + 1 + if self.rights not in RIGHTS: + raise CouplingError(f"entity set {self.data_structure}/{self.name}: rights '{self.rights}', one of {RIGHTS}") + + def to_json(self) -> dict: + return {"dataStructure": self.data_structure, "name": self.name, "rank": self.rank, "rights": self.rights} + + @classmethod + def from_json(cls, value: dict) -> "EntitySetSpec": + return cls(value["dataStructure"], value["name"], value.get("rank"), value.get("rights", "follows")) + + +@dataclass +class VariableSpec: + """A variable a participant provides (an output) or requires (an input), on one entity set: its name in the + participant, its unit (UCUM), standard name and kind. An optional input may stay unprovided.""" + data_structure: str + entity_set: str + name: str + direction: str + unit: str = "1" + standard_name: Optional[str] = None + kind: Optional[str] = None + optional: bool = False + + def __post_init__(self): + if self.direction not in ("provides", "requires"): + raise CouplingError(f"variable {self.name}: direction '{self.direction}', 'provides' or 'requires'") + + @property + def where(self) -> tuple: + return self.data_structure, self.entity_set + + def to_json(self) -> dict: + value = {"dataStructure": self.data_structure, "entitySet": self.entity_set, "name": self.name, + "direction": self.direction, "unit": self.unit} + if self.standard_name: + value["standardName"] = self.standard_name + if self.kind: + value["kind"] = self.kind + if self.optional: + value["optional"] = True + return value + + @classmethod + def from_json(cls, value: dict) -> "VariableSpec": + return cls(value["dataStructure"], value["entitySet"], value["name"], value["direction"], + value.get("unit", "1"), value.get("standardName"), value.get("kind"), value.get("optional", False)) + + +@dataclass +class MappingSpec: + """A mapping a participant provides (its updates carry it, as a frame's Mapping of that name): weights between the + entities of *source* and *target*, each (data structure, entity set), for links between two data structures.""" + name: str + source: tuple + target: tuple + + def to_json(self) -> dict: + return {"name": self.name, "source": {"dataStructure": self.source[0], "entitySet": self.source[1]}, + "target": {"dataStructure": self.target[0], "entitySet": self.target[1]}} + + @classmethod + def from_json(cls, value: dict) -> "MappingSpec": + return cls(value["name"], (value["source"]["dataStructure"], value["source"]["entitySet"]), + (value["target"]["dataStructure"], value["target"]["entitySet"])) + + +@dataclass +class Declaration: + """A participant's declaration: its name, platform, entity sets, variables, and the mappings it provides.""" + participant: str + platform: Optional[str] = None + entity_sets: list = field(default_factory=list) + variables: list = field(default_factory=list) + mappings: list = field(default_factory=list) + + def provides(self) -> list: + return [variable for variable in self.variables if variable.direction == "provides"] + + def requires(self) -> list: + return [variable for variable in self.variables if variable.direction == "requires"] + + def entity_set(self, data_structure: str, name: str) -> Optional[EntitySetSpec]: + for entity_set in self.entity_sets: + if (entity_set.data_structure, entity_set.name) == (data_structure, name): + return entity_set + return None + + def rights(self, data_structure: str, name: str) -> str: + entity_set = self.entity_set(data_structure, name) + return entity_set.rights if entity_set is not None else "follows" + + def complete(self) -> "Declaration": + """The declaration with an entity set (following it) for each one its variables are on and it did not + list.""" + for variable in self.variables: + if self.entity_set(*variable.where) is None: + self.entity_sets.append(EntitySetSpec(variable.data_structure, variable.entity_set)) + return self + + def to_json(self) -> dict: + value = {"participant": self.participant, "entitySets": [item.to_json() for item in self.entity_sets], + "variables": [item.to_json() for item in self.variables]} + if self.platform: + value["platform"] = self.platform + if self.mappings: + value["mappings"] = [item.to_json() for item in self.mappings] + return value + + @classmethod + def from_json(cls, value: dict) -> "Declaration": + return cls(value["participant"], value.get("platform"), + [EntitySetSpec.from_json(item) for item in value.get("entitySets", [])], + [VariableSpec.from_json(item) for item in value.get("variables", [])], + [MappingSpec.from_json(item) for item in value.get("mappings", [])]).complete() + + +@dataclass +class Link: + """An input's provider: the receiver's variable (*entity_set* when its name is ambiguous) fed by the provider's + *source* variable, times *factor* (after the unit conversion), changed of scale by *aggregation* (sum, mean, max, + min from a finer entity set; copy from a coarser one), and between a replayed sequence's times by + *interpolation* (hold, linear). Between two data structures, through the *mapping* a participant provides: + "sum" shares each source entity's value among its targets in proportion to their weights (an amount is kept), + "mean" gives each target the weighted mean of its sources.""" + receiver: str + variable: str + provider: str + source: str + entity_set: Optional[str] = None + source_entity_set: Optional[str] = None + factor: float = 1. + aggregation: Optional[str] = None + interpolation: str = "hold" + mapping: Optional[str] = None + + def __post_init__(self): + if self.aggregation is not None and self.aggregation not in AGGREGATIONS: + raise CouplingError(f"link {self.receiver}.{self.variable}: aggregation '{self.aggregation}', one of " + f"{AGGREGATIONS}") + if self.interpolation not in INTERPOLATIONS: + raise CouplingError(f"link {self.receiver}.{self.variable}: interpolation '{self.interpolation}', one " + f"of {INTERPOLATIONS}") + + def to_json(self) -> dict: + value = {"receiver": self.receiver, "variable": self.variable, "provider": self.provider, + "source": self.source} + for key, name in (("entity_set", "entitySet"), ("source_entity_set", "sourceEntitySet"), + ("aggregation", "aggregation"), ("mapping", "mapping")): + if getattr(self, key) is not None: + value[name] = getattr(self, key) + if self.factor != 1.: + value["factor"] = self.factor + if self.interpolation != "hold": + value["interpolation"] = self.interpolation + return value + + @classmethod + def from_json(cls, value: dict) -> "Link": + return cls(value["receiver"], value["variable"], value["provider"], value["source"], + value.get("entitySet"), value.get("sourceEntitySet"), float(value.get("factor", 1.)), + value.get("aggregation"), value.get("interpolation", "hold"), value.get("mapping")) diff --git a/bindings/python/src/openalea/encoding3d/coupling/participant.py b/bindings/python/src/openalea/encoding3d/coupling/participant.py new file mode 100644 index 0000000..0fa3836 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/participant.py @@ -0,0 +1,93 @@ +""" +A participant's side of a session: what a platform calls to take part, whatever its own model is:: + + from openalea.encoding3d.coupling import Declaration, Participant + + participant = Participant.connect(declaration) # endpoint and name from the environment if launched + for turn in participant.turns(): # until the coordinator stops the session + inputs = turn.frame # its entity sets, their entities and its inputs + ... # its model, from turn.time to turn.until + participant.send(frame) # its outputs and topology changes (a partial frame) + +The ids of the entities it creates are in [participant.id_offset, participant.id_offset + ID_COUNT). +""" +import os +from dataclasses import dataclass + +from openalea.encoding3d.codec import from_bytes, to_bytes +from openalea.encoding3d.coupling.channels import (ENDPOINT_VARIABLE, PARTICIPANT_VARIABLE, participant_channel) +from openalea.encoding3d.coupling.messages import PROTOCOL_VERSION, CouplingError, Declaration, Link +from openalea.encoding3d.model import Frame, Update + + +@dataclass +class Turn: + """A turn: *step* (0: the initial state, "init"), the time it starts from and the time it reaches (*until*; + equal at init), and the frame of the participant's inputs (its entity sets, their entities, its inputs, and in + frame.update.removed the entities removed since its last turn).""" + step: int + phase: str + time: float + until: float + frame: Frame + + +class Participant: + def __init__(self, channel, declaration: Declaration, timeout=None): + self.channel, self.declaration, self.timeout = channel, declaration, timeout + self.id_offset = None + self.session = None + self.couplings = [] + self.step = None + self.stopped = None + + @classmethod + def connect(cls, declaration: Declaration, endpoint: str = None, links=(), timeout=None) -> "Participant": + """Join the session at *endpoint* (default: the environment's ENCODING3D_ENDPOINT, set by the coordinator + launching the participant; its name: ENCODING3D_PARTICIPANT if the declaration has none): send the declaration + (and the *links* it proposes), wait for the welcome.""" + endpoint = endpoint or os.environ.get(ENDPOINT_VARIABLE) + if not endpoint: + raise CouplingError(f"no endpoint: give one, or set {ENDPOINT_VARIABLE}") + if not declaration.participant: + declaration.participant = os.environ.get(PARTICIPANT_VARIABLE, "") + participant = cls(participant_channel(endpoint, declaration.participant), declaration.complete(), timeout) + participant.channel.send({"type": "hello", "protocol": PROTOCOL_VERSION, + "declaration": declaration.to_json(), + "links": [link.to_json() if isinstance(link, Link) else link for link in links]}) + header, _ = participant.channel.receive(timeout) + if header["type"] == "stop": + raise CouplingError(f"the coordinator refused {declaration.participant}: {header.get('reason')}") + if header["type"] != "welcome": + raise CouplingError(f"expected a welcome, got {header['type']}") + participant.id_offset = header["idOffset"] + participant.session = header["session"] + if hasattr(participant.channel, "keep_messages"): # the folder transport + participant.channel.keep_messages(header["session"].get("keepMessages", True)) + participant.couplings = header.get("couplings", []) + return participant + + def turns(self): + """The turns, until the coordinator stops the session (self.stopped: its reason).""" + while True: + header, payload = self.channel.receive(self.timeout) + if header["type"] == "stop": + self.stopped = header.get("reason", "") + self.channel.close() + return + if header["type"] != "turn": + raise CouplingError(f"expected a turn, got {header['type']}") + self.step = header["step"] + yield Turn(header["step"], header["phase"], header["time"], header["until"], from_bytes(payload)) + + def send(self, frame: Frame) -> None: + """Send the turn's outputs: *frame* holds, per entity set, the ids and the outputs of the rows sent (and the + topology columns of the entities it adds or changes); frame.update.removed the entities it removed.""" + if frame.update is None: + frame.update = Update(self.declaration.participant) + frame.update.participant = self.declaration.participant + self.channel.send({"type": "update", "step": self.step}, to_bytes(frame)) + + def fail(self, message: str) -> None: + """Tell the coordinator this participant cannot go on (the session stops).""" + self.channel.send({"type": "error", "message": message}) diff --git a/bindings/python/src/openalea/encoding3d/coupling/replay.py b/bindings/python/src/openalea/encoding3d/coupling/replay.py new file mode 100644 index 0000000..a966456 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/replay.py @@ -0,0 +1,145 @@ +""" +A recorded sequence as a participant: a platform's log (a sequence of full frames) providing its variables to a +session without the platform running. It owns its entity sets' topology: an entity appearing between two recorded +frames is added, one disappearing is removed. Between recorded times its values are those of the last frame at or +before the session's time ("hold"), or interpolated linearly for the variables linked with interpolation "linear". +""" +import numpy as np + +from openalea.encoding3d.codec import read, read_index +from openalea.encoding3d.coupling.messages import Declaration, EntitySetSpec, MappingSpec, VariableSpec +from openalea.encoding3d.model import NO_ID, DataStructure, EntitySet, Frame, Mapping, Update +from openalea.encoding3d.ucum import conversion_factor + +_ID_COLUMNS = ("id", "complex", "parent") + + +class Replay: + """A sequence (its index, scene.fspm.json) replayed as participant *name*.""" + + def __init__(self, name: str, index_path: str): + self.name = name + index = read_index(index_path) + self.time_unit = index.get("timeUnit", "d") + self.entries = sorted(index["frames"], key=lambda entry: entry["time"]) + self.paths = [entry["path"] for entry in self.entries] + self.times = np.array([entry["time"] for entry in self.entries], dtype=float) + self._cache = {} + self.id_offset = 0 + self.linear = set() # (data structure, entity set, variable) interpolated + self.needed = None # (data structure, entity set, variable) some link uses (None: all) + self.sent = {} # (data structure, entity set): ids sent at the last turn + self.id_map = {} # recorded id: session id + self.declaration = self._declare() + + def _frame(self, position: int) -> Frame: + if position not in self._cache: # one frame kept, read lazily: only the columns used + for frame in self._cache.values(): + frame.close() + self._cache = {position: read(self.paths[position], lazy=True)} + return self._cache[position] + + def _declare(self) -> Declaration: + """Its declaration: every entity set, variable and mapping of its frames (some appear later than others).""" + declaration = None + sets, variables, mappings = set(), set(), set() + self.own = set() + for position in range(len(self.paths)): + frame = self._frame(position) + if declaration is None: + declaration = Declaration(self.name, platform=frame.writer.get("platform", "replay")) + for data_structure in frame.data_structures: + self.own.add(data_structure.id) + for entity_set in data_structure.entity_sets: + if (data_structure.id, entity_set.name) not in sets: + sets.add((data_structure.id, entity_set.name)) + declaration.entity_sets.append(EntitySetSpec(data_structure.id, entity_set.name, + entity_set.rank, "owns")) + for name in entity_set.variable_names(): + if (data_structure.id, entity_set.name, name) in variables: + continue + variables.add((data_structure.id, entity_set.name, name)) + variable = entity_set.declaration(name) + declaration.variables.append(VariableSpec(data_structure.id, entity_set.name, name, "provides", + variable.unit, variable.standard_name, + variable.kind)) + for mapping in frame.mappings: + if mapping.name not in mappings: + mappings.add(mapping.name) + declaration.mappings.append(MappingSpec(mapping.name, tuple(mapping.source), + tuple(mapping.target))) + return declaration + + def update(self, time: float, time_unit: str) -> Frame: + """The update at the session's *time* (in *time_unit*): the recorded entities and values then.""" + recorded_time = time * conversion_factor(time_unit, self.time_unit) + position = int(np.searchsorted(self.times, recorded_time, side="right")) - 1 + position = max(position, 0) + frame = self._frame(position) + following = None + if self.linear and position + 1 < len(self.times) and recorded_time > self.times[position]: + following = read(self.paths[position + 1], lazy=True) + weight = (recorded_time - self.times[position]) / (self.times[position + 1] - self.times[position]) + structures, removed = [], {} + for data_structure in frame.data_structures: + out = DataStructure(data_structure.id) + for entity_set in data_structure.entity_sets: + columns = {} + for name in entity_set.columns: + if name not in entity_set.reserved and self.needed is not None \ + and (data_structure.id, entity_set.name, name) not in self.needed: + continue # a variable no link uses: not decoded + values = entity_set.columns[name] + if name in _ID_COLUMNS: + values = np.array([self._session_id(int(value)) for value in np.asarray(values).tolist()], + dtype=np.int64) + columns[name] = values + if following is not None: + self._interpolate(data_structure.id, entity_set, following, weight, columns) + out.entity_sets.append(EntitySet(entity_set.name, columns=columns, rank=entity_set.rank, + variables={name: variable for name, variable in + entity_set.variables.items() if name in columns})) + key = (data_structure.id, entity_set.name) + ids = set(columns["id"].tolist()) if "id" in columns else set() + gone = self.sent.get(key, set()) - ids + if gone: + removed[key] = np.array(sorted(gone), dtype=np.int64) + self.sent[key] = ids + structures.append(out) + mappings = [] + for mapping in frame.mappings: # ids renumbered at the ends in its own data structures + columns = dict(mapping.columns) + for end, (data_structure_id, _) in (("source", mapping.source), ("target", mapping.target)): + if data_structure_id in self.own: + columns[end] = np.array([self._session_id(int(value)) for value in columns[end].tolist()], + dtype=np.int64) + mappings.append(Mapping(mapping.name, columns=columns, source=mapping.source, target=mapping.target, + method=mapping.method)) + return Frame(data_structures=structures, mappings=mappings, update=Update(self.name, removed)) + + def _session_id(self, recorded: int) -> int: + """The session id of a recorded id: the replay's ids renumbered from its offset, in order of appearance.""" + if recorded == NO_ID: + return NO_ID + if recorded not in self.id_map: + self.id_map[recorded] = self.id_offset + len(self.id_map) + return self.id_map[recorded] + + def _interpolate(self, data_structure_id, entity_set, following, weight, columns) -> None: + try: + later = following.data_structure(data_structure_id).entity_set(entity_set.name) + except KeyError: + return + later_rows = {identifier: row for row, identifier in enumerate(later.columns["id"].tolist())} + rows = np.array([later_rows.get(identifier, -1) for identifier in entity_set.columns["id"].tolist()]) + found = rows >= 0 + for name in entity_set.variable_names(): + if (data_structure_id, entity_set.name, name) not in self.linear or name not in later.columns \ + or name not in columns: + continue + values = np.asarray(columns[name]) + if values.dtype.kind != "f": + continue + values = values.copy() + values[found] = (1 - weight) * values[found] + weight * np.asarray(later.columns[name])[rows[found]] + columns[name] = values diff --git a/bindings/python/src/openalea/encoding3d/coupling/resolution.py b/bindings/python/src/openalea/encoding3d/coupling/resolution.py new file mode 100644 index 0000000..ddb9a5f --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/resolution.py @@ -0,0 +1,196 @@ +""" +A session's resolution: from the participants' declarations and the links, each input's provider (Coupling), the +stages, and the checks that make a session run (one provider per input, convertible units, one owner per entity +set, no two participants changing one entity set's topology in the same stage). +""" +from dataclasses import dataclass +from typing import Optional + +from openalea.encoding3d.coupling.messages import CouplingError, Declaration, Link, VariableSpec +from openalea.encoding3d.ucum import UnitError, conversion_factor + + +@dataclass +class Coupling: + """An input and its provider: *factor* converts the source's unit to the input's (times the link's factor); + *aggregation* changes its scale (None: the same entity set); *how*: "link", "standard name" or "name".""" + receiver: str + variable: VariableSpec + provider: Optional[str] + source: Optional[VariableSpec] + factor: float = 1. + aggregation: Optional[str] = None + interpolation: str = "hold" + how: str = "link" + mapping: Optional[str] = None + reverse: bool = False # the mapping read from its target to its source + + def to_json(self) -> dict: + value = {"variable": self.variable.name, "entitySet": self.variable.entity_set, + "dataStructure": self.variable.data_structure, "provider": self.provider, "how": self.how} + if self.source is not None: + value.update(source=self.source.name, sourceEntitySet=self.source.entity_set, factor=self.factor) + if self.aggregation: + value["aggregation"] = self.aggregation + if self.interpolation != "hold": + value["interpolation"] = self.interpolation + if self.mapping: + value["mapping"] = self.mapping + return value + + +def ranks(declarations: dict) -> dict: + """{(data structure, entity set): rank}, the same in every declaration that gives one.""" + found = {} + for declaration in declarations.values(): + for entity_set in declaration.entity_sets: + key = (entity_set.data_structure, entity_set.name) + if entity_set.rank is None: + continue + if found.setdefault(key, entity_set.rank) != entity_set.rank: + raise CouplingError(f"entity set {key[0]}/{key[1]}: ranks {found[key]} and {entity_set.rank} declared") + return found + + +def resolve(declarations: dict, links=()) -> list: + """The couplings of every input of *declarations* ({participant: Declaration}), from *links* (Link) first, then + by standard name, then by name; CouplingError when an input has none (unless optional) or several.""" + known_ranks = ranks(declarations) + mappings = {} + for declaration in declarations.values(): + for mapping in declaration.mappings: + if mapping.name in mappings: + raise CouplingError(f"mapping {mapping.name}: provided by {mappings[mapping.name][0]} and " + f"{declaration.participant}") + mappings[mapping.name] = (declaration.participant, mapping) + owners = {} + for declaration in declarations.values(): + for entity_set in declaration.entity_sets: + if entity_set.rights == "owns": + key = (entity_set.data_structure, entity_set.name) + if key in owners: + raise CouplingError(f"entity set {key[0]}/{key[1]}: owned by {owners[key]} and " + f"{declaration.participant}") + owners[key] = declaration.participant + by_receiver = {} + for link in links: + if link.receiver not in declarations: + raise CouplingError(f"link to {link.receiver}.{link.variable}: no participant {link.receiver}") + if link.provider not in declarations: + raise CouplingError(f"link to {link.receiver}.{link.variable}: no participant {link.provider}") + by_receiver.setdefault(link.receiver, []).append(link) + + couplings = [] + for receiver, declaration in declarations.items(): + for variable in declaration.requires(): + links_here = [link for link in by_receiver.get(receiver, []) if link.variable == variable.name + and link.entity_set in (None, variable.entity_set)] + if len(links_here) > 1: + raise CouplingError(f"{receiver}.{variable.name}: {len(links_here)} links") + if links_here: + link = links_here[0] + sources = [source for source in declarations[link.provider].provides() if source.name == link.source + and link.source_entity_set in (None, source.entity_set)] + if len(sources) != 1: + raise CouplingError(f"link {receiver}.{variable.name} <- {link.provider}.{link.source}: " + f"{len(sources)} such outputs of {link.provider}") + couplings.append(_coupling(receiver, variable, link.provider, sources[0], known_ranks, "link", + link.factor, link.aggregation, link.interpolation, + mappings.get(link.mapping, (None, link.mapping))[1])) + continue + coupling = None + for how, same in (("standard name", lambda source: variable.standard_name is not None + and source.standard_name == variable.standard_name), + ("name", lambda source: source.name == variable.name)): + candidates = [(name, source) for name, other in declarations.items() if name != receiver + for source in other.provides() + if source.data_structure == variable.data_structure and same(source)] + if len(candidates) > 1: # the same entity set first + candidates = [(name, source) for name, source in candidates + if source.entity_set == variable.entity_set] or candidates + if len(candidates) > 1: + raise CouplingError(f"{receiver}.{variable.name}: provided by several outputs (by {how}): " + f"{[f'{name}.{source.name}' for name, source in candidates]}; link it") + if candidates: + coupling = _coupling(receiver, variable, *candidates[0], known_ranks, how) + break + if coupling is None: + if not variable.optional: + raise CouplingError(f"{receiver}.{variable.name}: no participant provides it (link it, or " + "declare it optional)") + coupling = Coupling(receiver, variable, None, None, how="not provided") + couplings.append(coupling) + return couplings + + +def _coupling(receiver, variable, provider, source, known_ranks, how, factor=1., aggregation=None, + interpolation="hold", mapping=None) -> Coupling: + where = f"{receiver}.{variable.name} <- {provider}.{source.name}" + try: + factor = factor * conversion_factor(source.unit, variable.unit) + except UnitError as error: + raise CouplingError(f"{where}: {error}") from None + if mapping is not None or source.data_structure != variable.data_structure: + if mapping is None or isinstance(mapping, str): + raise CouplingError(f"{where}: different data structures ({source.data_structure}, " + f"{variable.data_structure}) need a mapping a participant provides" + + (f" ({mapping} is none)" if mapping else "")) + if (mapping.source, mapping.target) == (source.where, variable.where): + reverse = False + elif (mapping.target, mapping.source) == (source.where, variable.where): + reverse = True + else: + raise CouplingError(f"{where}: mapping {mapping.name} joins {mapping.source} and {mapping.target}, not " + f"{source.where} and {variable.where}") + aggregation = aggregation or ("sum" if source.kind == "extensive" else "mean") + if aggregation not in ("sum", "mean"): + raise CouplingError(f"{where}: through a mapping, 'sum' or 'mean'") + return Coupling(receiver, variable, provider, source, factor, aggregation, interpolation, how, mapping.name, + reverse) + if source.entity_set != variable.entity_set: + source_rank = known_ranks.get(source.where) + target_rank = known_ranks.get(variable.where) + if source_rank is None or target_rank is None: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: the ranks of " + f"{source.entity_set} and {variable.entity_set} are needed (declare them)") + if source_rank > target_rank: + aggregation = aggregation or ("sum" if source.kind == "extensive" else "mean") + if aggregation == "copy": + raise CouplingError(f"{receiver}.{variable.name}: 'copy' goes from a coarser entity set") + else: + if aggregation not in (None, "copy"): + raise CouplingError(f"{receiver}.{variable.name}: '{aggregation}' goes from a finer entity set") + if source.kind == "extensive" and aggregation is None: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: an extensive " + "variable copied to finer entities counts it several times; link it with " + "aggregation 'copy' if meant") + aggregation = "copy" + elif aggregation is not None: + raise CouplingError(f"{receiver}.{variable.name}: aggregation '{aggregation}' on the same entity set") + return Coupling(receiver, variable, provider, source, factor, aggregation, interpolation, how) + + +def stages(declarations: dict, given=None) -> list: + """The stages: *given* (lists of participant names), else the owners, then the refiners, then the others; and + the check that no stage holds two participants changing one entity set's topology.""" + if given is None: + groups = {"owns": [], "refines": [], "follows": []} + for name, declaration in declarations.items(): + rights = {entity_set.rights for entity_set in declaration.entity_sets} + groups["owns" if "owns" in rights else "refines" if "refines" in rights else "follows"].append(name) + given = [group for group in groups.values() if group] + given = [list(stage) for stage in given] + listed = [name for stage in given for name in stage] + if sorted(listed) != sorted(declarations): + raise CouplingError(f"stages {given}: each participant ({sorted(declarations)}) exactly once") + for stage in given: + changing = {} + for name in stage: + for entity_set in declarations[name].entity_sets: + if entity_set.rights != "follows": + key = (entity_set.data_structure, entity_set.name) + if key in changing: + raise CouplingError(f"stage {stage}: {changing[key]} and {name} both change " + f"{key[0]}/{key[1]}'s topology") + changing[key] = name + return given diff --git a/bindings/python/src/openalea/encoding3d/coupling/store.py b/bindings/python/src/openalea/encoding3d/coupling/store.py new file mode 100644 index 0000000..6b2ca8f --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/coupling/store.py @@ -0,0 +1,465 @@ +""" +The session's structure, kept by the coordinator: per data structure and entity set, the entities (their ids and +topology, and which participant created each), and the values each participant provided. Updates are merged into +it, checked against the participants' rights; each participant's inputs are read from it, converted. +""" +import numpy as np + +from openalea.encoding3d.coupling.messages import CouplingError, ID_COUNT +from openalea.encoding3d.model import (BASE_SCALES, NO_ID, DataStructure, EntitySet, Frame, Update, Variable) + +TOPOLOGY = ("label", "complex", "parent", "edge_type", "sibling_order") + + +class _Column: + """A provider's values on an entity set's rows, with the rows it has given a value (*present*).""" + + def __init__(self, values: np.ndarray, size: int): + values = np.asarray(values) + if values.dtype.kind in "OUS": + dtype = object + elif values.dtype.kind == "b": + dtype = bool + elif values.dtype.kind in "iu": + dtype = np.int64 + else: + dtype = np.float64 + width = values.shape[1:] if values.ndim > 1 else () + self.values = np.zeros((size,) + width, dtype=dtype) + if dtype == np.float64: + self.values[:] = np.nan + self.present = np.zeros(size, dtype=bool) + + def place(self, rows: np.ndarray, values) -> None: + mask = np.ma.getmaskarray(values) if np.ma.isMaskedArray(values) else None + data = np.ma.getdata(values) if mask is not None else np.asarray(values) + if mask is not None and mask.ndim > 1: + mask = mask.any(axis=tuple(range(1, mask.ndim))) + if self.values.dtype == np.int64 and data.dtype.kind == "f": + self.values = self.values.astype(np.float64) # an integer column now holding floats + self.values[rows] = data + self.present[rows] = True if mask is None else ~mask + + def keep(self, keep: np.ndarray) -> None: + self.values, self.present = self.values[keep], self.present[keep] + + def grow(self, count: int) -> None: + extra = np.zeros((count,) + self.values.shape[1:], dtype=self.values.dtype) + if self.values.dtype == np.float64: + extra[:] = np.nan + self.values = np.concatenate([self.values, extra]) + self.present = np.concatenate([self.present, np.zeros(count, dtype=bool)]) + + +class EntitySetStore: + """An entity set's entities (ids, topology, creators) and the values the participants provided on them.""" + + def __init__(self, data_structure: str, name: str, rank): + self.data_structure, self.name, self.rank = data_structure, name, rank + self.ids = np.zeros(0, dtype=np.int64) + self._sorted = None # (ids sorted, their rows): the lookups by id + self.topology = {"label": np.zeros(0, dtype=object), "complex": np.zeros(0, dtype=np.int64), + "parent": np.zeros(0, dtype=np.int64), "edge_type": np.zeros(0, dtype=object), + "sibling_order": np.zeros(0, dtype=np.int64)} + self.creator = np.zeros(0, dtype=object) + self.columns = {} # (participant, variable): _Column + self.declarations = {} # (participant, variable): Variable + + def __len__(self): + return self.ids.size + + def _index(self) -> tuple: + if self._sorted is None: + order = np.argsort(self.ids, kind="stable") + self._sorted = (self.ids[order], order) + return self._sorted + + def rows(self, ids, strict: bool = True) -> np.ndarray: + """The rows of *ids* (-1 for an id not here; KeyError if *strict*).""" + ids = np.asarray(ids, dtype=np.int64).reshape(-1) + keys, order = self._index() + if keys.size == 0: + found, rows = np.zeros(ids.size, dtype=bool), np.full(ids.size, -1, dtype=np.int64) + else: + position = np.minimum(np.searchsorted(keys, ids), keys.size - 1) + found = keys[position] == ids + rows = np.where(found, order[position], -1) + if strict and not found.all(): + raise KeyError(f"ids not in {self.data_structure}/{self.name}: {ids[~found][:5].tolist()}") + return rows + + def contains(self, ids) -> np.ndarray: + return self.rows(ids, strict=False) >= 0 + + def add(self, ids: np.ndarray, topology: dict, creator: str) -> None: + count = ids.size + self.ids = np.concatenate([self.ids, ids]) + for name, default in (("label", self.name), ("complex", NO_ID), ("parent", NO_ID), ("edge_type", "none"), + ("sibling_order", NO_ID)): # an entity without a label: its entity set's name + values = topology.get(name) + column = self.topology[name] + added = np.full(count, default, dtype=column.dtype) if values is None else np.asarray(values, + dtype=column.dtype) + self.topology[name] = np.concatenate([column, added]) + self.creator = np.concatenate([self.creator, np.full(count, creator, dtype=object)]) + for column in self.columns.values(): + column.grow(count) + self._sorted = None + + def remove(self, ids) -> None: + keep = ~np.isin(self.ids, np.asarray(list(ids), dtype=np.int64)) + self.ids = self.ids[keep] + for name in self.topology: + self.topology[name] = self.topology[name][keep] + self.creator = self.creator[keep] + for column in self.columns.values(): + column.keep(keep) + self._sorted = None + + def place(self, participant: str, name: str, rows: np.ndarray, values, variable) -> None: + key = (participant, name) + if key not in self.columns: + self.columns[key] = _Column(np.ma.getdata(values) if np.ma.isMaskedArray(values) else values, + self.ids.size) + if variable is not None: + self.declarations[key] = variable + self.columns[key].place(rows, values) + + +class SessionStore: + """The session's data structures (DataStructureStore: {entity set name: EntitySetStore}), which entity set each + id is in, and the log of removed entities (each participant is sent those removed since its last turn).""" + + def __init__(self, ranks: dict): + self.ranks = dict(ranks) # (data structure, entity set): rank + self.sets = {} # (data structure, entity set): EntitySetStore + self.set_of = {} # (data structure, id): entity set name + self.removed_log = [] # (data structure, entity set, ids) + self.removed_ids = set() # (data structure, id) of every entity removed + self.mappings = {} # name: the last Mapping its provider sent + + def entity_set(self, data_structure: str, name: str, rank=None) -> EntitySetStore: + key = (data_structure, name) + if key not in self.sets: + rank = self.ranks.get(key, rank) + if rank is None and name in BASE_SCALES: + rank = BASE_SCALES.index(name) + 1 + self.ranks[key] = rank + self.sets[key] = EntitySetStore(data_structure, name, rank) + return self.sets[key] + + # ── merging a participant's update ───────────────────────────────────────────── + def merge(self, participant: str, declaration, frame: Frame, id_offset: int) -> None: + """Merge *participant*'s update: its new entities, topology changes and removals (within its rights), and its + outputs (only the variables it provides).""" + where = f"{participant}'s update" + provided = {(variable.data_structure, variable.entity_set, variable.name): variable + for variable in declaration.provides()} + for data_structure in frame.data_structures: + for entity_set in sorted(data_structure.entity_sets, + key=lambda item: (item.rank if item.rank is not None else 0)): + self._merge_set(participant, declaration, data_structure.id, entity_set, provided, id_offset, where) + declared = {mapping.name: mapping for mapping in getattr(declaration, "mappings", [])} + for mapping in frame.mappings: + spec = declared.get(mapping.name) + if spec is None: + raise CouplingError(f"{where}: mapping {mapping.name} is not one it declared") + if (tuple(mapping.source), tuple(mapping.target)) != (spec.source, spec.target): + raise CouplingError(f"{where}: mapping {mapping.name} joins {mapping.source} and {mapping.target}, " + f"declared {spec.source} and {spec.target}") + self.mappings[mapping.name] = mapping + if frame.update is not None: + for (data_structure, set_name), ids in sorted(frame.update.removed.items(), + key=lambda item: self.ranks.get(item[0]) or 0): + self._remove(participant, declaration, data_structure, set_name, ids, where) + + def _merge_set(self, participant, declaration, data_structure, entity_set, provided, id_offset, where): + key = (data_structure, entity_set.name) + store = self.entity_set(data_structure, entity_set.name, entity_set.rank) + if "id" not in entity_set.columns: + raise CouplingError(f"{where}: {data_structure}/{entity_set.name} has no 'id' column") + ids = np.asarray(entity_set.columns["id"], dtype=np.int64) + rights = declaration.rights(*key) + known = store.contains(ids) + new = ids[~known] + if new.size: + if rights == "follows": + raise CouplingError(f"{where}: {new.size} new entities in {data_structure}/{entity_set.name}, which " + f"it follows (declare 'owns' or 'refines' to add entities)") + outside = new[(new < id_offset) | (new >= id_offset + ID_COUNT)] + if outside.size: + raise CouplingError(f"{where}: ids {outside[:5].tolist()} outside its range [{id_offset}, " + f"{id_offset + ID_COUNT})") + elsewhere = [int(identifier) for identifier in new.tolist() if (data_structure, int(identifier)) + in self.set_of] + if elsewhere: + raise CouplingError(f"{where}: ids {elsewhere[:5]} already in another entity set") + topology = {name: np.asarray(entity_set.columns[name])[~known] for name in TOPOLOGY + if name in entity_set.columns} + store.add(new, topology, participant) + for identifier in new.tolist(): + self.set_of[(data_structure, identifier)] = entity_set.name + rows = store.rows(ids) + changes = [name for name in TOPOLOGY if name in entity_set.columns] + if known.any() and changes: # topology of existing entities: its own, or its set's + for name in changes: + given = np.asarray(entity_set.columns[name])[known] + current = store.topology[name][rows[known]] + differs = given != current + if differs.any(): + allowed = (rights == "owns") | (store.creator[rows[known]] == participant) + if (differs & ~allowed).any(): + raise CouplingError(f"{where}: changes the {name} of entities of {data_structure}/" + f"{entity_set.name} it did not create") + store.topology[name][rows[known][differs]] = given[differs].astype(store.topology[name].dtype) + for name in ("complex", "parent"): + if name in entity_set.columns: + values = np.asarray(entity_set.columns[name], dtype=np.int64) + values = values[values != NO_ID] + found = np.zeros(values.size, dtype=bool) + for (other_structure, _), other in self.sets.items(): + if other_structure == data_structure and values.size: + found |= other.contains(values) + missing = values[~found].tolist() + if missing: + raise CouplingError(f"{where}: {data_structure}/{entity_set.name}.{name} refers to unknown " + f"entities {missing[:5]}") + for name in entity_set.variable_names(): + variable = provided.get((data_structure, entity_set.name, name)) + if variable is None: + raise CouplingError(f"{where}: '{name}' on {data_structure}/{entity_set.name} is not one of its " + "declared outputs") + store.place(participant, name, rows, entity_set.columns[name], + Variable(unit=variable.unit, standard_name=variable.standard_name, kind=variable.kind)) + + def _remove(self, participant, declaration, data_structure, set_name, ids, where): + store = self.sets.get((data_structure, set_name)) + ids = [int(identifier) for identifier in np.asarray(ids).tolist() + if (data_structure, int(identifier)) not in self.removed_ids] # e.g. with their complex already + if not ids: + return + if store is None or not store.contains(ids).all(): + raise CouplingError(f"{where}: removes entities not in {data_structure}/{set_name}") + if declaration.rights(data_structure, set_name) != "owns": + created = store.creator[store.rows(ids)] + if (created != participant).any(): + raise CouplingError(f"{where}: removes entities of {data_structure}/{set_name} it did not create") + self.remove(data_structure, set_name, ids) + + def remove(self, data_structure: str, set_name: str, ids) -> None: + """Remove *ids* and, down the finer entity sets, the entities whose complex is removed.""" + removed = set(ids) + pending = [(set_name, removed)] + while pending: + name, identifiers = pending.pop() + store = self.sets[(data_structure, name)] + store.remove(identifiers) + self.removed_log.append((data_structure, name, np.array(sorted(identifiers), dtype=np.int64))) + for identifier in identifiers: + self.set_of.pop((data_structure, identifier), None) + self.removed_ids.add((data_structure, identifier)) + for (other_structure, other_name), other in self.sets.items(): + if other_structure != data_structure or not len(other): + continue + parent = other.topology["parent"] + orphans = np.isin(parent, list(identifiers)) + if orphans.any(): # successors of removed entities become roots + parent[orphans] = NO_ID + other.topology["edge_type"][orphans] = "none" + inside = other.ids[np.isin(other.topology["complex"], list(identifiers))] + if inside.size: + pending.append((other_name, set(inside.tolist()))) + + # ── a participant's inputs ───────────────────────────────────────────────────── + def ancestors(self, data_structure: str, ids: np.ndarray, target: str) -> np.ndarray: + """Each id's complex (or complex's complex, …) in the entity set *target*, NO_ID if none.""" + ids = np.asarray(ids, dtype=np.int64) + found = np.full(ids.size, NO_ID, dtype=np.int64) + current, active = ids.copy(), ids != NO_ID + stores = [(name, store) for (structure, name), store in self.sets.items() if structure == data_structure] + for _ in range(len(stores) + 1): # one scale up per pass + if not active.any(): + break + snapshot, moved = current.copy(), np.zeros(ids.size, dtype=bool) + for name, store in stores: + rows = store.rows(snapshot, strict=False) + here = active & (rows >= 0) + if name == target: + found[here] = snapshot[here] + active[here] = False + else: + current[here] = store.topology["complex"][rows[here]] + moved |= here + active &= moved & (current != NO_ID) + return found + + def values(self, coupling) -> tuple: + """The input of *coupling* on its entity set's rows: (values, missing mask).""" + variable, source = coupling.variable, coupling.source + target = self.entity_set(variable.data_structure, variable.entity_set) + count = len(target) + origin = self.sets.get((source.data_structure, source.entity_set)) + column = None if origin is None else origin.columns.get((coupling.provider, source.name)) + if column is None: + return np.full(count, np.nan), np.ones(count, dtype=bool) + values, present = column.values, column.present + if coupling.mapping is not None: + return self._through_mapping(coupling, origin, values, present, target) + if coupling.aggregation is None: + rows = origin.rows(target.ids, strict=False) + found = rows >= 0 + out = _empty(values, count) + out[found] = values[rows[found]] + missing = ~found + missing[found] |= ~present[rows[found]] + elif coupling.aggregation == "copy": + above = self.ancestors(variable.data_structure, target.ids, source.entity_set) + rows = origin.rows(above, strict=False) + found = rows >= 0 + out = _empty(values, count) + out[found] = values[rows[found]] + missing = ~found + missing[found] |= ~present[rows[found]] + else: + if values.dtype.kind not in "fi": + raise CouplingError(f"{coupling.receiver}.{variable.name}: {coupling.aggregation} of non-numeric " + f"values") + above = self.ancestors(variable.data_structure, origin.ids, variable.entity_set) + target_rows = target.rows(above, strict=False) + used = (target_rows >= 0) & present + target_rows, x = target_rows[used], values[used].astype(float) + counts = np.bincount(target_rows, minlength=count) + missing = counts == 0 + if coupling.aggregation in ("sum", "mean"): + out = np.bincount(target_rows, weights=x, minlength=count) + if coupling.aggregation == "mean": + out = np.divide(out, counts, out=np.zeros(count), where=counts > 0) + else: + start = -np.inf if coupling.aggregation == "max" else np.inf + out = np.full(count, start) + (np.maximum if coupling.aggregation == "max" else np.minimum).at(out, target_rows, x) + out = np.where(missing, np.nan, out) + if coupling.factor != 1. and out.dtype.kind in "fi": + out = out.astype(np.float64) * coupling.factor + return out, missing + + def _through_mapping(self, coupling, origin, values, present, target) -> tuple: + """An input from another data structure through a mapping: "sum" shares each source's value among its + targets in proportion to their weights; "mean" gives each target its sources' weighted mean.""" + count = len(target) + out, missing = np.full(count, np.nan), np.ones(count, dtype=bool) + mapping = self.mappings.get(coupling.mapping) + if mapping is None: + return out, missing + if values.dtype.kind not in "fi": + raise CouplingError(f"{coupling.receiver}.{coupling.variable.name}: non-numeric values through a " + "mapping") + sources, targets = np.asarray(mapping.columns["source"]), np.asarray(mapping.columns["target"]) + if coupling.reverse: + sources, targets = targets, sources + weights = np.asarray(mapping.columns["weight"], dtype=float) + source_rows = origin.rows(sources, strict=False) + target_rows = target.rows(targets, strict=False) + used = (source_rows >= 0) & (target_rows >= 0) + used[used] &= present[source_rows[used]] + source_rows, target_rows, weights = source_rows[used], target_rows[used], weights[used] + x = values[source_rows].astype(float) + if coupling.aggregation == "sum": # each source's amount shared among its targets + totals = np.bincount(source_rows, weights=weights, minlength=len(origin)) + shares = np.divide(weights, totals[source_rows], out=np.zeros_like(weights), + where=totals[source_rows] > 0) + out_values = np.bincount(target_rows, weights=x * shares, minlength=count) + else: # each target's weighted mean + weighted = np.bincount(target_rows, weights=x * weights, minlength=count) + totals = np.bincount(target_rows, weights=weights, minlength=count) + out_values = np.divide(weighted, totals, out=np.full(count, np.nan), where=totals > 0) + reached = np.bincount(target_rows, minlength=count) > 0 + out[reached], missing[reached] = out_values[reached], False + return out * coupling.factor, missing + + def inputs(self, receiver: str, declaration, couplings, removed_since: int, time: float, time_unit: str, + step: int) -> Frame: + """*receiver*'s turn frame: the entity sets it works on (all their entities and topology) with its inputs, + and the entities removed since its last turn (the removed log from *removed_since*).""" + wanted = [(entity_set.data_structure, entity_set.name) for entity_set in declaration.entity_sets] + structures = {} + for key in wanted: + data_structure, name = key + store = self.sets.get(key) + if store is None: + continue + columns = {"id": store.ids.copy()} + for column in TOPOLOGY: + values = store.topology[column] + if column == "sibling_order" and not (values != NO_ID).any(): + continue + columns[column] = values.copy() + entity_set = EntitySet(name, columns=columns, rank=store.rank, + scale=name if name in BASE_SCALES and store.rank == BASE_SCALES.index(name) + 1 + else None) + for coupling in couplings: + if coupling.receiver != receiver or coupling.variable.where != key or coupling.source is None: + continue + values, missing = self.values(coupling) + entity_set.columns[coupling.variable.name] = _with_missing(values, missing) + entity_set.variables[coupling.variable.name] = Variable( + unit=coupling.variable.unit, standard_name=coupling.variable.standard_name, + kind=coupling.variable.kind, role="input") + structures.setdefault(data_structure, DataStructure(data_structure)).entity_sets.append(entity_set) + removed = {} + for data_structure, name, ids in self.removed_log[removed_since:]: + if (data_structure, name) in wanted: + previous = removed.get((data_structure, name), np.zeros(0, dtype=np.int64)) + removed[(data_structure, name)] = np.union1d(previous, ids) + return Frame(data_structures=list(structures.values()), time=time, time_unit=time_unit, step=step, + update=Update("coordinator", removed)) + + def frame(self, time: float, time_unit: str, step: int) -> Frame: + """The whole session as a frame (its record): every entity set, each participant's outputs (named + participant__variable when two participants provide one name).""" + structures = {} + for (data_structure, name), store in sorted(self.sets.items(), key=lambda item: (item[0][0], + item[1].rank or 0)): + columns = {"id": store.ids.copy()} + for column in TOPOLOGY: + if column == "sibling_order" and not (store.topology[column] != NO_ID).any(): + continue + columns[column] = store.topology[column].copy() + counts = {} + for _, variable in store.columns: + counts[variable] = counts.get(variable, 0) + 1 + entity_set = EntitySet(name, columns=columns, rank=store.rank, + scale=name if name in BASE_SCALES and store.rank == BASE_SCALES.index(name) + 1 + else None) + for (participant, variable), column in store.columns.items(): + column_name = variable if counts[variable] == 1 else f"{participant}__{variable}" + entity_set.columns[column_name] = _with_missing(column.values, ~column.present) + if (participant, variable) in store.declarations: + entity_set.variables[column_name] = store.declarations[(participant, variable)] + structures.setdefault(data_structure, DataStructure(data_structure)).entity_sets.append(entity_set) + return Frame(data_structures=list(structures.values()), mappings=list(self.mappings.values()), time=time, + time_unit=time_unit, step=step, + writer={"platform": "encoding3d", "adapter": "openalea.encoding3d.coupling"}) + + +def _empty(values: np.ndarray, count: int) -> np.ndarray: + out = np.zeros((count,) + values.shape[1:], dtype=values.dtype) + if values.dtype.kind == "f": + out[:] = np.nan + return out + + +def _with_missing(values: np.ndarray, missing: np.ndarray): + """The column of *values*, its *missing* rows as the format writes them: NaN, masked, None.""" + if not missing.any(): + return values + if values.dtype.kind == "f": + values = values.copy() + values[missing] = np.nan + return values + if values.dtype == object: + values = values.copy() + values[missing] = None + return values + mask = np.broadcast_to(missing.reshape((-1,) + (1,) * (values.ndim - 1)), values.shape) + return np.ma.MaskedArray(values, mask=mask) diff --git a/bindings/python/src/openalea/fspm_tf/display.py b/bindings/python/src/openalea/encoding3d/display.py similarity index 92% rename from bindings/python/src/openalea/fspm_tf/display.py rename to bindings/python/src/openalea/encoding3d/display.py index 112a547..f28b1dd 100644 --- a/bindings/python/src/openalea/fspm_tf/display.py +++ b/bindings/python/src/openalea/encoding3d/display.py @@ -1,6 +1,6 @@ """ Display copies: float32 per-vertex copies of variables, for viewers that do not read the property tables -(ParaView without the fspm-TF plugin, Blender, three.js). They are added from the tables, flagged in FSPM_scene, +(ParaView without the encoding3d plugin, Blender, three.js). They are added from the tables, flagged in FSPM_scene, and never read as state:: add_display(frame, ["water_potential"]) each mesh gets a _WATER_POTENTIAL attribute @@ -8,7 +8,7 @@ """ import numpy as np -from openalea.fspm_tf.model import Frame, NO_ID +from openalea.encoding3d.model import Frame, NO_ID def add_display(frame: Frame, names, entity_set: str = None) -> Frame: diff --git a/bindings/python/src/openalea/fspm_tf/dump.py b/bindings/python/src/openalea/encoding3d/dump.py similarity index 69% rename from bindings/python/src/openalea/fspm_tf/dump.py rename to bindings/python/src/openalea/encoding3d/dump.py index fcda7f6..295b16b 100644 --- a/bindings/python/src/openalea/fspm_tf/dump.py +++ b/bindings/python/src/openalea/encoding3d/dump.py @@ -3,14 +3,14 @@ conformance files' expected values (``*.expected.json``) are written with it, so that a reader in any language can compare what it reads with them:: - frame_to_dict(frame) -> dict NaN as null, ±∞ as "Infinity" / "-Infinity"; floats as the shortest decimal - giving the same double + frame_to_dict(frame) -> dict NaN and missing values as null, ±∞ as "Infinity" / "-Infinity"; floats as + the shortest decimal giving the same double """ import math import numpy as np -from openalea.fspm_tf.model import Frame, Table +from openalea.encoding3d.model import Frame, Table def _value(value): @@ -26,6 +26,12 @@ def _value(value): def _column(values: np.ndarray): + if np.ma.isMaskedArray(values): # missing values: null + missing = np.ma.getmaskarray(values) + if values.ndim == 2: + return [[None if gone else _value(item) for item, gone in zip(row, row_missing)] + for row, row_missing in zip(values.data, missing)] + return [None if gone else _value(item) for item, gone in zip(values.data, missing)] if values.ndim == 2: return [[_value(item) for item in row] for row in values] return [_value(item) for item in values] @@ -44,6 +50,8 @@ def _table(table: Table) -> dict: for key in ("references", "doi"): if getattr(variable, key): declaration[key] = list(getattr(variable, key)) + if variable.format is not None: + declaration["format"] = variable.format declarations[name] = declaration if declarations: entry["variables"] = declarations @@ -55,13 +63,16 @@ def frame_to_dict(frame: Frame) -> dict: for data_structure in frame.data_structures: entry = {"id": data_structure.id, "kind": data_structure.kind, "origin": data_structure.origin.tolist(), - "entity_sets": [dict(_table(entity_set), rank=entity_set.rank) + "entity_sets": [dict(_table(entity_set), rank=entity_set.rank, + **({"scale": entity_set.scale} if entity_set.scale is not None else {})) for entity_set in data_structure.entity_sets]} for key in ("platform", "model"): if getattr(data_structure, key) is not None: entry[key] = getattr(data_structure, key) if data_structure.extras: entry["extras"] = data_structure.extras + if data_structure.labels: + entry["labels"] = dict(data_structure.labels) if data_structure.relations: entry["relations"] = [dict(_table(relation), source_set=relation.source_set, target_set=relation.target_set) for relation in data_structure.relations] @@ -83,7 +94,12 @@ def frame_to_dict(frame: Frame) -> dict: for name, values in mesh.display.items()}} if mesh.display else {})} for mesh in data_structure.meshes] data_structures.append(entry) - return {"time": frame.time, "time_unit": frame.time_unit, "step": frame.step, - "data_structures": data_structures, - "mappings": [dict(_table(mapping), source=list(mapping.source), target=list(mapping.target), - method=mapping.method) for mapping in frame.mappings]} + out = {"time": frame.time, "time_unit": frame.time_unit, "step": frame.step, + "data_structures": data_structures, + "mappings": [dict(_table(mapping), source=list(mapping.source), target=list(mapping.target), + method=mapping.method) for mapping in frame.mappings]} + if frame.update is not None: # a partial frame: its sender and removed entities + out["update"] = {"participant": frame.update.participant, + "removed": [{"data_structure": key[0], "entity_set": key[1], "ids": ids.tolist()} + for key, ids in sorted(frame.update.removed.items())]} + return out diff --git a/bindings/python/src/openalea/fspm_tf/glb.py b/bindings/python/src/openalea/encoding3d/glb.py similarity index 91% rename from bindings/python/src/openalea/fspm_tf/glb.py rename to bindings/python/src/openalea/encoding3d/glb.py index 69ac098..84ba00e 100644 --- a/bindings/python/src/openalea/fspm_tf/glb.py +++ b/bindings/python/src/openalea/encoding3d/glb.py @@ -24,7 +24,8 @@ def pack(document: dict, binary: bytes) -> bytes: def unpack(data: bytes) -> tuple: - """(JSON document, binary buffer) of a GLB file.""" + """(JSON document, binary buffer) of a GLB file (*data*: bytes, or a memoryview: the binary chunk then a view of + it, not a copy).""" if len(data) < 20: raise ValueError("not a GLB file: too short") magic, version, length = struct.unpack_from(" tuple: chunk_length, chunk_type = struct.unpack_from(" bytes: def view_bytes(document: dict, binary: bytes, index: int) -> bytes: view = document["bufferViews"][index] if view.get("buffer", 0) != 0: - raise ValueError("fspm-TF files keep every buffer view in the GLB binary chunk") + raise ValueError("encoding3d files keep every buffer view in the GLB binary chunk") start = view.get("byteOffset", 0) return binary[start:start + view["byteLength"]] diff --git a/bindings/python/src/openalea/fspm_tf/model.py b/bindings/python/src/openalea/encoding3d/model.py similarity index 58% rename from bindings/python/src/openalea/fspm_tf/model.py rename to bindings/python/src/openalea/encoding3d/model.py index b4aa0c2..51bfab8 100644 --- a/bindings/python/src/openalea/fspm_tf/model.py +++ b/bindings/python/src/openalea/encoding3d/model.py @@ -1,5 +1,5 @@ """ -The fspm-TF data model in Python: plain dataclasses holding numpy columns:: +The encoding3d data model in Python: plain dataclasses holding numpy columns:: Frame one time step of a scene DataStructure one plant population's graph, or one environment model's grid @@ -9,10 +9,15 @@ Grid a regular or adaptive grid's layout Mesh triangles, each vertex carrying the row of its entity in one or several entity sets Mapping source entity ↔ target entity weights between two data structures + Update a partial frame's sender and removed entities (a coupling session's message) Every table holds columns of equal length (its rows), and declarations (Variable) for its variables. The columns reserved by the specification (RESERVED) are the structure: ids, the MTG relations, a relation's or a mapping's ends. + +Missing values: NaN in float columns; elsewhere a numpy masked array (np.ma, the masked rows missing) for integers and +booleans, None in a text column. """ +from collections.abc import MutableMapping, MutableSequence from dataclasses import dataclass, field from typing import Optional @@ -22,22 +27,99 @@ # Columns whose meaning the specification fixes, per kind of table RESERVED = { - "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index"), + "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order"), "relation": ("id", "source", "target", "relation"), "mapping": ("source", "target", "weight"), "scalars": (), } -ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index") # reserved columns of ids, int64 -EDGE_TYPES = ("none", "successor", "branch") # MTG: root of its scale, '<', '+' +ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", + "sibling_order") # reserved, int64 +EDGE_TYPES = ("none", "successor", "branch", "unknown") # MTG: root of its scale, '<', '+'; a link not known +BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", + "Connection") # an entity set's scale: one of these, or "mixed" +FORMATS = ("date", "date-time", "time") # a text variable holding ISO 8601 dates or times ENUM_COLUMNS = ("label", "edge_type", "relation") DATA_STRUCTURE_KINDS = ("graph", "regular_grid", "adaptive_grid", "table") ROLES = ("state", "input", "parameter", "derived", "setting") +class LazyColumns(MutableMapping): + """A table's columns read lazily (codec.read(path, lazy=True)): their names known, each column decoded from the + file on first access, then kept. Setting or deleting a column works as on a dict.""" + + def __init__(self, loaders: dict, count: int): + self._loaders = dict(loaders) # name: a function decoding the column + self._values = {} + self._order = list(loaders) + self.count = count + + def __getitem__(self, name): + if name not in self._values: + if name not in self._loaders: + raise KeyError(name) + self._values[name] = self._loaders.pop(name)() + return self._values[name] + + def __setitem__(self, name, values): + if name not in self._order: + self._order.append(name) + self._loaders.pop(name, None) + self._values[name] = _as_column(name, values) + + def __delitem__(self, name): + if name not in self._order: + raise KeyError(name) + self._order.remove(name) + self._loaders.pop(name, None) + self._values.pop(name, None) + + def __contains__(self, name): # without decoding it + return name in self._order + + def __iter__(self): + return iter(list(self._order)) + + def __len__(self): + return len(self._order) + + def decoded(self) -> list: + """The names of the columns decoded so far.""" + return [name for name in self._order if name in self._values] + + +class LazyList(MutableSequence): + """A list whose items (a data structure's meshes, read lazily) are decoded on first access.""" + + def __init__(self, loaders=()): + self._items = [(False, loader) for loader in loaders] + + def __getitem__(self, index): + if isinstance(index, slice): + return [self[position] for position in range(*index.indices(len(self)))] + done, item = self._items[index] + if not done: + item = item() + self._items[index] = (True, item) + return item + + def __setitem__(self, index, value): + self._items[index] = (True, value) + + def __delitem__(self, index): + del self._items[index] + + def __len__(self): + return len(self._items) + + def insert(self, index, value): + self._items.insert(index, (True, value)) + + @dataclass class Variable: """The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its - kind (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance.""" + kind (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance; the format + of a text column holding dates or times ("date", "date-time", "time").""" unit: str = "1" standard_name: Optional[str] = None kind: Optional[str] = None @@ -47,6 +129,7 @@ class Variable: maximum: Optional[float] = None references: tuple = () doi: tuple = () + format: Optional[str] = None @dataclass @@ -60,16 +143,22 @@ class Table: table_kind = "scalars" def __post_init__(self): + if isinstance(self.columns, LazyColumns): # read lazily: each column checked as it is decoded + return self.columns = {name: _as_column(name, values) for name, values in self.columns.items()} for name in set(self.columns) & set(ID_COLUMNS) & set(self.reserved): # ids are int64, whatever the platform's if self.columns[name].dtype.kind in "iu": # default integer self.columns[name] = self.columns[name].astype(np.int64) + if np.ma.isMaskedArray(self.columns[name]): + raise ValueError(f"table '{self.name}': the reserved column '{name}' has no missing values") counts = {name: len(values) for name, values in self.columns.items()} if len(set(counts.values())) > 1: raise ValueError(f"table '{self.name}': columns of different lengths {counts}") @property def count(self) -> int: + if isinstance(self.columns, LazyColumns): + return self.columns.count return len(next(iter(self.columns.values()))) if self.columns else 0 @property @@ -88,8 +177,11 @@ class EntitySet(Table): """The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (rank None). Columns: id (int64, stable across frames, never reused), label (str), complex (the id of the containing entity at the coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), edge_type - (str: 'none', 'successor', 'branch'); an adaptive grid's cells: level and index; then the variables.""" + (str: 'none', 'successor', 'branch', 'unknown'); sibling_order (int64, optional: its position among its tree parent's + children); an adaptive grid's cells: level and index; then the variables. *scale*: its base scale + (BASE_SCALES), or "mixed".""" rank: Optional[int] = None + scale: Optional[str] = None table_kind = "entity_set" @@ -154,7 +246,7 @@ def __post_init__(self): class DataStructure: """One plant population's graph, or one environment model's grid. *origin*: its frame in the scene (4×4). *extras*: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); other - readers keep them without interpreting them.""" + readers keep them without interpreting them. *labels*: {the platform's label: the vocabulary's label}.""" id: str kind: str = "graph" entity_sets: list = field(default_factory=list) @@ -166,6 +258,7 @@ class DataStructure: platform: Optional[str] = None model: Optional[str] = None extras: dict = field(default_factory=dict) + labels: dict = field(default_factory=dict) def __post_init__(self): self.origin = np.asarray(self.origin, dtype=np.float64).reshape(4, 4) @@ -188,10 +281,24 @@ class Mapping(Table): table_kind = "mapping" +@dataclass +class Update: + """What makes a frame partial: a coupling session's message (specification §11). *participant*: its sender + ("coordinator" for a turn's inputs). Its tables hold some rows and columns only: the sender's outputs and its + topology changes, entities referring to others the frame does not hold. *removed*: {(data structure id, entity + set name): int64 ids} of the entities removed.""" + participant: str + removed: dict = field(default_factory=dict) + + def __post_init__(self): + self.removed = {key: np.asarray(ids, dtype=np.int64).reshape(-1) for key, ids in self.removed.items()} + + @dataclass class Frame: """A scene at one time step (*time* in *time_unit*, *step*), or a parameter set (data structures holding only - parameters). *unknown*: content of a read file this package does not interpret, written back as it was.""" + parameters). *update*: set for a partial frame (Update), a coupling session's message. *unknown*: content of a + read file this package does not interpret, written back as it was.""" data_structures: list = field(default_factory=list) mappings: list = field(default_factory=list) time: float = 0. @@ -199,6 +306,7 @@ class Frame: step: int = 0 writer: dict = field(default_factory=dict) vocabulary_version: Optional[str] = None + update: Optional[Update] = None unknown: dict = field(default_factory=dict) def data_structure(self, id: str) -> DataStructure: @@ -207,15 +315,40 @@ def data_structure(self, id: str) -> DataStructure: return data_structure raise KeyError(f"the frame has no data structure '{id}'") + def close(self) -> None: + """Release the file of a frame read lazily (codec.read(path, lazy=True)); its columns not decoded yet can no + longer be read. Nothing to do for a frame read whole.""" + source = getattr(self, "_source", None) + if source is None: + return + mapped, binary = source + self._source = None + try: + binary.release() + mapped.close() + except BufferError: # a view of the file still alive: closed when it goes + pass + + def __enter__(self): + return self + + def __exit__(self, *exception): + self.close() + def _as_column(name: str, values) -> np.ndarray: - array = np.asarray(values) + if np.ma.isMaskedArray(values): # integers or booleans with missing values + array = values if np.ma.getmaskarray(values).any() else np.asarray(values.data) + else: + array = np.asarray(values) if array.ndim == 0: array = array.reshape(1) if array.dtype.kind in "OU": - if not all(isinstance(value, str) for value in array.reshape(-1)): - raise TypeError(f"column '{name}': object columns must hold strings") + if not set(map(type, array.reshape(-1).tolist())) <= {str, type(None)}: + raise TypeError(f"column '{name}': object columns must hold strings (None: missing)") array = array.astype(object) + elif np.ma.isMaskedArray(array) and array.dtype.kind == "f": + array = array.filled(np.nan) # missing floats are NaN if array.ndim > 2: raise ValueError(f"column '{name}': one value or one vector per row (got shape {array.shape})") return array diff --git a/bindings/python/src/openalea/encoding3d/schema.py b/bindings/python/src/openalea/encoding3d/schema.py new file mode 100644 index 0000000..1c53fc2 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/schema.py @@ -0,0 +1,13 @@ +"""The JSON Schemas of encoding3d (spec/schema/): fspm_scene.schema.json (the FSPM_scene extension), +index.schema.json (a sequence's scene.fspm.json), views.schema.json (a sequence's views.json) and coupling.schema.json +(a coupling session's messages).""" +import json +from functools import lru_cache + +from openalea.encoding3d import _spec + + +@lru_cache(maxsize=None) +def load(name: str) -> dict: + with open(_spec.path("schema", f"{name}.schema.json"), encoding="utf-8") as file: + return json.load(file) diff --git a/bindings/python/src/openalea/fspm_tf/ucum.py b/bindings/python/src/openalea/encoding3d/ucum.py similarity index 62% rename from bindings/python/src/openalea/fspm_tf/ucum.py rename to bindings/python/src/openalea/encoding3d/ucum.py index 0ad27b0..e4837b1 100644 --- a/bindings/python/src/openalea/fspm_tf/ucum.py +++ b/bindings/python/src/openalea/encoding3d/ucum.py @@ -1,10 +1,13 @@ """ -Units as UCUM codes (https://ucum.org), the subset fspm-TF accepts: SI and common atoms, metric prefixes on the +Units as UCUM codes (https://ucum.org), the subset encoding3d accepts: SI and common atoms, metric prefixes on the metric ones, products ('.'), quotients ('/'), integer exponents, parentheses, annotations ({...}, dimensionless) and -'1'. dimension(code) gives a unit's dimension, to check that two units are convertible:: +'1'. dimension(code) gives a unit's dimension, to check that two units are convertible, and conversion_factor the +factor from one to the other:: dimension("mm3/(s.MPa)") == dimension("m4.s.kg-1") True: the same physical dimension + conversion_factor("g", "kg") 0.001: a value in g times it is in kg """ +import math import re # Dimensions: length, mass, time, temperature, amount, luminous intensity, current @@ -31,6 +34,16 @@ def _dims(**exponents) -> tuple: "min": _dims(T=1), "h": _dims(T=1), "d": _dims(T=1), "wk": _dims(T=1), "mo": _dims(T=1), "a": _dims(T=1), "deg": NONE, "%": NONE, "[ppm]": NONE, "[ppb]": NONE, }) +# Each atom's magnitude in the coherent units m, g, s, K, mol, cd, A (g, not kg: prefixes apply to g) +MAGNITUDES = { + "N": 1e3, "Pa": 1e3, "J": 1e3, "W": 1e3, "V": 1e3, "Ohm": 1e3, "S": 1e-3, "F": 1e-3, "T": 1e3, "Wb": 1e3, + "L": 1e-3, "l": 1e-3, "bar": 1e8, "ar": 1e2, "t": 1e6, "eq": 1., "m[H2O]": 9.80665e6, "m[Hg]": 1.33322387415e8, + "min": 60., "h": 3600., "d": 86400., "wk": 604800., "mo": 2629800., "a": 31557600., "deg": math.pi / 180, + "%": 1e-2, "[ppm]": 1e-6, "[ppb]": 1e-9, +} +PREFIX_FACTORS = {"da": 1e1, "Y": 1e24, "Z": 1e21, "E": 1e18, "P": 1e15, "T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3, + "h": 1e2, "d": 1e-1, "c": 1e-2, "m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12, "f": 1e-15, + "a": 1e-18, "z": 1e-21, "y": 1e-24} SPECIAL = {"Cel"} # an offset scale: not allowed in products, quotients, powers PREFIXES = ("da", "Y", "Z", "E", "P", "T", "G", "M", "k", "h", "d", "c", "m", "u", "n", "p", "f", "a", "z", "y") @@ -42,18 +55,42 @@ class UnitError(ValueError): def _atom(text: str) -> tuple: - """(dimension, special) of a unit symbol, with its prefix.""" + """(dimension, special, magnitude) of a unit symbol, with its prefix.""" if text in ATOMS: - return ATOMS[text], text in SPECIAL + return ATOMS[text], text in SPECIAL, MAGNITUDES.get(text, 1.) for prefix in PREFIXES: if text.startswith(prefix) and text[len(prefix):] in METRIC_ATOMS: - return METRIC_ATOMS[text[len(prefix):]], text[len(prefix):] in SPECIAL + atom = text[len(prefix):] + return METRIC_ATOMS[atom], atom in SPECIAL, PREFIX_FACTORS[prefix] * MAGNITUDES.get(atom, 1.) raise UnitError(f"unknown unit symbol '{text}'") def dimension(code: str) -> tuple: - """The dimension of the UCUM *code* (exponents of L, M, T, Θ, N, J, I); raises UnitError if not a unit fspm-TF + """The dimension of the UCUM *code* (exponents of L, M, T, Θ, N, J, I); raises UnitError if not a unit encoding3d accepts.""" + return _parse(code)[0] + + +def magnitude(code: str) -> float: + """The size of the unit *code* in the coherent units m, g, s, K, mol, cd, A (e.g. 'kPa': 1e6 g.m-1.s-2).""" + return _parse(code)[2] + + +def conversion_factor(code: str, other: str) -> float: + """The factor converting a value in *code* to *other* (value_in_other = value_in_code * factor); UnitError if + they are not convertible, or if one is Cel and the other not (an offset scale: no factor).""" + dims, special, size = _parse(code) + other_dims, other_special, other_size = _parse(other) + if dims != other_dims: + raise UnitError(f"'{code}' and '{other}' are not convertible (different dimensions)") + if special != other_special: + raise UnitError(f"'{code}' and '{other}': Cel is an offset scale, converted to K by adding 273.15, not by a " + "factor") + return size / other_size + + +def _parse(code: str) -> tuple: + """(dimension, special, magnitude) of the UCUM *code*.""" if not isinstance(code, str) or not code or code != code.strip() or " " in code: raise UnitError(f"'{code}' is not a UCUM code (no spaces: write 'mol/(m2.s)' or 'mol.m-2.s-1')") tokens = _TOKEN.findall(code) @@ -63,15 +100,16 @@ def dimension(code: str) -> tuple: def term(): nonlocal position - total, special_seen, count = [0] * 7, False, 0 + total, special_seen, count, size = [0] * 7, False, 0, 1. sign = 1 if position < len(tokens) and tokens[position] == "/": sign, position = -1, position + 1 while True: - dims, special = component() + dims, special, factor = component() count += 1 special_seen = special_seen or special total = [a + sign * b for a, b in zip(total, dims)] + size = size * factor if sign > 0 else size / factor if position < len(tokens) and tokens[position] in (".", "/"): sign = 1 if tokens[position] == "." else -1 position += 1 @@ -79,7 +117,7 @@ def term(): break if special_seen and count > 1: raise UnitError(f"'{code}': Cel (a temperature scale) cannot be combined with other units; use K") - return tuple(total), special_seen + return tuple(total), special_seen, size def component(): nonlocal position @@ -88,34 +126,35 @@ def component(): token = tokens[position] position += 1 if token == "(": - dims, special = term() + dims, special, factor = term() if position >= len(tokens) or tokens[position] != ")": raise UnitError(f"'{code}': unbalanced parenthesis") position += 1 elif token.startswith("{"): - return NONE, False + return NONE, False, 1. elif re.fullmatch(r"[+-]?\d+", token): if int(token) != 1 and not token.lstrip("+-").startswith("10"): raise UnitError(f"'{code}': numbers other than 1 are not units") - return NONE, False + return NONE, False, float(int(token)) elif token in (".", "/", ")"): raise UnitError(f"'{code}': unexpected '{token}'") else: - dims, special = _atom(token) + dims, special, factor = _atom(token) if position < len(tokens) and re.fullmatch(r"[+-]?\d+", tokens[position]): exponent = int(tokens[position]) position += 1 if special: raise UnitError(f"'{code}': Cel cannot have an exponent") dims = tuple(exponent * value for value in dims) + factor = factor ** exponent if position < len(tokens) and tokens[position].startswith("{"): position += 1 - return dims, special + return dims, special, factor - dims, _ = term() + parsed = term() if position != len(tokens): raise UnitError(f"'{code}': unexpected '{tokens[position]}'") - return dims + return parsed def is_valid(code: str) -> bool: diff --git a/bindings/python/src/openalea/fspm_tf/validate.py b/bindings/python/src/openalea/encoding3d/validate.py similarity index 81% rename from bindings/python/src/openalea/fspm_tf/validate.py rename to bindings/python/src/openalea/encoding3d/validate.py index 921b6fb..a8348c7 100644 --- a/bindings/python/src/openalea/fspm_tf/validate.py +++ b/bindings/python/src/openalea/encoding3d/validate.py @@ -1,5 +1,5 @@ """ -The fspm-TF validator: the checks a file must pass to follow the specification, beyond being valid glTF (which +The encoding3d validator: the checks a file must pass to follow the specification, beyond being valid glTF (which Khronos' glTF-Validator checks):: validate_file(path) -> Report a .glb frame, or a sequence's scene.fspm.json (each frame, and across them) @@ -9,17 +9,17 @@ is likely wrong or incomplete). """ from dataclasses import dataclass, field +import datetime import json import os -import jsonschema import numpy as np -from openalea.fspm_tf import schema as schemas, ucum, vocabulary -from openalea.fspm_tf.codec import (DTYPES, EDGE_TYPE_ENUM, EXTENSION, FEATURES, METADATA, Z_UP_TO_Y_UP, +from openalea.encoding3d import schema as schemas, ucum, vocabulary +from openalea.encoding3d.codec import (DTYPES, EDGE_TYPE_ENUM, EXTENSION, FEATURES, METADATA, Z_UP_TO_Y_UP, from_bytes) -from openalea.fspm_tf.glb import unpack -from openalea.fspm_tf.model import EDGE_TYPES, NO_ID +from openalea.encoding3d.glb import unpack +from openalea.encoding3d.model import BASE_SCALES, EDGE_TYPES, NO_ID @dataclass @@ -67,6 +67,12 @@ def __str__(self): # ── One frame ──────────────────────────────────────────────────────────── +def _jsonschema(): + """jsonschema, imported when a file is validated (the reading code, e.g. ParaView's reader, works without it).""" + import jsonschema + return jsonschema + + def validate_bytes(data: bytes, name: str = "") -> Report: report = Report(name) try: @@ -103,10 +109,10 @@ def _check_document(document: dict, binary: bytes, report: Report) -> bool: "lying down") extensions = document.get("extensions", {}) if EXTENSION not in extensions: - report.error("extensions", f"no {EXTENSION} extension: not an fspm-TF file") + report.error("extensions", f"no {EXTENSION} extension: not an encoding3d file") return False scene = extensions[EXTENSION] - for error in jsonschema.Draft202012Validator(schemas.load("fspm_scene")).iter_errors(scene): + for error in _jsonschema().Draft202012Validator(schemas.load("fspm_scene")).iter_errors(scene): report.error(f"{EXTENSION}/{'/'.join(str(part) for part in error.absolute_path)}", error.message) metadata = extensions.get(METADATA) if metadata is None: @@ -157,6 +163,9 @@ def length(key): return views[index]["byteLength"] kind = prop.get("type") + if prop.get("extras", {}).get("FSPM_boolean") and (kind != "SCALAR" or prop.get("componentType") != "UINT8" + or prop.get("noData") != 255 or prop.get("array")): + report.error(where, "a boolean column with missing values is UINT8 (0, 1) with noData 255") values_length = length("values") if values_length is None: return @@ -165,7 +174,7 @@ def length(key): report.error(where, f"component type {prop.get('componentType')} is not supported") return if prop.get("array") and "count" not in prop: - report.error(where, "variable-length arrays are not part of fspm-TF 0.1") + report.error(where, "variable-length arrays are not part of encoding3d 0.1") return expected = count * prop.get("count", 1) * DTYPES[prop["componentType"]].itemsize elif kind == "BOOLEAN": @@ -183,7 +192,7 @@ def length(key): report.error(where, f"{offsets_length} bytes of string offsets, {(count + 1) * offset_size} expected") return else: - report.error(where, f"property type {kind} is not part of fspm-TF 0.1") + report.error(where, f"property type {kind} is not part of encoding3d 0.1") return if values_length != expected: report.error(where, f"{values_length} bytes of values, {expected} expected for {count} rows") @@ -206,16 +215,31 @@ def _check_frame(frame, document: dict, report: Report) -> None: if not (data_structure.grid is not None and name == data_structure.grid.cells)] if explicit and sum(len(values) for values in explicit) != len(set().union(*explicit)): report.error(where, "ids are unique across the data structure's entity sets and relations") - _check_mtg(data_structure, sets, ids, report) + partial = frame.update is not None # a partial frame refers to entities it does not hold + _check_mtg(data_structure, sets, ids, report, partial=partial) for relation in data_structure.relations: for end, set_name in (("source", relation.source_set), ("target", relation.target_set)): if set_name not in ids: - report.error(f"{where}/{relation.name}", f"{end} set '{set_name}' does not exist") + if not partial: + report.error(f"{where}/{relation.name}", f"{end} set '{set_name}' does not exist") elif end not in relation.columns: report.error(f"{where}/{relation.name}", f"no '{end}' column") - else: + elif not partial: _check_references(f"{where}/{relation.name}/{end}", relation.columns[end], ids[set_name], report, allow_none=False) + names = [relation.name for relation in data_structure.relations] + if len(set(names)) != len(names): + report.error(where, "two relations have the same name") + for relation in data_structure.relations: + if relation.name.startswith("bearer") and "relation" in relation.columns: + unknown = set(relation.columns["relation"].tolist()) - set(enums["bearer_relation"]) + if unknown: + report.warning(f"{where}/{relation.name}/relation", + f"bearer labels {sorted(unknown)}: expected {sorted(enums['bearer_relation'])}") + _check_scales_and_labels(data_structure, report) + for key in data_structure.extras: + if key != key.lower(): + report.warning(f"{where}/extras", f"key '{key}': the platform's name, in lower case") if data_structure.grid is not None: _check_grid(data_structure, sets, report) elif data_structure.kind in ("regular_grid", "adaptive_grid"): @@ -231,7 +255,13 @@ def _check_frame(frame, document: dict, report: Report) -> None: data_structures = {data_structure.id: data_structure for data_structure in frame.data_structures} if len(data_structures) != len(frame.data_structures): report.error("dataStructures", "two data structures have the same id") + if frame.update is not None: + for (data_structure_id, set_name), removed in frame.update.removed.items(): + if (removed < 0).any(): + report.error(f"update/removed/{data_structure_id}/{set_name}", "ids are non-negative") for mapping in frame.mappings: + if frame.update is not None: + continue where = f"mapping {mapping.name}" for end, (data_structure_id, set_name) in (("source", mapping.source), ("target", mapping.target)): try: @@ -280,14 +310,17 @@ def _check_references(where, values, ids: set, report, allow_none: bool = True) report.error(where, f"{len(missing)} id(s) not in the referenced entity set, e.g. {missing[:5]}") -def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: - """complex points to an entity of a coarser entity set; parent and edge_type agree; parents form no cycle.""" +def _check_mtg(data_structure, sets: dict, ids: dict, report, partial: bool = False) -> None: + """complex points to an entity of a coarser entity set; parent and edge_type agree; parents form no cycle. In a + partial frame (*partial*), complex and parent may refer to entities it does not hold, and may be absent.""" ranked = sorted((entity_set for entity_set in data_structure.entity_sets if entity_set.rank is not None), key=lambda entity_set: entity_set.rank) for position, entity_set in enumerate(ranked): where = f"{data_structure.id}/{entity_set.name}" columns = entity_set.columns - if position > 0: + if partial: + pass + elif position > 0: coarser = ranked[position - 1] if "complex" not in columns: report.error(where, f"no 'complex' column (its entities belong to {coarser.name})") @@ -305,7 +338,8 @@ def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: if "parent" not in columns: continue parent, edge_type = columns["parent"], columns["edge_type"] - _check_references(f"{where}/parent", parent, ids[entity_set.name], report) + if not partial: + _check_references(f"{where}/parent", parent, ids[entity_set.name], report) unknown = set(edge_type.tolist()) - set(EDGE_TYPES) if unknown: report.error(f"{where}/edge_type", f"unknown values {sorted(unknown)}") @@ -323,6 +357,47 @@ def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: report.error(f"{where}/parent", "the parents form a cycle") +def _check_scales_and_labels(data_structure, report) -> None: + """Each entity set's scale and rank; its order column; the data structure's label map.""" + used = set() + for entity_set in data_structure.entity_sets: + where = f"{data_structure.id}/{entity_set.name}" + if entity_set.scale in BASE_SCALES and entity_set.rank is not None \ + and entity_set.rank != BASE_SCALES.index(entity_set.scale) + 1: + report.warning(where, f"rank {entity_set.rank}: the base scale {entity_set.scale}'s position is " + f"{BASE_SCALES.index(entity_set.scale) + 1}") + if "sibling_order" in entity_set.columns and (np.asarray(entity_set.columns["sibling_order"]) < 0).any(): + report.error(f"{where}/sibling_order", "positions among siblings are non-negative") + if "label" in entity_set.columns: + used |= {label for label in entity_set.columns["label"].tolist() if label is not None} + vocabulary_labels = vocabulary.labels() + for label, standard in data_structure.labels.items(): + if label not in used: + report.warning(f"{data_structure.id}/labels", f"'{label}' is no entity's label") + if standard not in vocabulary_labels: + report.warning(f"{data_structure.id}/labels", f"'{standard}' is not a label of the vocabulary " + f"{vocabulary.version()}") + + +def _check_format(where, values, variable_format, report) -> None: + """A text column's dates or times, as ISO 8601.""" + if np.asarray(values).dtype != object: + report.error(where, f"format '{variable_format}' is for text columns") + return + parse = {"date": datetime.date.fromisoformat, "date-time": datetime.datetime.fromisoformat, + "time": datetime.time.fromisoformat}[variable_format] + bad = [] + for value in values: + if value is None: + continue + try: + parse(value) + except ValueError: + bad.append(value) + if bad: + report.error(where, f"{len(bad)} value(s) not ISO 8601 {variable_format}, e.g. {bad[:3]}") + + def _check_grid(data_structure, sets: dict, report) -> None: grid, where = data_structure.grid, f"{data_structure.id}/grid" dimensions = len(grid.shape) @@ -369,6 +444,8 @@ def _check_variables(where, table, standard_names: dict, enums: dict, report) -> report.error(f"{where}/{name}", f"kind '{variable.kind}' is not one of {sorted(enums['kind'])}") if variable.role not in enums["role"]: report.error(f"{where}/{name}", f"role '{variable.role}' is not one of {sorted(enums['role'])}") + if variable.format is not None: + _check_format(f"{where}/{name}", table.columns[name], variable.format, report) if variable.standard_name is None: continue entry = standard_names.get(variable.standard_name) @@ -434,7 +511,7 @@ def validate_sequence(index_path: str) -> Report: except (OSError, ValueError) as error: report.error("index", f"cannot be read: {error}") return report - for error in jsonschema.Draft202012Validator(schemas.load("index")).iter_errors(index): + for error in _jsonschema().Draft202012Validator(schemas.load("index")).iter_errors(index): report.error(f"index/{'/'.join(str(part) for part in error.absolute_path)}", error.message) if not report.ok: return report diff --git a/bindings/python/src/openalea/encoding3d/views.py b/bindings/python/src/openalea/encoding3d/views.py new file mode 100644 index 0000000..2b74d58 --- /dev/null +++ b/bindings/python/src/openalea/encoding3d/views.py @@ -0,0 +1,551 @@ +""" +Frame views (specification §8.1): views.json, an optional companion of a sequence or a frame, names configurations of +layers; each layer is a part of a data structure (a mesh, segments or a grid), the entities it selects and how it is +drawn:: + + {"encoding3d_views": 1, "active": "default", + "configurations": {"default": {"layers": [ + {"name": "roots", "source": {"data_structure": "plants", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "colormap": "Viridis (matplotlib)", + "range": "auto", "log": false, "opacity": 1, "visible": true, "representation": "Surface", + "tube": {"radius": "radius", "scale": 1}}]}}} + + read_views(path), write_views(path, views) + views_path(path, views=None) the views file of a frame or sequence: *views*, else the sibling views.json + check_views(views, frame=None) a Report: the schema, and every name against *frame* + configuration(views, name=None) (its name, its layers, each with its defaults filled) + selected(table, where) the rows of *table* where every condition holds + +ParaView's plugin (paraview/plugin_encoding3d.py) draws each layer as a block of the reader's representation, with its +own colouring and opacity: block_displays(layers, arrays) (the layers to the representation's block properties) and +displays_from_blocks(layers, representation) then layer_from_display(layer, display) (back, to save them), plain +dicts and lists both ways; display_settings(layer) gives one layer's options. + +Selection by scale (the plugin's filter "encoding3d Select at scale", its Isolate, Show In Panel and Put Back +buttons): the reader gives every cell the ids of its entity and of the entities containing it (".id", +ancestor_ids), and each block its data structure and its graph's scales as field data (scale_names: the filter's +drop-down); +picked_entities(picked, scale) grows picked cells to their entities at *scale*, selection_mask(...) gives the cells of +any block those entities contain, at any finer scale. +""" +import copy +import json +import os +import re + +import numpy as np + +from openalea.encoding3d import schema as schemas + +VIEWS_FILE = "views.json" +FORMAT = 1 +DEFAULTS = {"color_by": None, "solid_color": [1., 1., 1.], "colormap": "Viridis (matplotlib)", "range": "auto", + "log": False, "opacity": 1., "visible": True, "representation": "Surface"} +TUBE_RADIUS = "tube_radius" # the point data a segments layer's Tube varies its radius by +SEGMENT_ENDS = (("proximal_end_x", "x1"), ("proximal_end_y", "y1"), ("proximal_end_z", "z1"), + ("distal_end_x", "x2"), ("distal_end_y", "y2"), ("distal_end_z", "z2")) + +_CONDITION = re.compile(r"^\s*(<=|>=|==|!=|<|>)\s*([-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\s*$") +_COMPARISONS = {"<": np.less, "<=": np.less_equal, ">": np.greater, ">=": np.greater_equal, "==": np.equal, + "!=": np.not_equal} + + +# ── Files ──────────────────────────────────────────────────────────────── + +def views_path(path: str, views: str = None) -> str: + """The views file of the frame or sequence *path*: *views* when given (relative to *path*'s folder), else the + views.json next to it; None when that file does not exist.""" + folder = os.path.dirname(os.path.abspath(path)) if path else os.getcwd() + found = os.path.join(folder, views) if views else os.path.join(folder, VIEWS_FILE) + return found if os.path.exists(found) else None + + +def read_views(path: str) -> dict: + with open(path, encoding="utf-8") as file: + return json.load(file) + + +def write_views(path: str, views: dict) -> None: + """Write *views* to *path* (checked against the schema first).""" + report = check_views(views) + if not report.ok: + raise ValueError(str(report)) + temporary = path + ".part" + with open(temporary, "w", encoding="utf-8", newline="\n") as file: + json.dump(views, file, indent=1) + file.write("\n") + os.replace(temporary, path) # a reader never sees half a file + + +def configuration_names(views: dict) -> list: + """The configurations' names, the active one first.""" + names = list(views.get("configurations", {})) + active = views.get("active") + return ([active] if active in names else []) + [name for name in names if name != active] + + +def configuration(views: dict, name: str = None) -> tuple: + """(name, layers) of the configuration *name* (None: the active one, else the first), each layer with its + defaults filled.""" + names = configuration_names(views) + if not names: + raise ValueError("the views have no configuration") + name = name or names[0] + if name not in views["configurations"]: + raise KeyError(f"no configuration '{name}' in the views (configurations: {names})") + return name, [filled(layer) for layer in views["configurations"][name]["layers"]] + + +def other_layer_names(views: dict, name: str) -> list: + """The names of the layers of the other configurations than *name* that it has not: the items "Apply encoding3d + views" made for them, replaced when *name* is applied.""" + own = {layer["name"] for layer in views["configurations"][name]["layers"]} + others = [layer["name"] for other, content in views["configurations"].items() if other != name + for layer in content["layers"] if layer["name"] not in own] + return list(dict.fromkeys(others)) + + +def filled(layer: dict) -> dict: + """*layer* with the defaults of what it leaves out (a tube's scale: 1).""" + complete = dict(copy.deepcopy(DEFAULTS), **copy.deepcopy(layer)) + if "tube" in complete: + complete["tube"].setdefault("scale", 1.) + return complete + + +# ── Checks ─────────────────────────────────────────────────────────────── + +def check_views(views: dict, frame=None): + """A validate.Report of *views*: the schema, an active configuration that exists, unique layer names, and with + *frame* every name a layer gives (data structure, mesh, entity sets, columns) found in it.""" + from openalea.encoding3d.validate import Report + report = Report("views") + try: + import jsonschema + except ImportError: # e.g. in ParaView's Python: the structure the code reads + for where, message in _structure_errors(views): + report.error(where, message) + else: + for error in jsonschema.Draft202012Validator(schemas.load("views")).iter_errors(views): + report.error(f"views/{'/'.join(str(part) for part in error.absolute_path)}", error.message) + if not report.ok: + return report + if "active" in views and views["active"] not in views["configurations"]: + report.error("views/active", f"no configuration '{views['active']}'") + for name, content in views["configurations"].items(): + seen = set() + for number, layer in enumerate(content["layers"]): + where = f"{name}/layers/{number} ({layer['name']})" + if layer["name"] in seen: + report.error(where, "another layer of the configuration has this name") + seen.add(layer["name"]) + if frame is not None: + _check_layer(layer, frame, where, report) + return report + + +def _structure_errors(views) -> list: + """[(where, message)]: what the schema requires that the reader and the plugin rely on, without jsonschema.""" + if not isinstance(views, dict): + return [("views", "not an object")] + errors = [] + if views.get("encoding3d_views") != FORMAT: + errors.append(("views/encoding3d_views", f"must be {FORMAT}")) + configurations = views.get("configurations") + if not isinstance(configurations, dict) or not configurations: + return errors + [("views/configurations", "an object of at least one configuration")] + for name, content in configurations.items(): + layers = content.get("layers") if isinstance(content, dict) else None + if not isinstance(layers, list): + errors.append((f"views/configurations/{name}", "'layers' is a required list")) + continue + for number, layer in enumerate(layers): + where = f"views/configurations/{name}/layers/{number}" + source = layer.get("source") if isinstance(layer, dict) else None + if not isinstance(layer, dict) or not isinstance(layer.get("name"), str) or not layer["name"]: + errors.append((where, "'name' is a required text")) + if not isinstance(source, dict) or not isinstance(source.get("data_structure"), str): + errors.append((where, "'source' with its 'data_structure' is required")) + elif source.get("kind") not in ("mesh", "segments", "grid"): + errors.append((f"{where}/source", "'kind' is one of mesh, segments, grid")) + elif source["kind"] in ("mesh", "segments") and not isinstance( + source.get("mesh" if source["kind"] == "mesh" else "entity_set"), str): + errors.append((f"{where}/source", f"a {source['kind']} layer names its " + f"{'mesh' if source['kind'] == 'mesh' else 'entity_set'}")) + return errors + + +def segment_columns(entity_set) -> list: + """The six segment-end columns of *entity_set* (by standard name, else x1 ... z2), or [] when it has none.""" + by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() + if variable.standard_name} + found = [] + for standard, plain in SEGMENT_ENDS: + name = by_standard.get(standard, plain) + column = entity_set.columns.get(name) + if column is None or column.dtype == object or column.dtype.kind not in "fiu": + return [] + found.append(name) + return found + + +def _sets_of(data_structure, layer: dict, where: str, report) -> tuple: + """(the entity sets the layer may select and colour by, the name of the set its entities are in), or None.""" + source = layer["source"] + sets = {entity_set.name: entity_set for entity_set in data_structure.entity_sets} + if source["kind"] == "mesh": + meshes = {mesh.name: mesh for mesh in data_structure.meshes} + if source["mesh"] not in meshes: + report.error(where, f"no mesh '{source['mesh']}' in '{data_structure.id}' (meshes: {sorted(meshes)})") + return None + return {name: sets[name] for name in meshes[source["mesh"]].feature_rows if name in sets}, None + if source["kind"] == "segments": + entity_set = sets.get(source["entity_set"]) + if entity_set is None: + report.error(where, f"no entity set '{source['entity_set']}' in '{data_structure.id}'") + return None + if not segment_columns(entity_set): + report.error(where, f"'{entity_set.name}' has no segment ends (proximal_end_x ... distal_end_z, or x1 ... " + "z2)") + return None + return {entity_set.name: entity_set}, entity_set.name + if data_structure.grid is None: + report.error(where, f"'{data_structure.id}' has no grid") + return None + return {data_structure.grid.cells: sets[data_structure.grid.cells]}, data_structure.grid.cells + + +def _check_layer(layer: dict, frame, where: str, report) -> None: + source = layer["source"] + found = {data_structure.id: data_structure for data_structure in frame.data_structures} + data_structure = found.get(source["data_structure"]) + if data_structure is None: + report.error(where, f"no data structure '{source['data_structure']}' in the frame (data structures: " + f"{sorted(found)})") + return + usable = _sets_of(data_structure, layer, where, report) + if usable is None: + return + sets, _ = usable + for part, columns in (("select", list(layer.get("select", {}).get("where", {}))), + ("color_by", [(layer.get("color_by") or {}).get("variable")]), + ("tube", [layer.get("tube", {}).get("radius")])): + names = [column for column in columns if column] + if not names: + continue + set_name = (layer[part].get("entity_set") if part != "tube" else source.get("entity_set")) + if set_name not in sets: + report.error(f"{where}/{part}", f"'{set_name}' is not an entity set this layer's entities are in " + f"({sorted(sets)})") + continue + for column in names: + if column not in sets[set_name].columns: + report.error(f"{where}/{part}", f"no column '{column}' in '{data_structure.id}/{set_name}'") + if layer.get("tube") is not None and source["kind"] != "segments": + report.error(f"{where}/tube", "a tube is for segments") + + +# ── Selection ──────────────────────────────────────────────────────────── + +def _holds(column: np.ndarray, condition) -> np.ndarray: + missing = np.ma.getmaskarray(column) if np.ma.isMaskedArray(column) else np.zeros(len(column), dtype=bool) + values = np.ma.getdata(column) + comparison = _CONDITION.match(condition) if isinstance(condition, str) else None + if comparison is not None and values.dtype != object: + operator, threshold = comparison.group(1), float(comparison.group(2)) + with np.errstate(invalid="ignore"): + result = _COMPARISONS[operator](values.astype(np.float64), threshold) + elif isinstance(condition, (list, tuple)): + result = np.isin(values, np.asarray(condition, dtype=object if values.dtype == object else None)) + else: + result = values == condition + return np.asarray(result, dtype=bool) & ~missing + + +def selected(table, where: dict) -> np.ndarray: + """The rows of *table* (an entity set) where every {column: condition} of *where* holds: a condition is a value, + a list of values (labels and other text by name), or a comparison (">=0", "!=2"); a missing value never holds.""" + result = np.ones(table.count, dtype=bool) + for column, condition in where.items(): + result &= _holds(table.columns[column], condition) + return result + + +# ── Display options (ParaView's plugin) ────────────────────────────────── + +def property_text(value) -> str: + """A ParaView string property's value as text: ParaView gives a str, a property (its GetData(): a str, or a list + of one value), or a list-like of one value (empty: "").""" + if hasattr(value, "GetData"): + value = value.GetData() + if value is None or isinstance(value, str): + return value or "" + values = list(value) + return str(values[0]) if values else "" + + +def block_selector(name: str) -> str: + """The selector of the reader's block *name* in ParaView (a representation's BlockSelectors, an ExtractBlock): + /Root/ and the name as ParaView makes it a valid node name: VTK's vtkDataAssembly.MakeValidNodeName when VTK is + there (its rule changed between versions), else ParaView 6's (the characters other than letters, digits, '_', '-' + and '.' removed, and '_' before a first character that is not a letter or '_').""" + try: + from vtkmodules.vtkCommonDataModel import vtkDataAssembly + except ImportError: + valid = re.sub(r"[^A-Za-z0-9_.\-]", "", name) + if not re.match(r"[A-Za-z_]", valid): + valid = "_" + valid + else: + valid = vtkDataAssembly.MakeValidNodeName(name) + return f"/Root/{valid}" + + +def preset_names(colormap: str) -> list: + """The ParaView preset names to try for *colormap*: itself, then its name in other ParaView versions (ParaView 6 + dropped the " (matplotlib)" of 5's "Viridis (matplotlib)", "Inferno (matplotlib)" ...).""" + suffix = " (matplotlib)" + other = colormap[:-len(suffix)] if colormap.endswith(suffix) else colormap + suffix + return [colormap, other] + + +def colour_array(layer: dict, arrays=()) -> str: + """The cell data array a layer is coloured by: ".", or its codes ". + _code" when *arrays* (the block's arrays) hold those instead (a text variable); None for a solid colour.""" + colour = layer.get("color_by") + if not colour: + return None + name = f"{colour['entity_set']}.{colour['variable']}" + if arrays and name not in arrays and f"{name}_code" in arrays: + return f"{name}_code" + return name + + +def display_settings(layer: dict, arrays=()) -> dict: + """How ParaView's plugin draws *layer*: its block (selector), a tube or not (the reader draws a segments layer + with a "tube" as tubes, their radius also as point data TUBE_RADIUS), the colouring (a cell array, its colour map, + its range: None for the data's), the solid colour, the opacity, the visibility and the representation.""" + layer = filled(layer) + return {"name": layer["name"], "selector": block_selector(layer["name"]), + "tube": {"radius_array": TUBE_RADIUS} if layer.get("tube") else None, + "color_array": colour_array(layer, arrays), "colormap": layer["colormap"], + "range": None if layer["range"] == "auto" else [float(value) for value in layer["range"]], + "log": bool(layer["log"]), "solid_color": [float(value) for value in layer["solid_color"]], + "opacity": float(layer["opacity"]), "visible": bool(layer["visible"]), + "representation": layer["representation"]} + + +def layer_from_display(layer: dict, display: dict) -> dict: + """*layer* updated from its display options in ParaView (displays_from_blocks'), to save them: visible, + opacity, representation, color_array (None: a solid colour), solid_color, range (None: the data's), log and + colormap, each optional (absent: the layer's own). A "._code" array is the variable.""" + updated = copy.deepcopy(layer) + for key in ("visible", "opacity", "representation", "solid_color", "log", "colormap"): + if display.get(key) is not None: + updated[key] = display[key] + if "opacity" in updated: + updated["opacity"] = float(updated["opacity"]) + if "range" in display: + updated["range"] = "auto" if display["range"] is None else \ + [float(f"{float(value):.12g}") for value in display["range"]] # (a colour map's float noise off) + if "color_array" in display: + array = display["color_array"] + if not array: + updated["color_by"] = None + else: + entity_set, _, variable = array.partition(".") + previous = layer.get("color_by") or {} + if variable.endswith("_code") and previous.get("variable") != variable: + variable = variable[:-len("_code")] + updated["color_by"] = {"entity_set": entity_set, "variable": variable} + return updated + + +def block_displays(layers: list, arrays=()) -> dict: + """What ParaView's plugin sets on the reader's representation for the *layers* of a configuration (its blocks, + *arrays* the cell arrays they hold), each layer drawn as its own block: + + selectors the visible layers' blocks (the representation's BlockSelectors) + colourings [(selector, cell array)] of the coloured layers (BlockColorArrayNames) + solid_colours [(selector, RGB)] of the others (BlockColors) + opacities [(selector, opacity)] of the layers not opaque (BlockOpacities) + colour_maps {cell array: {"colormap", "range", "log"}}, the first layer coloured by it giving it + main_array the array the representation itself is coloured by (the first visible coloured layer's, else + the first coloured one's; None: no layer coloured): ParaView colours a block by its own array + only when the representation is coloured by one + scalar_bars the arrays of the visible coloured layers + representation the visible layers' representation when they agree, else "Surface" + """ + settings = [display_settings(layer, arrays) for layer in layers] + colourings = [(setting["selector"], setting["color_array"]) for setting in settings if setting["color_array"]] + colour_maps = {} + for setting in settings: + if setting["color_array"] and setting["color_array"] not in colour_maps: + colour_maps[setting["color_array"]] = {key: setting[key] for key in ("colormap", "range", "log")} + visible = [setting for setting in settings if setting["visible"]] + shown_arrays = [setting["color_array"] for setting in visible if setting["color_array"]] + representations = {setting["representation"] for setting in visible} + return {"selectors": [setting["selector"] for setting in visible], "colourings": colourings, + "solid_colours": [(setting["selector"], setting["solid_color"]) for setting in settings + if not setting["color_array"]], + "opacities": [(setting["selector"], setting["opacity"]) for setting in settings + if setting["opacity"] != 1.], + "colour_maps": colour_maps, + "main_array": shown_arrays[0] if shown_arrays else (colourings[0][1] if colourings else None), + "scalar_bars": list(dict.fromkeys(shown_arrays)), + "representation": representations.pop() if len(representations) == 1 else "Surface"} + + +def _groups(values, size: int) -> list: + values = [str(value) for value in values] + return [values[start:start + size] for start in range(0, len(values) - size + 1, size)] + + +def _shows(selectors: list, selector: str) -> bool: + """Whether the BlockSelectors *selectors* show the block *selector* (itself, or a block holding it: "/Root").""" + return any(shown in ("/", selector) or selector.startswith(shown.rstrip("/") + "/") for shown in selectors) + + +def displays_from_blocks(layers: list, representation: dict) -> list: + """The display options of each of *layers* (as layer_from_display takes them) read back from the reader's + representation: *representation* holds its properties as ParaView gives them, flat lists: BlockSelectors, + BlockColorArrayNames (selector, association, array ...), BlockColors (selector, r, g, b ...), BlockOpacities + (selector, opacity ...), and its own ColorArrayName (the array name), DiffuseColor, Opacity and Representation; + with "colour_maps", {array: (range, or None when it follows the data; log scale)}. A block without a colouring of + its own has the representation's.""" + selectors = [str(value) for value in representation.get("BlockSelectors", [])] + colourings = {selector: array for selector, _, array in _groups(representation.get("BlockColorArrayNames", []), 3)} + solid = {group[0]: [float(value) for value in group[1:]] + for group in _groups(representation.get("BlockColors", []), 4)} + opacities = {selector: float(value) for selector, value in _groups(representation.get("BlockOpacities", []), 2)} + colour_maps = representation.get("colour_maps", {}) + found = [] + for layer in layers: + selector = block_selector(layer["name"]) + display = {"visible": _shows(selectors, selector), + "opacity": opacities.get(selector, float(representation.get("Opacity", 1.))), + "representation": representation.get("Representation") or None} + if colourings.get(selector): + array = colourings[selector] + elif selector in solid: + array = None + display["solid_color"] = solid[selector] + else: # the representation's own colouring + array = representation.get("ColorArrayName") or None + if array is None and representation.get("DiffuseColor") is not None: + display["solid_color"] = [float(value) for value in representation["DiffuseColor"]] + display["color_array"] = array + if array is not None and array in colour_maps: + display["range"], display["log"] = colour_maps[array][0], bool(colour_maps[array][1]) + found.append(display) + return found + + +def save_configuration(views: dict, name: str, layers: list, active: bool = True) -> dict: + """A copy of *views* with configuration *name* set to *layers* (overwritten, or added), made active when + *active*.""" + saved = copy.deepcopy(views) + saved.setdefault("encoding3d_views", FORMAT) + saved.setdefault("configurations", {})[name] = {"layers": copy.deepcopy(list(layers))} + if active: + saved["active"] = name + return saved + + +# ── Selection by scale (ParaView's "Select at scale") ───────────────────── + +DATA_STRUCTURE_FIELD = "encoding3d_data_structure" # field data of each reader block: its data structure's id +SCALES_FIELD = "encoding3d_scales" # and its graph's ranked entity sets, coarser first + + +def ranked_sets(data_structure) -> list: + """The names of a graph's ranked entity sets (its scales), coarser first; [] for a grid.""" + ranked = [entity_set for entity_set in data_structure.entity_sets if entity_set.rank is not None] + return [entity_set.name for entity_set in sorted(ranked, key=lambda entity_set: entity_set.rank)] + + +def scale_names(scales: list) -> list: + """The scales of several blocks (*scales*: each block's, coarser first) in one list, coarser first: the "Select at + scale" filter's drop-down.""" + longest_first = sorted(scales, key=len, reverse=True) + return list(dict.fromkeys(name for names in longest_first for name in names)) + + +def ancestor_ids(data_structure, entity_set: str, rows: np.ndarray) -> dict: + """{entity set: ids per row (NO_ID: none)} of the entities containing the entities at *rows* of *entity_set*, at + each coarser scale their complex chain reaches.""" + from openalea.encoding3d.model import NO_ID + rows = np.asarray(rows, dtype=np.int64) + sets = [data_structure.entity_set(name) for name in ranked_sets(data_structure)] + if not sets or data_structure.entity_set(entity_set).rank is None: + return {} + ids = np.concatenate([np.asarray(table.columns["id"], dtype=np.int64) for table in sets]) + complexes = np.concatenate([np.asarray(table.columns["complex"], dtype=np.int64) if "complex" in table.columns + else np.full(table.count, NO_ID, dtype=np.int64) for table in sets]) + owner = np.concatenate([np.full(table.count, number) for number, table in enumerate(sets)]) + order = np.argsort(ids, kind="stable") + ids, complexes, owner = ids[order], complexes[order], owner[order] + start = data_structure.entity_set(entity_set) + valid = rows >= 0 + current = np.where(valid, np.asarray(start.columns.get("complex", np.full(start.count, NO_ID)), + dtype=np.int64)[np.where(valid, rows, 0)], NO_ID) + found = {} + for _ in range(len(sets)): + if not (current != NO_ID).any() or not ids.size: + break + position = np.minimum(np.searchsorted(ids, current), ids.size - 1) + known = (current != NO_ID) & (ids[position] == current) + for number in np.unique(owner[position[known]]).tolist(): + here = known & (owner[position] == number) + name = sets[number].name + found.setdefault(name, np.full(rows.size, NO_ID, dtype=np.int64))[here] = current[here] + current = np.where(known, complexes[position], NO_ID) + return found + + +def id_array(scale: str) -> str: + """The reader's cell array of the ids at *scale*: ".id".""" + return f"{scale}.id" + + +def _ids(values) -> np.ndarray: + """Ids read back from a cell array (float64, NaN where no entity): int64, -1 where none.""" + values = np.asarray(values, dtype=np.float64).reshape(-1) + return np.where(np.isfinite(values), np.nan_to_num(values), -1).astype(np.int64) + + +def picked_entities(picked: list, scale: str) -> dict: + """{data structure id: sorted ids} of the entities at *scale* containing picked cells; *picked*: [(data structure + id, {cell array: values of the picked cells})], e.g. one entry per block of ParaView's extracted selection. A + ValueError when no picked cell has an entity at *scale* (a scale finer than the picked elements, or none + picked).""" + found = {} + for data_structure, arrays in picked: + if id_array(scale) not in arrays: + continue + ids = _ids(arrays[id_array(scale)]) + ids = ids[ids >= 0] + if ids.size: + found[data_structure] = sorted(set(found.get(data_structure, [])) | set(ids.tolist())) + if not found: + scales = sorted({name[:-3] for _, arrays in picked for name in arrays if name.endswith(".id")}) + raise ValueError(f"no picked cell has an entity at scale '{scale}'" + + (f" (they have {scales})" if scales else " (pick cells in the view first)")) + return found + + +def selection_mask(data_structure: str, arrays: dict, chosen: dict, scale: str) -> np.ndarray: + """The cells of a block (of *data_structure*, its cell arrays *arrays*) inside the entities *chosen* + ({data structure id: ids at *scale*}, picked_entities'): those whose ".id" is one of them, whatever + finer scale the block's entities are at; none in a block without that array (a grid, a coarser layer).""" + name = id_array(scale) + count = len(next(iter(arrays.values()))) if arrays else 0 + if name not in arrays or data_structure not in chosen: + return np.zeros(count, dtype=bool) + return np.isin(_ids(arrays[name]), np.asarray(chosen[data_structure], dtype=np.int64)) + + +def isolated(items: dict, selection: list) -> list: + """The items to hide to isolate a selection: of *items* ({name: visible}), the visible ones not in *selection* + (restored by clearing it).""" + return [name for name, visible in items.items() if visible and name not in selection] diff --git a/bindings/python/src/openalea/fspm_tf/vocabulary.py b/bindings/python/src/openalea/encoding3d/vocabulary.py similarity index 96% rename from bindings/python/src/openalea/fspm_tf/vocabulary.py rename to bindings/python/src/openalea/encoding3d/vocabulary.py index afc8fe4..195b1a6 100644 --- a/bindings/python/src/openalea/fspm_tf/vocabulary.py +++ b/bindings/python/src/openalea/encoding3d/vocabulary.py @@ -1,5 +1,5 @@ """ -The fspm-TF vocabulary (spec/vocabulary/):: +The encoding3d vocabulary (spec/vocabulary/):: standard_names.csv the standard names: canonical unit (UCUM), kind, I-ADOPT decomposition, scales, definition, and the names each platform gives the quantity (one column per platform) @@ -12,7 +12,7 @@ import json from functools import lru_cache -from openalea.fspm_tf import _spec +from openalea.encoding3d import _spec PLATFORMS = ("metafspm", "cplantbox", "groimp", "virtualplantlab") # the platform columns of standard_names.csv diff --git a/bindings/python/src/openalea/fspm_tf/__init__.py b/bindings/python/src/openalea/fspm_tf/__init__.py deleted file mode 100644 index bbc30cf..0000000 --- a/bindings/python/src/openalea/fspm_tf/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -fspm-TF: an exchange format for whole FSPM scenes. This package is its Python reference implementation: the codec -(openalea.fspm_tf.codec), the validator (openalea.fspm_tf.validate), the display copies (openalea.fspm_tf.display) and the vocabulary -(openalea.fspm_tf.vocabulary). See the specification in the repository's spec/ folder. -""" -SPECIFICATION_VERSION = "0.1" - -from openalea.fspm_tf.model import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, # noqa: E402 - Table, Variable) -from openalea.fspm_tf.codec import (from_bytes, read, read_index, read_sequence, to_bytes, write, # noqa: E402 - write_index, write_sequence) - -__all__ = ["SPECIFICATION_VERSION", "DataStructure", "EntitySet", "Frame", "Grid", "Mapping", "Mesh", "NO_ID", - "Relation", "Table", "Variable", "from_bytes", "read", "read_index", "read_sequence", "to_bytes", "write", - "write_index", "write_sequence"] diff --git a/bindings/python/src/openalea/fspm_tf/cli.py b/bindings/python/src/openalea/fspm_tf/cli.py deleted file mode 100644 index 247223e..0000000 --- a/bindings/python/src/openalea/fspm_tf/cli.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -The fspm-tf command line:: - - fspm-tf validate FILE... check .glb frames or sequences (scene.fspm.json); exit 1 on errors - fspm-tf info FILE what a frame holds - fspm-tf display FILE NAME... [-o OUT] add display copies of variables (in place, or to OUT) - fspm-tf display FILE --remove [-o OUT] remove them - fspm-tf compare ACTUAL EXPECTED a codec's JSON dump (or a .glb) against expected values; exit 1 if they - differ - fspm-tf compare-all DUMPS CONFORMANCE every conformance file's expected values against the dump at the same - path (x.expected.json: DUMPS/x.json); exit 1 if any differ -""" -import argparse -import sys - -from openalea.fspm_tf import read, write -from openalea.fspm_tf.display import add_display, remove_display -from openalea.fspm_tf.validate import validate_file - - -def _validate(arguments) -> int: - failed = False - for path in arguments.files: - report = validate_file(path) - print(report) - failed = failed or not report.ok - return 1 if failed else 0 - - -def _info(arguments) -> int: - frame = read(arguments.file) - print(f"{arguments.file}: time {frame.time} {frame.time_unit}, step {frame.step}, writer {frame.writer or '-'}") - for data_structure in frame.data_structures: - print(f" {data_structure.id} ({data_structure.kind}{', ' + data_structure.model if data_structure.model else ''})") - for entity_set in data_structure.entity_sets: - rank = "" if entity_set.rank is None else f" rank {entity_set.rank}" - print(f" {entity_set.name}{rank}: {entity_set.count} entities, variables " - f"{', '.join(entity_set.variable_names()) or '-'}") - for relation in data_structure.relations: - print(f" relation {relation.name} ({relation.source_set} -> {relation.target_set}): {relation.count}") - if data_structure.scalars is not None: - print(f" scalars: {', '.join(data_structure.scalars.columns)}") - for mesh in data_structure.meshes: - print(f" mesh {mesh.name}: {len(mesh.positions)} vertices, entity sets {', '.join(mesh.feature_rows)}" - f"{', display ' + ', '.join(mesh.display) if mesh.display else ''}") - for mapping in frame.mappings: - print(f" mapping {mapping.name}: {mapping.source[0]}/{mapping.source[1]} -> " - f"{mapping.target[0]}/{mapping.target[1]}, {mapping.count} pairs") - return 0 - - -def _display(arguments) -> int: - frame = read(arguments.file) - if arguments.remove: - remove_display(frame, arguments.names or None) - else: - if not arguments.names: - print("give the names of the variables to copy, or --remove", file=sys.stderr) - return 2 - add_display(frame, arguments.names, entity_set=arguments.entity_set) - write(arguments.output or arguments.file, frame) - return 0 - - -def _print_differences(name: str, differences: list, shown: int = 20) -> None: - print(f"{name}: {'equal' if not differences else f'{len(differences)} difference(s)'}") - for line in differences[:shown]: - print(f" {line}") - if len(differences) > shown: - print(f" ... {len(differences) - shown} more") - - -def _compare(arguments) -> int: - from openalea.fspm_tf.compare import compare_files - differences = compare_files(arguments.actual, arguments.expected) - _print_differences(arguments.actual, differences) - return 1 if differences else 0 - - -def _compare_all(arguments) -> int: - from openalea.fspm_tf.compare import compare_all - results = compare_all(arguments.dumps, arguments.conformance) - for name, differences in results.items(): - _print_differences(name, differences) - failed = sum(1 for differences in results.values() if differences) - print(f"{len(results) - failed} of {len(results)} conformance files read as expected") - return 1 if failed or not results else 0 - - -def main(argv=None) -> int: - parser = argparse.ArgumentParser(prog="fspm-tf", description="fspm-TF files: validate, inspect, display") - commands = parser.add_subparsers(dest="command", required=True) - validate = commands.add_parser("validate", help="check frames (.glb) or sequences (scene.fspm.json)") - validate.add_argument("files", nargs="+") - validate.set_defaults(run=_validate) - info = commands.add_parser("info", help="what a frame holds") - info.add_argument("file") - info.set_defaults(run=_info) - display = commands.add_parser("display", help="add or remove display copies of variables") - display.add_argument("file") - display.add_argument("names", nargs="*") - display.add_argument("--entity-set", default=None, help="the entity set the variables come from") - display.add_argument("--remove", action="store_true", help="remove display copies (all, or the names given)") - display.add_argument("-o", "--output", default=None, help="write to this file instead of in place") - display.set_defaults(run=_display) - compare = commands.add_parser("compare", help="a codec's JSON dump (or a .glb) against expected values") - compare.add_argument("actual") - compare.add_argument("expected") - compare.set_defaults(run=_compare) - compare_all = commands.add_parser("compare-all", help="dumps of every conformance file against their expected " - "values") - compare_all.add_argument("dumps", help="the folder of the codec's dumps (x.json for x.expected.json)") - compare_all.add_argument("conformance", help="fspm-TF's conformance folder") - compare_all.set_defaults(run=_compare_all) - arguments = parser.parse_args(argv) - return arguments.run(arguments) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bindings/python/src/openalea/fspm_tf/schema.py b/bindings/python/src/openalea/fspm_tf/schema.py deleted file mode 100644 index b9f3e8b..0000000 --- a/bindings/python/src/openalea/fspm_tf/schema.py +++ /dev/null @@ -1,12 +0,0 @@ -"""The JSON Schemas of fspm-TF (spec/schema/): fspm_scene.schema.json (the FSPM_scene extension) and -index.schema.json (a sequence's scene.fspm.json).""" -import json -from functools import lru_cache - -from openalea.fspm_tf import _spec - - -@lru_cache(maxsize=None) -def load(name: str) -> dict: - with open(_spec.path("schema", f"{name}.schema.json"), encoding="utf-8") as file: - return json.load(file) diff --git a/bindings/python/test/test_codec.py b/bindings/python/test/test_codec.py index b7677db..a3a068b 100644 --- a/bindings/python/test/test_codec.py +++ b/bindings/python/test/test_codec.py @@ -6,10 +6,10 @@ import pytest import generate -from openalea.fspm_tf import (DataStructure, EntitySet, Frame, Mesh, Table, Variable, from_bytes, read_index, read_sequence, +from openalea.encoding3d import (DataStructure, EntitySet, Frame, Mesh, Table, Variable, from_bytes, read_index, read_sequence, to_bytes, write_sequence) -from openalea.fspm_tf.dump import frame_to_dict -from openalea.fspm_tf.glb import unpack +from openalea.encoding3d.dump import frame_to_dict +from openalea.encoding3d.glb import unpack def same_columns(a, b): @@ -17,7 +17,11 @@ def same_columns(a, b): for name in a.columns: x, y = a.columns[name], b.columns[name] assert x.dtype == y.dtype and x.shape == y.shape, name - if x.dtype.kind == "f": + assert np.ma.isMaskedArray(x) == np.ma.isMaskedArray(y), name + if np.ma.isMaskedArray(x): # missing values: the same rows, the same others + assert np.array_equal(np.ma.getmaskarray(x), np.ma.getmaskarray(y)), name + assert np.array_equal(x.compressed(), y.compressed()), name + elif x.dtype.kind == "f": assert np.ascontiguousarray(x).tobytes() == np.ascontiguousarray(y).tobytes(), name # bit for bit else: assert np.array_equal(x, y), name @@ -36,6 +40,8 @@ def test_round_trip(name): for a, b in zip(written.entity_sets + written.relations, read.entity_sets + read.relations): same_columns(a, b) assert [entity_set.rank for entity_set in written.entity_sets] == [s.rank for s in read.entity_sets] + assert [entity_set.scale for entity_set in written.entity_sets] == [s.scale for s in read.entity_sets] + assert written.labels == read.labels if written.scalars is not None: same_columns(written.scalars, read.scalars) assert written.grid == read.grid @@ -59,6 +65,16 @@ def test_glb_layout(): assert "FSPM_scene" not in document.get("extensionsRequired", []) +def test_buffer_length_holds_the_chunk_padding(): + """A binary chunk of 4 bytes of data is padded to 8: the buffer's byteLength counts the padding, so that the + chunk is no longer than the buffer (glTF-Validator: BUFFER_GLB_CHUNK_TOO_BIG beyond 3 bytes).""" + frame = Frame(data_structures=[DataStructure("plants", entity_sets=[ + EntitySet("Plant", columns={"id": np.array([0], dtype=np.int64), "count": np.array([1], dtype=np.int32)}, + rank=1)])]) + document, binary = unpack(to_bytes(frame)) + assert len(binary) % 8 == 0 and document["buffers"][0]["byteLength"] == len(binary) + + def test_column_types(): columns = dict(id=np.arange(3, dtype=np.int64), f64=np.array([0.1, np.nan, -2.5e-300]), infinite=np.array([np.inf, -np.inf, np.nan]), infinite32=np.array([1., np.inf, -np.inf], dtype=np.float32), diff --git a/bindings/python/test/test_compare.py b/bindings/python/test/test_compare.py index 8d4502f..09e4a27 100644 --- a/bindings/python/test/test_compare.py +++ b/bindings/python/test/test_compare.py @@ -1,4 +1,4 @@ -"""Comparing a codec's dumps with the conformance files' expected values (fspm-tf compare, compare-all).""" +"""Comparing a codec's dumps with the conformance files' expected values (encoding3d compare, compare-all).""" import json import os import shutil @@ -6,8 +6,8 @@ import pytest from conftest import CONFORMANCE -from openalea.fspm_tf.cli import main -from openalea.fspm_tf.compare import compare, compare_all +from openalea.encoding3d.cli import main +from openalea.encoding3d.compare import compare, compare_all @pytest.mark.parametrize("actual, expected, equal", [ @@ -39,7 +39,7 @@ def test_compare_all(tmp_path): target.parent.mkdir(parents=True, exist_ok=True) shutil.copy(os.path.join(folder, name), target) results = compare_all(str(dumps), CONFORMANCE) - assert len(results) == 9 and not any(results.values()) + assert len(results) == 10 and not any(results.values()) assert main(["compare-all", str(dumps), CONFORMANCE]) == 0 with open(dumps / "root_system.json") as file: dump = json.load(file) diff --git a/bindings/python/test/test_conformance.py b/bindings/python/test/test_conformance.py index 68aa410..df52685 100644 --- a/bindings/python/test/test_conformance.py +++ b/bindings/python/test/test_conformance.py @@ -6,9 +6,9 @@ import generate from conftest import CONFORMANCE -from openalea.fspm_tf import from_bytes -from openalea.fspm_tf.dump import frame_to_dict -from openalea.fspm_tf.validate import validate_file +from openalea.encoding3d import from_bytes +from openalea.encoding3d.dump import frame_to_dict +from openalea.encoding3d.validate import validate_file FILES = generate.files() diff --git a/bindings/python/test/test_coupling.py b/bindings/python/test/test_coupling.py new file mode 100644 index 0000000..bba44fe --- /dev/null +++ b/bindings/python/test/test_coupling.py @@ -0,0 +1,435 @@ +""" +Coupling sessions (openalea.encoding3d.coupling): two participants on one plant at two scales, a grower owning the +organs and a function refining them into segments and feeding a supply back, over ZeroMQ and over a folder; the +resolution's checks and the rights; a recorded session replayed as a participant. +""" +import os +import threading + +import numpy as np +import pytest + +from openalea import encoding3d +from openalea.encoding3d.coupling import (Coordinator, CouplingError, Declaration, EntitySetSpec, Link, Participant, + VariableSpec, resolve, stages) +from openalea.encoding3d.model import NO_ID +from openalea.encoding3d.validate import validate_sequence + +pytest.importorskip("zmq") + +# The command running Julia with encoding3d and ZMQ (e.g. "julia --project=env"): the tests with a Julia participant +JULIA = os.environ.get("ENCODING3D_JULIA") +JULIA_FUNCTION = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "julia", "encoding3d", "test", "coupling", + "function.jl") + +# The session's links: the segments' length is half their organ's (copied from the coarser organs); the organs' +# supply is their segments' sugar (summed, mol to umol) +LINKS = [Link("function", "segment_length", "grower", "length", factor=0.5, aggregation="copy"), + Link("grower", "supply", "function", "sugar")] + + +def grower_declaration(name="grower"): + return Declaration(name, "test", [EntitySetSpec("plant", "Organ", rights="owns")], [ + VariableSpec("plant", "Organ", "length", "provides", "cm", "length", "extensive"), + VariableSpec("plant", "Organ", "supply", "requires", "umol", kind="extensive")]) + + +def function_declaration(name="function", rights="refines"): + return Declaration(name, "test", [EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights=rights)], [ + VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol", kind="extensive")]) + + +def organ_frame(name, organs, removed=None): + """The grower's update: its organs in a chain, each 2 cm longer than the previous one.""" + count = len(organs) + columns = {"id": np.array(organs, dtype=np.int64), "length": 2. * np.arange(1, count + 1), + "parent": np.array([NO_ID] + organs[:-1], dtype=np.int64), + "edge_type": np.array(["none"] + ["successor"] * (count - 1), dtype=object), + "label": np.array(["Leaf"] * count, dtype=object)} + return encoding3d.Frame(data_structures=[encoding3d.DataStructure("plant", entity_sets=[ + encoding3d.EntitySet("Organ", rank=5, columns=columns)])], update=encoding3d.Update(name, removed or {})) + + +def run_grower(endpoint, log): + """One organ more each step; the first one removed at step 3. The supply it receives is logged.""" + participant = Participant.connect(grower_declaration(), endpoint, timeout=300) + organs = [] + for turn in participant.turns(): + organ_set = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + if organ_set is not None and organ_set.count: + log[turn.step] = dict(zip(organ_set.columns["id"].tolist(), + np.asarray(organ_set.columns["supply"]).tolist())) + organs.append(participant.id_offset + turn.step) + removed = {} + if turn.step == 3: + removed = {("plant", "Organ"): [organs.pop(0)]} + participant.send(organ_frame("grower", organs, removed)) + + +def run_function(endpoint, log, rights="refines"): + """Two segments under each new organ; each segment's sugar 1e-6 mol per m of its length.""" + participant = Participant.connect(function_declaration(rights=rights), endpoint, timeout=30) + segments = {} # segment id: (its organ, its parent) + for turn in participant.turns(): + plant = turn.frame.data_structure("plant") + removed = turn.frame.update.removed.get(("plant", "SubOrgan"), np.zeros(0)).tolist() + log[turn.step] = sorted(removed) + for segment in removed: + segments.pop(segment, None) # removed with their organ by the coordinator + lengths = {} + try: + known = plant.entity_set("SubOrgan") + lengths = dict(zip(known.columns["id"].tolist(), np.asarray(known.columns["segment_length"]).tolist())) + except KeyError: + pass + with_segments = {organ for organ, _ in segments.values()} + for organ in plant.entity_set("Organ").columns["id"].tolist(): + if organ not in with_segments: + first = participant.id_offset + len(segments) + 2 * turn.step * 1000 + segments[first], segments[first + 1] = (organ, NO_ID), (organ, first) + ids = list(segments) + columns = {"id": np.array(ids, dtype=np.int64), + "complex": np.array([segments[segment][0] for segment in ids], dtype=np.int64), + "parent": np.array([segments[segment][1] for segment in ids], dtype=np.int64), + "edge_type": np.array(["none" if segments[segment][1] == NO_ID else "successor" + for segment in ids], dtype=object), + "sugar": np.array([1e-6 * np.nan_to_num(lengths.get(segment, 0.)) for segment in ids])} + participant.send(encoding3d.Frame(data_structures=[encoding3d.DataStructure("plant", entity_sets=[ + encoding3d.EntitySet("SubOrgan", rank=6, columns=columns)])])) + + +def run_session(endpoint, record=None, function=run_function, **options): + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["grower", "function"], links=LINKS, + record=record, timeout=30, **options) + logs = {"grower": {}, "function": {}} + threads = [threading.Thread(target=target, args=(coordinator.endpoint, logs[name]), daemon=True) + for name, target in (("grower", run_grower), ("function", function))] + for thread in threads: + thread.start() + try: + store = coordinator.run() + finally: + for thread in threads: + thread.join(30) + return store, logs + + +@pytest.fixture(params=["zmq", "folder"]) +def endpoint(request, tmp_path): + return "tcp://127.0.0.1:*" if request.param == "zmq" else f"files:{tmp_path / 'session'}" + + +def check_session(store, supply, removed, record): + first, second, third = (2 ** 32 + step for step in range(3)) + # a segment's length: half its organ's (0.5 * 0.01 m/cm); its sugar 1e-6 mol/m of it; summed and in umol: + # an organ's supply is 0.01 umol per cm of the organ, from the step after its segments were made + assert supply[1] == pytest.approx({first: 0.}) + assert supply[2] == pytest.approx({first: 0.02, second: 0.}) + assert supply[3] == pytest.approx({first: 0.02, second: 0.04, third: 0.}) + assert set(supply[4]) == {second, third, third + 1} # the first organ removed at step 3 + assert supply[4][second] == pytest.approx(0.02) # now the first of the chain: 2 cm + if removed is not None: + assert len(removed[3]) == 2 # its segments, removed with it in the step + assert removed[4] == [] + organs = store.sets[("plant", "Organ")] + assert sorted(organs.ids.tolist()) == [second, third, third + 1, third + 2] + assert len(store.sets[("plant", "SubOrgan")]) == 8 + report = validate_sequence(os.path.join(record, "scene.fspm.json")) + assert report.ok, report + last = encoding3d.read(os.path.join(record, "frame_00004.glb")) + assert set(last.data_structure("plant").entity_set("SubOrgan").columns) >= {"id", "complex", "sugar"} + + +def test_two_platforms_on_one_plant_at_two_scales(endpoint, tmp_path): + store, logs = run_session(endpoint, record=str(tmp_path / "record")) + check_session(store, logs["grower"], logs["function"], str(tmp_path / "record")) + + +@pytest.mark.skipif(not JULIA, reason="ENCODING3D_JULIA (the command running Julia with encoding3d and ZMQ) is not set") +def test_a_julia_participant(endpoint, tmp_path): + """The function participant in Julia (encoding3d.jl), launched by the coordinator.""" + record = str(tmp_path / "record") + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["grower"], links=LINKS, record=record, + timeout=300) + coordinator.launch("function", f"{JULIA} {JULIA_FUNCTION}") + log = {} + thread = threading.Thread(target=run_grower, args=(coordinator.endpoint, log), daemon=True) + thread.start() + store = coordinator.run() + thread.join(30) + check_session(store, log, None, record) + + +def test_the_messages_follow_their_schema(tmp_path): + """Every message of a session (the folder transport's log: both directions) is valid against + spec/schema/coupling.schema.json.""" + import glob + import json + import jsonschema + from openalea.encoding3d.schema import load + run_session(f"files:{tmp_path / 'session'}", keep_messages=True) + validator = jsonschema.Draft202012Validator(load("coupling")) + headers = sorted(glob.glob(str(tmp_path / "session" / "*" / "*" / "*.json"))) + types = set() + for path in headers: + with open(path, encoding="utf-8") as file: + header = json.load(file) + errors = list(validator.iter_errors(header)) + assert not errors, f"{path}: {errors[0].message}" + types.add(header["type"]) + assert types == {"hello", "welcome", "turn", "update", "stop"} + + +def run_follower_adding(endpoint, log): + run_function(endpoint, log, rights="follows") + + +def test_a_participant_adding_entities_it_follows_stops_the_session(tmp_path): + with pytest.raises(CouplingError, match="which it follows"): + run_session("tcp://127.0.0.1:*", function=run_follower_adding) + + +def test_resolution(): + declarations = {"grower": grower_declaration(), "function": function_declaration()} + with pytest.raises(CouplingError, match="extensive variable copied"): + resolve(declarations, [LINKS[1]]) # length by standard name, copied to segments + with pytest.raises(CouplingError, match="no participant provides"): + resolve(declarations, [LINKS[0]]) # the supply unlinked + couplings = {coupling.variable.name: coupling for coupling in resolve(declarations, LINKS)} + assert couplings["segment_length"].factor == pytest.approx(0.005) + assert couplings["supply"].aggregation == "sum" and couplings["supply"].factor == pytest.approx(1e6) + assert stages(declarations) == [["grower"], ["function"]] + with pytest.raises(CouplingError, match="owned by"): + resolve({"a": grower_declaration("a"), "b": grower_declaration("b")}) + with pytest.raises(CouplingError, match="not convertible"): + resolve(declarations, [LINKS[0], Link("grower", "supply", "grower", "length")]) + with pytest.raises(CouplingError, match="both change"): + stages({"a": function_declaration("a"), "b": function_declaration("b")}, [["a", "b"]]) + + +def run_reader(endpoint, log): + declaration = Declaration("reader", "test", [EntitySetSpec("plant", "Organ")], [ + VariableSpec("plant", "Organ", "length", "requires", "cm", "length")]) + participant = Participant.connect(declaration, endpoint, timeout=30) + for turn in participant.turns(): + organs = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + log[turn.until] = [] if organs is None else np.asarray(organs.columns["length"]).tolist() + participant.send(encoding3d.Frame()) + + +def test_a_recorded_session_replayed(tmp_path): + run_session("tcp://127.0.0.1:*", record=str(tmp_path / "record")) + # the recorded organ lengths: 0 d [2], 1 d [2, 4], 2 d [2, 4, 6], 3 d [2, 4, 6] (the first removed, the chain + # renumbered), 4 d [2, 4, 6, 8] + for interpolation, expected in (("hold", [2., 4., 6.]), ("linear", [2., 3., 5.])): + coordinator = Coordinator("tcp://127.0.0.1:*", steps=8, interval=0.5, expect=["reader"], timeout=30, + replays={"recorded": str(tmp_path / "record" / "scene.fspm.json")}, + links=[Link("reader", "length", "recorded", "length", interpolation=interpolation)]) + log = {} + thread = threading.Thread(target=run_reader, args=(coordinator.endpoint, log), daemon=True) + thread.start() + coordinator.run() + thread.join(30) + assert log[0.] == [2.] + assert log[1.5] == pytest.approx([2., 4.]) + assert log[2.5] == pytest.approx(expected) # between 2 d and 3 d + assert log[3.] == pytest.approx([2., 4., 6.]) # after the removal + assert len(log[4.]) == 4 + + +READER_SCRIPT = """ +import numpy as np +from openalea import encoding3d +from openalea.encoding3d.coupling import Declaration, EntitySetSpec, Participant, VariableSpec +declaration = Declaration("", "test", [EntitySetSpec("plant", "Organ")], + [VariableSpec("plant", "Organ", "length", "requires", "m", "length")]) +participant = Participant.connect(declaration, timeout=60) +for turn in participant.turns(): + participant.send(encoding3d.Frame()) +""" + + +def test_the_command_line(tmp_path): + """encoding3d couple: a launched participant (its name and endpoint from the environment) and a replay, over a + folder, recorded.""" + import sys + from openalea.encoding3d.cli import main + run_session("tcp://127.0.0.1:*", record=str(tmp_path / "record")) + script = tmp_path / "reader.py" + script.write_text(READER_SCRIPT) + code = main(["couple", "--steps", "3", "--interval", "1", "--endpoint", f"files:{tmp_path / 'session'}", + "--replay", f"recorded={tmp_path / 'record' / 'scene.fspm.json'}", + "--participant", f"reader={sys.executable} {script}", "--record", str(tmp_path / "again"), + "--timeout", "60"]) + assert code == 0 + assert validate_sequence(str(tmp_path / "again" / "scene.fspm.json")).ok + assert not list((tmp_path / "session" / "reader" / "to_coordinator").iterdir()) # each message read, deleted + + +WATER = np.array([0.1, 0.2, 0.3]) # the soil's three cells + + +def run_plant(endpoint, log): + """Two organs, then three; each takes up 1 mol per step and logs the water it receives from the soil.""" + from openalea.encoding3d.coupling import Declaration as D + declaration = D("plant", "test", [EntitySetSpec("plant", "Organ", rights="owns")], [ + VariableSpec("plant", "Organ", "uptake", "provides", "mol", kind="extensive"), + VariableSpec("plant", "Organ", "water", "requires", "m3/m3", kind="intensive")]) + participant = Participant.connect(declaration, endpoint, timeout=60) + for turn in participant.turns(): + organs = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + if organs is not None and organs.count: + log[turn.step] = dict(zip(organs.columns["id"].tolist(), np.asarray(organs.columns["water"]).tolist())) + count = 2 if turn.step < 2 else 3 + ids = participant.id_offset + np.arange(count) + participant.send(encoding3d.Frame(data_structures=[encoding3d.DataStructure("plant", entity_sets=[ + encoding3d.EntitySet("Organ", rank=5, columns={"id": ids, "uptake": np.ones(count)})])])) + + +def run_soil(endpoint, log): + """Three cells; organ i in cells i % 3 (weight 1) and (i + 1) % 3 (weight 3): the mapping it provides.""" + from openalea.encoding3d.coupling import MappingSpec + declaration = Declaration("soil", "test", [EntitySetSpec("soil", "Cell", rank=1, rights="owns"), + EntitySetSpec("plant", "Organ")], [ + VariableSpec("soil", "Cell", "water_content", "provides", "m3/m3", kind="intensive"), + VariableSpec("soil", "Cell", "uptake", "requires", "mol", kind="extensive")], + mappings=[MappingSpec("organs_in_soil", ("plant", "Organ"), ("soil", "Cell"))]) + participant = Participant.connect(declaration, endpoint, timeout=60) + cells = participant.id_offset + np.arange(3) + for turn in participant.turns(): + if turn.frame.data_structures and any(item.id == "soil" for item in turn.frame.data_structures): + soil = turn.frame.data_structure("soil").entity_set("Cell") + log[turn.step] = np.asarray(soil.columns["uptake"]).tolist() + organs = [] + if any(item.id == "plant" for item in turn.frame.data_structures): + organs = turn.frame.data_structure("plant").entity_set("Organ").columns["id"].tolist() + source = np.repeat(np.array(organs, dtype=np.int64), 2) + target = np.array([cells[(i + k) % 3] for i in range(len(organs)) for k in (0, 1)], dtype=np.int64) + weight = np.tile([1., 3.], len(organs)) + frame = encoding3d.Frame(data_structures=[encoding3d.DataStructure("soil", kind="table", entity_sets=[ + encoding3d.EntitySet("Cell", rank=1, columns={"id": cells, "water_content": WATER})])]) + frame.mappings.append(encoding3d.Mapping("organs_in_soil", source=("plant", "Organ"), target=("soil", "Cell"), + columns={"source": source, "target": target, "weight": weight})) + participant.send(frame) + + +def test_a_link_between_two_data_structures(endpoint, tmp_path): + """The plant's organs and the soil's cells, two data structures of two participants, linked by the mapping the + soil provides: the organs' water, the cells' weighted mean; the cells' uptake, the organs' shared by weight.""" + links = [Link("plant", "water", "soil", "water_content", mapping="organs_in_soil"), + Link("soil", "uptake", "plant", "uptake", mapping="organs_in_soil")] + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["plant", "soil"], links=links, timeout=60, + record=str(tmp_path / "record")) + logs = {"plant": {}, "soil": {}} + threads = [threading.Thread(target=target, args=(coordinator.endpoint, logs[name]), daemon=True) + for name, target in (("plant", run_plant), ("soil", run_soil))] + for thread in threads: + thread.start() + coordinator.run() + for thread in threads: + thread.join(30) + first = 2 ** 32 + # explicit coupling, the stage [plant, soil]: at step 4 the mapping the soil sent at step 3 (from the plant's + # three organs of step 2) and the uptake the plant sent at step 3 + water = logs["plant"][4] + for i in range(3): + assert water[first + i] == pytest.approx((WATER[i % 3] + 3 * WATER[(i + 1) % 3]) / 4) + uptake = logs["soil"][4] + assert uptake == pytest.approx([1., 1., 1.]) # 1/4 of one organ and 3/4 of another, each + assert validate_sequence(str(tmp_path / "record" / "scene.fspm.json")).ok + with pytest.raises(CouplingError, match="need a mapping"): + resolve({"plant": Declaration.from_json(Declaration("plant", entity_sets=[], variables=[ + VariableSpec("plant", "Organ", "water", "requires", "1")]).to_json()), + "soil": Declaration("soil", variables=[VariableSpec("soil", "Cell", "water_content", "provides", + "1")]).complete()}, + [Link("plant", "water", "soil", "water_content")]) + + +def test_the_conformance_sessions(tmp_path): + """Python's participant against conformance/coupling, over both transports (encoding3d couple-conformance).""" + import sys + from openalea.encoding3d.cli import main + sessions = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "conformance", "coupling") + command = f"{sys.executable} -m openalea.encoding3d.coupling.conformance subject" + assert main(["couple-conformance", sessions, "--participant", command, "--out", str(tmp_path)]) == 0 + + +def test_the_conformance_sessions_are_what_their_generator_writes(tmp_path): + import importlib.util + folder = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "conformance", "coupling") + spec = importlib.util.spec_from_file_location("coupling_generate", os.path.join(folder, "generate.py")) + generate = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generate) + for name, session in generate.SESSIONS.items(): + generate.generate(name, session, str(tmp_path)) + for root, _, files in os.walk(tmp_path / name): + for file_name in files: + written = os.path.join(root, file_name) + committed = os.path.join(folder, os.path.relpath(written, tmp_path)) + with open(written, "rb") as left, open(committed, "rb") as right: + assert left.read() == right.read(), committed + + +# Platforms' logs replayed (their index, scene.fspm.json): CPlantBox's (bindings/cpp/cplantbox's tests write +# anagallis_log/) and GroIMP's (bindings/java/groimp-plugin/test/run_in_groimp.sh writes log/), when those adapters are +# tested by hand (their README.md): encoding3d's CI depends on no platform +PLATFORM_LOGS = {"conformance": os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, + "conformance", "growing_sequence", "scene.fspm.json"), + "CPlantBox": os.environ.get("ENCODING3D_CPLANTBOX_LOG"), "GroIMP": os.environ.get("ENCODING3D_GROIMP_LOG")} + + +def _sorted(values): + """The values, missing ones (NaN, masked) as None, sorted: to compare a column whatever its rows' order.""" + def plain(value): + if isinstance(value, list): + return [plain(item) for item in value] + return None if value is None or (isinstance(value, float) and value != value) else value + values = np.ma.filled(np.ma.asarray(values, dtype=object), None).tolist() if np.ma.isMaskedArray(values) \ + else np.asarray(values).tolist() + return sorted((plain(value) for value in values), key=repr) + + +@pytest.mark.parametrize("platform", sorted(PLATFORM_LOGS)) +def test_a_platform_log_replayed(platform, tmp_path): + """A platform's log as a participant: a reader following every entity set and requiring every variable gets, at + each step, the logged entities (renumbered) and values.""" + from openalea.encoding3d.coupling import Replay + from openalea.encoding3d.codec import read_index + index = PLATFORM_LOGS[platform] + if not index: + pytest.skip(f"no {platform} log (ENCODING3D_{platform.upper()}_LOG)") + entries = read_index(index)["frames"] + times = [entry["time"] for entry in entries] + replay = Replay("log", index) # its declaration: what the reader requires + declaration = Declaration("reader", "test", [EntitySetSpec(item.data_structure, item.name, item.rank) + for item in replay.declaration.entity_sets], + [VariableSpec(item.data_structure, item.entity_set, item.name, "requires", item.unit, + kind=item.kind) for item in replay.declaration.variables]) + log = {} + + def reader(endpoint): + participant = Participant.connect(declaration, endpoint, timeout=120) + for turn in participant.turns(): + log[turn.step] = {(data_structure.id, entity_set.name): entity_set + for data_structure in turn.frame.data_structures + for entity_set in data_structure.entity_sets} + participant.send(encoding3d.Frame()) + + interval = times[1] - times[0] if len(times) > 1 else 1. + coordinator = Coordinator("tcp://127.0.0.1:*", steps=len(entries) - 1, interval=interval, start=times[0], + time_unit=read_index(index).get("timeUnit", "d"), expect=["reader"], + replays={"log": index}, timeout=120) + thread = threading.Thread(target=reader, args=(coordinator.endpoint,), daemon=True) + thread.start() + coordinator.run() + thread.join(60) + for step, entry in enumerate(entries): + frame = encoding3d.read(entry["path"]) + for data_structure in frame.data_structures: + for entity_set in data_structure.entity_sets: + received = log[step][(data_structure.id, entity_set.name)] + assert received.count == entity_set.count, (step, entity_set.name) + for name in entity_set.variable_names(): + assert _sorted(received.columns[name]) == _sorted(entity_set.columns[name]), (step, name) diff --git a/bindings/python/test/test_lazy.py b/bindings/python/test/test_lazy.py new file mode 100644 index 0000000..157f360 --- /dev/null +++ b/bindings/python/test/test_lazy.py @@ -0,0 +1,50 @@ +""" +Frames read lazily (read(path, lazy=True)): the same frame as read whole, each column decoded only when read, the meshes +too, written back the same, and the file released by close() or a with block. +""" +import os + +import pytest + +from openalea import encoding3d +from openalea.encoding3d.dump import frame_to_dict +from openalea.encoding3d.model import LazyColumns + +from conftest import CONFORMANCE + +FILES = sorted(os.path.join(folder, name) for folder, _, names in os.walk(CONFORMANCE) for name in names + if name.endswith(".glb") and "coupling" not in os.path.relpath(folder, CONFORMANCE)) + + +@pytest.mark.parametrize("path", FILES, ids=[os.path.relpath(path, CONFORMANCE) for path in FILES]) +def test_read_lazily_the_same(path): + with encoding3d.read(path, lazy=True) as frame: + assert frame_to_dict(frame) == frame_to_dict(encoding3d.read(path)) + assert encoding3d.to_bytes(frame) == encoding3d.to_bytes(encoding3d.read(path)) # written back the same + + +def test_only_what_is_read_is_decoded(): + frame = encoding3d.read(os.path.join(CONFORMANCE, "root_system.glb"), lazy=True) + segments = frame.data_structure("root_system").entity_set("SubOrgan") + assert isinstance(segments.columns, LazyColumns) + assert segments.count == 10 and "length" in segments.columns and segments.columns.decoded() == [] + assert segments.variables["length"].unit == "m" # declarations: from the JSON + lengths = segments.columns["length"] + assert segments.columns.decoded() == ["length"] and lengths.size == 10 + meshes = frame.data_structure("root_system").meshes + assert meshes._items[0][0] is False # not decoded until read + assert len(meshes[0].positions) > 0 + frame.close() + with pytest.raises(ValueError): # the file released + segments.columns["radius"] + assert lengths.sum() > 0 # what was read stays + + +def test_a_lazy_column_set_and_deleted(): + frame = encoding3d.read(os.path.join(CONFORMANCE, "root_system.glb"), lazy=True) + segments = frame.data_structure("root_system").entity_set("SubOrgan") + segments.columns["doubled"] = 2 * segments.columns["length"] + del segments.columns["radius"] + again = encoding3d.from_bytes(encoding3d.to_bytes(frame)).data_structure("root_system").entity_set("SubOrgan") + assert "radius" not in again.columns and (again.columns["doubled"] == 2 * again.columns["length"]).all() + frame.close() diff --git a/bindings/python/test/test_paraview_reader.py b/bindings/python/test/test_paraview_reader.py index 0563251..313a592 100644 --- a/bindings/python/test/test_paraview_reader.py +++ b/bindings/python/test/test_paraview_reader.py @@ -1,6 +1,6 @@ -"""The ParaView reader's conversion (paraview/fspm_tf_reader.py), with VTK alone: meshes with every entity set's -variables as cell data, segments as lines, grids in VTK's cell order, sequences as time steps. Run inside ParaView -too when FSPM_TF_PVPYTHON gives a pvpython.""" +"""The ParaView reader's conversion (paraview/plugin_encoding3d.py), with VTK alone: meshes with every entity set's +variables as cell data, segments as lines (a views layer's as tubes), grids in VTK's cell order, sequences as time +steps. Run inside ParaView too when ENCODING3D_PVPYTHON gives a pvpython.""" import importlib.util import os import subprocess @@ -14,8 +14,8 @@ pytest.importorskip("vtkmodules") from vtkmodules.util.numpy_support import vtk_to_numpy # noqa: E402 -PLUGIN = os.path.join(ROOT, "paraview", "fspm_tf_reader.py") -_specification = importlib.util.spec_from_file_location("fspm_tf_reader", PLUGIN) +PLUGIN = os.path.join(ROOT, "paraview", "plugin_encoding3d.py") +_specification = importlib.util.spec_from_file_location("plugin_encoding3d", PLUGIN) reader = importlib.util.module_from_spec(_specification) _specification.loader.exec_module(reader) @@ -104,13 +104,180 @@ def test_a_sequence_is_time_steps(): source = reader.Source(os.path.join(CONFORMANCE, "growing_sequence", "scene.fspm.json")) assert source.times == [0., 1., 2.] assert source.frame_at(1.5).time == 1. and source.frame_at(None).time == 0. and source.frame_at(9.).time == 2. + # the reader's Time Unit: the times ParaView shows, converted from the files' (days here), the files unchanged + assert source.time_unit == "d" + assert source.factor("d") == 1. and source.factor("h") == 24. and source.factor("s") == 86400. + assert source.factor("") == 1. and source.factor("week") == 1. # empty or unknown: the files' times def test_inside_paraview(): - pvpython = os.environ.get("FSPM_TF_PVPYTHON") + pvpython = os.environ.get("ENCODING3D_PVPYTHON") if not pvpython: - pytest.skip("FSPM_TF_PVPYTHON (a pvpython) not given") + pytest.skip("ENCODING3D_PVPYTHON (a pvpython) not given") script = os.path.join(ROOT, "paraview", "check_in_paraview.py") result = subprocess.run([pvpython, script, CONFORMANCE], capture_output=True, text=True, timeout=600) assert result.returncode == 0, result.stdout + result.stderr - assert "fspm-TF reader: ok" in result.stdout + assert "encoding3d reader: ok" in result.stdout + + +def test_missing_values_are_nan_and_code_minus_one(): + frame = generate.root_system() + segments = frame.data_structures[0].entity_set("SubOrgan") + count = segments.count + segments.columns["tissue"] = np.array([None] + ["cortex"] * (count - 1), dtype=object) + segments.columns["rank"] = np.ma.MaskedArray(np.arange(count, dtype=np.int32), mask=[True] + [False] * (count - 1)) + lines = blocks_of(frame)[("root_system", "SubOrgan segments")] + assert np.isnan(cell_array(lines, "SubOrgan.rank")[0]) and cell_array(lines, "SubOrgan.rank")[1] == 1. + assert cell_array(lines, "SubOrgan.tissue_code")[0] == -1 and cell_array(lines, "SubOrgan.tissue_code")[1] == 0 + + +# ── Views: one block per layer ─────────────────────────────────────────── + +def layers_of(frame, views, configuration=None) -> dict: + output = reader.frame_to_vtk(frame, views=views, configuration=configuration) + return {output.GetMetaData(number).Get(output.NAME()): output.GetBlock(number) + for number in range(output.GetNumberOfBlocks())} + + +def views_module(): + from openalea.encoding3d import views + return views + + +def test_views_give_a_block_per_layer_with_its_entities_only(): + from test_views import roots_views + frame = generate.root_system() + source = frame.data_structures[0] + blocks = layers_of(frame, roots_views()) + assert list(blocks) == ["first order", "roots"] # the hidden layer too + mesh = blocks["first order"] + triangles = source.meshes[0].feature_rows["Axis"][::3] + first_order = source.entity_set("Axis").columns["branching_order"][triangles] == 1 + assert 0 < mesh.GetNumberOfCells() == first_order.sum() < len(triangles) + assert set(cell_array(mesh, "Axis.branching_order")) == {1.} + assert mesh.GetCellData().GetScalars().GetName() == "SubOrgan.water_potential" + segments = source.entity_set("SubOrgan") + tubes = blocks["roots"] # a "tube": drawn as tubes + kept = (segments.columns["label"] == "RootSegment") & (segments.columns["length"] > 0) + assert tubes.GetNumberOfCells() == tubes.GetNumberOfStrips() == kept.sum() # a strip per segment + assert tubes.GetNumberOfPoints() == 2 * reader.TUBE_SIDES * kept.sum() + ring = vtk_to_numpy(tubes.GetPoints().GetData())[:2 * reader.TUBE_SIDES:2] # the first tube's first end + start = np.array([segments.columns[name][kept][0] for name in ("x1", "y1", "z1")]) + np.testing.assert_allclose(np.linalg.norm(ring - start, axis=1), 2 * segments.columns["radius"][kept][0]) + np.testing.assert_array_equal(cell_array(tubes, "SubOrgan.radius"), segments.columns["radius"][kept]) + radius = vtk_to_numpy(tubes.GetPointData().GetArray("tube_radius")) + np.testing.assert_allclose(np.unique(radius), np.unique(segments.columns["radius"][kept] * 2)) # its scale + bounds = np.array(tubes.GetBounds()).reshape(3, 2) + ends = np.stack([segments.columns[name][kept] for name in ("x1", "y1", "z1")], axis=1) + assert (bounds[:, 0] <= ends.min(axis=0)).all() and (bounds[:, 1] >= ends.max(axis=0)).all() + assert tubes.GetCellData().GetScalars().GetName() == "SubOrgan.label_code" # text: its codes + assert tubes.GetFieldData().GetAbstractArray(views_module().DATA_STRUCTURE_FIELD) is not None + plain = layers_of(frame, roots_views(), "plain")["all"] + assert plain.GetNumberOfCells() == len(triangles) and plain.GetCellData().GetScalars() is None + + +def test_grid_layers_hide_the_cells_not_selected(): + frame = generate.soil_and_roots() + layer = {"name": "wet", "source": {"data_structure": "soil", "kind": "grid"}, + "select": {"entity_set": "Voxel", "where": {"soil_water_potential": ">=-15000"}}, + "color_by": {"entity_set": "Voxel", "variable": "soil_water_potential"}, "opacity": 0.3} + image = layers_of(frame, {"encoding3d_views": 1, "configurations": {"soil": {"layers": [layer]}}})["wet"] + ghosts = vtk_to_numpy(image.GetCellData().GetArray("vtkGhostType")) + voxels = frame.data_structures[1].entity_set("Voxel") + order = np.arange(12).reshape((2, 2, 3)).ravel(order="F") + np.testing.assert_array_equal(ghosts == 0, voxels.columns["soil_water_potential"][order] >= -15000) + assert 0 < (ghosts == 0).sum() < 12 + assert image.GetCellData().GetScalars().GetName() == "Voxel.soil_water_potential" + adaptive = generate.adaptive_soil() + layer = {"name": "fine", "source": {"data_structure": "soil", "kind": "grid"}, + "select": {"entity_set": "Cell", "where": {"level": ">0"}}} + cells = layers_of(adaptive, {"encoding3d_views": 1, "configurations": {"soil": {"layers": [layer]}}})["fine"] + assert cells.GetNumberOfCells() == (adaptive.data_structures[0].entity_set("Cell").columns["level"] > 0).sum() + + +def test_views_from_a_file_and_wrong_names(tmp_path): + from openalea.encoding3d import views + from test_views import roots_views + path = str(tmp_path / "views.json") + views.write_views(path, roots_views()) + assert list(layers_of(generate.root_system(), path)) == ["first order", "roots"] + empty = layers_of(generate.wheat_shoot(), path) # names the frame lacks: empty blocks + assert list(empty) == ["first order", "roots"] and all(block.GetNumberOfCells() == 0 for block in empty.values()) + with pytest.raises(ValueError, match="encoding3d_views"): + reader.frame_to_vtk(generate.root_system(), views={"encoding3d_views": 2, "configurations": {}}) + assert reader.frame_to_vtk(generate.root_system()).GetNumberOfBlocks() == 1 # no views: as before + + +def test_cells_carry_the_ids_of_the_entities_containing_theirs(): + """What selection by scale reads: each cell the ids of its entity's containing ones, each block its data + structure and scales.""" + from openalea.encoding3d import views + frame = generate.root_system() + blocks = blocks_of(frame) + lines = blocks[("root_system", "SubOrgan segments")] + segments = frame.data_structures[0].entity_set("SubOrgan") + np.testing.assert_array_equal(cell_array(lines, "Axis.id"), segments.columns["complex"]) + assert set(cell_array(lines, "Plant.id")) == {1000.} + fields = lines.GetFieldData() + assert fields.GetAbstractArray(views.DATA_STRUCTURE_FIELD).GetValue(0) == "root_system" + scales = fields.GetAbstractArray(views.SCALES_FIELD) + assert [scales.GetValue(number) for number in range(scales.GetNumberOfValues())] == ["Plant", "Axis", "SubOrgan"] + mesh = blocks[("root_system", "mesh roots")] + arrays = {name: cell_array(mesh, name) for name in ("Plant.id", "Axis.id", "SubOrgan.id")} + chosen = views.picked_entities([("root_system", {"Axis.id": np.array([101.])})], "Axis") + lateral = views.selection_mask("root_system", arrays, chosen, "Axis") + assert 0 < lateral.sum() < mesh.GetNumberOfCells() + assert set(arrays["SubOrgan.id"][lateral]) == {60., 61.} # the lateral's segments, not its apex + soil = blocks_of(generate.soil_and_roots())[("soil", "grid")] + assert soil.GetFieldData().GetAbstractArray(views.DATA_STRUCTURE_FIELD).GetValue(0) == "soil" + + +def test_select_at_scale_grows_picked_cells_in_every_layer(): + """The "Select at scale" filter's work: the cells picked on a layer (a vtkSelection, as ParaView's) grown to their + axis or plant in every layer, the hidden one too, the same blocks, polygonal data kept polygonal; the frame found + upstream of items drawn from the reader.""" + from vtkmodules.vtkCommonCore import vtkIdTypeArray + from vtkmodules.vtkCommonDataModel import vtkPolyData, vtkSelection, vtkSelectionNode + from vtkmodules.vtkCommonExecutionModel import vtkTrivialProducer + from vtkmodules.vtkFiltersCore import vtkPassThrough + from test_views import roots_views + frame = generate.root_system() + data = reader.frame_to_vtk(frame, views=roots_views()) # layers "first order", "roots" (hidden) + lines = data.GetBlock(1) + cell = int(np.flatnonzero(cell_array(lines, "Axis.id") == 101.)[0]) + node = vtkSelectionNode() + node.SetFieldType(vtkSelectionNode.CELL) + node.SetContentType(vtkSelectionNode.INDICES) + ids = vtkIdTypeArray() + ids.InsertNextValue(cell) + node.SetSelectionList(ids) + node.GetProperties().Set(vtkSelectionNode.COMPOSITE_INDEX(), 2) # the roots block (flat index 2) + selection = vtkSelection() + selection.AddNode(node) + picked = reader.picked_cells(data, selection) + assert [data_structure for data_structure, _ in picked] == ["root_system"] + assert set(picked[0][1]["Axis.id"]) == {101.} + assert reader.scales_of(data) == ["Plant", "Axis", "SubOrgan"] + grown = reader.grow_selection(data, picked, "Axis") + assert [grown.GetMetaData(number).Get(grown.NAME()) for number in range(2)] == ["first order", "roots"] + axis_lines = grown.GetBlock(1) + assert isinstance(axis_lines, vtkPolyData) and set(cell_array(axis_lines, "Axis.id")) == {101.} + assert axis_lines.GetNumberOfCells() == (cell_array(lines, "Axis.id") == 101.).sum() + assert axis_lines.GetCellData().GetScalars().GetName() == "SubOrgan.label_code" # the layer's colouring + assert axis_lines.GetNumberOfStrips() == axis_lines.GetNumberOfCells() # tubes still + first_order = grown.GetBlock(0) + assert first_order.GetNumberOfCells() == 0 # the lateral is not of the first order + plant = reader.grow_selection(data, picked, "Plant") + assert plant.GetBlock(0).GetNumberOfCells() == data.GetBlock(0).GetNumberOfCells() + assert plant.GetBlock(1).GetNumberOfCells() == lines.GetNumberOfCells() + with pytest.raises(ValueError, match="Compartment"): + reader.grow_selection(data, picked, "Compartment") + assert reader.picked_cells(data, vtkSelection()) == [] + producer = vtkTrivialProducer() # the reader, an item drawn from it, the filter + producer.SetOutput(data) + item, filter_algorithm = vtkPassThrough(), vtkPassThrough() + item.SetInputConnection(producer.GetOutputPort()) + filter_algorithm.SetInputConnection(item.GetOutputPort()) + filter_algorithm.Update() + assert reader.frame_upstream(filter_algorithm) is data + assert reader.frame_upstream(producer) is None diff --git a/bindings/python/test/test_tools.py b/bindings/python/test/test_tools.py index 6027372..b3d9ffa 100644 --- a/bindings/python/test/test_tools.py +++ b/bindings/python/test/test_tools.py @@ -3,9 +3,9 @@ import pytest import generate -from openalea.fspm_tf import from_bytes, read, to_bytes, ucum, vocabulary, write -from openalea.fspm_tf.cli import main -from openalea.fspm_tf.display import add_display, remove_display +from openalea.encoding3d import from_bytes, read, to_bytes, ucum, vocabulary, write +from openalea.encoding3d.cli import main +from openalea.encoding3d.display import add_display, remove_display @pytest.mark.parametrize("code, other", [("mm3/(s.MPa)", "m3/(s.Pa)"), ("cm4/(hPa.d)", "m4/(s.Pa)"), diff --git a/bindings/python/test/test_validate.py b/bindings/python/test/test_validate.py index d645ed2..6fd19c8 100644 --- a/bindings/python/test/test_validate.py +++ b/bindings/python/test/test_validate.py @@ -5,9 +5,9 @@ import pytest import generate -from openalea.fspm_tf import NO_ID, Variable, to_bytes, write_sequence -from openalea.fspm_tf.glb import pack, unpack -from openalea.fspm_tf.validate import validate_bytes, validate_sequence +from openalea.encoding3d import NO_ID, Variable, to_bytes, write_sequence +from openalea.encoding3d.glb import pack, unpack +from openalea.encoding3d.validate import validate_bytes, validate_sequence def messages(frame) -> str: @@ -109,3 +109,50 @@ def test_index_order(tmp_path): with open(index, "w") as file: json.dump(content, file) assert "frames are in time order" in str(validate_sequence(index)) + + +def organs_of(frame): + return frame.data_structures[0].entity_set("Organ") + + +def test_missing_values_scene_is_valid(): + report = validate_bytes(to_bytes(generate.missing_values())) + assert report.ok and not report.warnings, str(report) + + +def test_rules_of_0_2(): + frame = generate.missing_values() + organs_of(frame).columns["emergence"][0] = "1 March 2026" + assert "1 value(s) not ISO 8601 date" in messages(frame) + frame = generate.missing_values() + organs_of(frame).columns["edge_type"][0] = "unknown" # an unknown link needs a parent + assert "'none' exactly for the entities without a parent" in messages(frame) + frame = generate.missing_values() + organs_of(frame).columns["sibling_order"][0] = -1 + assert "positions among siblings are non-negative" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].relations[1].name = "bearer_Axis_Organ" + assert "two relations have the same name" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].relations[0].columns["relation"][0] = "sideways" + assert "bearer labels ['sideways']" in messages(frame) + frame = generate.missing_values() + organs_of(frame).scale = "Organ" + organs_of(frame).rank = 3 + assert "rank 3: the base scale Organ's position is 5" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].labels["Petal"] = "Corolla" + text = messages(frame) + assert "'Petal' is no entity's label" in text and "'Corolla' is not a label of the vocabulary" in text + frame = generate.missing_values() + frame.data_structures[0].extras = {"MyPlatform": {}} + assert "key 'MyPlatform': the platform's name, in lower case" in messages(frame) + + +def test_boolean_with_missing_values_is_uint8(): + data = to_bytes(generate.missing_values()) + document, binary = unpack(data) + classes = document["extensions"]["EXT_structural_metadata"]["schema"]["classes"] + classes["plant__Organ"]["properties"]["alive"]["noData"] = 2 + assert "a boolean column with missing values is UINT8 (0, 1) with noData 255" in \ + str(validate_bytes(pack(document, binary))) diff --git a/bindings/python/test/test_views.py b/bindings/python/test/test_views.py new file mode 100644 index 0000000..4d9d67b --- /dev/null +++ b/bindings/python/test/test_views.py @@ -0,0 +1,228 @@ +"""Frame views (openalea.encoding3d.views, specification §8.1): the schema, the names checked against a frame, the +selection, and the display options ParaView's plugin draws as blocks and saves back.""" +import copy + +import numpy as np +import pytest + +import generate + +from openalea.encoding3d import views + + +def roots_views() -> dict: + return {"encoding3d_views": 1, "active": "default", "configurations": { + "default": {"layers": [ + {"name": "first order", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "select": {"entity_set": "Axis", "where": {"branching_order": [1]}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}}, + {"name": "roots", "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "label"}, "tube": {"radius": "radius", "scale": 2}, + "visible": False}]}, + "plain": {"layers": [ + {"name": "all", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "color_by": None, "solid_color": [0.5, 0.4, 0.3], "opacity": 0.3}]}}} + + +def test_a_views_document_follows_the_schema(): + assert views.check_views(roots_views()).ok + wrong = roots_views() + del wrong["configurations"]["default"]["layers"][0]["source"]["mesh"] # a mesh layer names its mesh + wrong["configurations"]["plain"]["layers"][0]["range"] = [0.] + report = views.check_views(wrong) + assert len(report.errors) == 2, str(report) + + +def test_the_active_configuration_exists_and_layer_names_are_unique(): + document = roots_views() + document["active"] = "missing" + plain = document["configurations"]["plain"]["layers"] + plain.append(copy.deepcopy(plain[0])) + report = views.check_views(document) + assert {issue.where for issue in report.errors} == {"views/active", "plain/layers/1 (all)"}, str(report) + + +def test_names_are_checked_against_a_frame(): + frame = generate.root_system() + assert views.check_views(roots_views(), frame).ok + document = roots_views() + layers = document["configurations"]["default"]["layers"] + layers[0]["source"]["mesh"] = "shoots" + layers[1]["select"]["where"]["width"] = ">0" + layers[1]["color_by"]["entity_set"] = "Axis" # segments are coloured by their own entity set + document["configurations"]["plain"]["layers"][0]["source"]["data_structure"] = "soil" + messages = [issue.message for issue in views.check_views(document, frame).errors] + assert len(messages) == 4, messages + assert any("no mesh 'shoots'" in message for message in messages) + assert any("no column 'width'" in message for message in messages) + assert any("'Axis' is not an entity set" in message for message in messages) + assert any("no data structure 'soil'" in message for message in messages) + + +def test_a_grid_and_segment_ends_are_needed(): + frame = generate.soil_and_roots() + layer = {"name": "axes", "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "Axis"}} + grid = {"name": "soil", "source": {"data_structure": "root_system", "kind": "grid"}} + report = views.check_views({"encoding3d_views": 1, "configurations": {"c": {"layers": [layer, grid]}}}, frame) + assert len(report.errors) == 2 and "no segment ends" in report.errors[0].message + assert "has no grid" in report.errors[1].message + + +def test_selection_by_values_labels_and_comparisons(): + segments = generate.root_system().data_structures[0].entity_set("SubOrgan") + count = segments.count + assert views.selected(segments, {}).all() + np.testing.assert_array_equal(views.selected(segments, {"label": ["RootSegment"]}), + segments.columns["label"] == "RootSegment") + np.testing.assert_array_equal(views.selected(segments, {"label": "RootSegment", "length": ">=0.01"}), + (segments.columns["label"] == "RootSegment") & (segments.columns["length"] >= 0.01)) + np.testing.assert_array_equal(views.selected(segments, {"id": [10, 12]}), np.isin(segments.columns["id"], [10, 12])) + segments.columns["rank"] = np.ma.MaskedArray(np.arange(count), mask=[True] + [False] * (count - 1)) + assert not views.selected(segments, {"rank": "!=5"})[0] # a missing value never holds + assert views.selected(segments, {"rank": "!=5"}).sum() == count - 2 + + +def test_configurations_and_their_defaults(tmp_path): + document = roots_views() + assert views.configuration_names(document) == ["default", "plain"] + name, layers = views.configuration(document) + assert name == "default" and layers[0]["opacity"] == 1. and layers[0]["colormap"] == views.DEFAULTS["colormap"] + assert views.configuration(document, "plain")[1][0]["opacity"] == 0.3 + with pytest.raises(KeyError, match="nothing"): + views.configuration(document, "nothing") + path = str(tmp_path / views.VIEWS_FILE) + views.write_views(path, document) + assert views.read_views(path) == document + frame_path = str(tmp_path / "frame_000000.glb") + assert views.views_path(frame_path) == path and views.views_path(frame_path, "other.json") is None + with pytest.raises(ValueError, match="views"): + views.write_views(path, {"encoding3d_views": 2, "configurations": {}}) + + +def test_display_settings_and_back(): + _, layers = views.configuration(roots_views()) + mesh, segments = (views.display_settings(layer) for layer in layers) + assert mesh["selector"] == "/Root/firstorder" and mesh["tube"] is None and mesh["range"] is None + assert mesh["color_array"] == "SubOrgan.water_potential" and mesh["visible"] + assert segments["tube"] == {"radius_array": views.TUBE_RADIUS} and not segments["visible"] + assert views.display_settings(layers[1], {"SubOrgan.label_code"})["color_array"] == "SubOrgan.label_code" + assert views.block_selector("2 roots/p0") == "/Root/_2rootsp0" # as ParaView names blocks + unchanged = views.layer_from_display(layers[0], {}) + assert unchanged == layers[0] + saved = views.layer_from_display(layers[1], {"visible": True, "opacity": 0.5, "representation": "Wireframe", + "color_array": "SubOrgan.label_code", "range": [0., 2.], + "log": True}) + assert saved["color_by"] == {"entity_set": "SubOrgan", "variable": "label"} + assert saved["range"] == [0., 2.] and saved["log"] and saved["visible"] and saved["representation"] == "Wireframe" + assert saved["tube"] == layers[1]["tube"] and saved["select"] == layers[1]["select"] + solid = views.layer_from_display(layers[0], {"color_array": None, "solid_color": [1., 0., 0.], "range": None}) + assert solid["color_by"] is None and solid["solid_color"] == [1., 0., 0.] and solid["range"] == "auto" + assert views.display_settings(solid)["color_array"] is None + document = views.save_configuration(roots_views(), "mine", [saved, solid]) + assert document["active"] == "mine" and views.check_views(document, generate.root_system()).ok + assert views.configuration(document)[1][0]["opacity"] == 0.5 + assert "mine" not in roots_views()["configurations"] # a copy + + +# ── The layers as the blocks of one representation (ParaView's plugin) ── + +def test_layers_become_block_properties(): + _, layers = views.configuration(roots_views()) + layers[0]["opacity"] = 0.4 + displays = views.block_displays(layers, {"SubOrgan.water_potential", "SubOrgan.label_code"}) + assert displays["selectors"] == ["/Root/firstorder"] # "roots" hidden + assert displays["colourings"] == [("/Root/firstorder", "SubOrgan.water_potential"), + ("/Root/roots", "SubOrgan.label_code")] # text: its codes + assert displays["solid_colours"] == [] and displays["opacities"] == [("/Root/firstorder", 0.4)] + assert displays["main_array"] == "SubOrgan.water_potential" + assert displays["scalar_bars"] == ["SubOrgan.water_potential"] and displays["representation"] == "Surface" + assert displays["colour_maps"]["SubOrgan.label_code"] == {"colormap": views.DEFAULTS["colormap"], "range": None, + "log": False} + plain = views.block_displays(views.configuration(roots_views(), "plain")[1]) + assert plain["solid_colours"] == [("/Root/all", [0.5, 0.4, 0.3])] and plain["main_array"] is None + assert plain["opacities"] == [("/Root/all", 0.3)] + + +def test_block_properties_read_back_into_layers(): + """What "Save Views" reads from the representation, after edits in ParaView, saved into the layers.""" + _, layers = views.configuration(roots_views()) + representation = {"BlockSelectors": ["/Root/firstorder", "/Root/roots"], # "roots" shown + "BlockColorArrayNames": ["/Root/roots", "1", "SubOrgan.radius"], + "BlockColors": ["/Root/firstorder", "1", "0", "0"], # "first order" solid red + "BlockOpacities": ["/Root/roots", "0.5"], "ColorArrayName": "SubOrgan.water_potential", + "DiffuseColor": [1., 1., 1.], "Opacity": 1., "Representation": "Surface", + "colour_maps": {"SubOrgan.radius": ([0.001, 0.004], True)}} + first, roots = views.displays_from_blocks(layers, representation) + assert first == {"visible": True, "opacity": 1., "representation": "Surface", "solid_color": [1., 0., 0.], + "color_array": None} + assert roots == {"visible": True, "opacity": 0.5, "representation": "Surface", "color_array": "SubOrgan.radius", + "range": [0.001, 0.004], "log": True} + saved = [views.layer_from_display(layer, display) for layer, display in zip(layers, (first, roots))] + assert saved[0]["color_by"] is None and saved[0]["solid_color"] == [1., 0., 0.] + assert saved[1]["color_by"] == {"entity_set": "SubOrgan", "variable": "radius"} and saved[1]["log"] + assert saved[1]["tube"] == layers[1]["tube"] and saved[1]["visible"] and saved[1]["opacity"] == 0.5 + inherited = views.displays_from_blocks(layers, {"BlockSelectors": ["/Root"], "ColorArrayName": "SubOrgan.id"}) + assert [display["visible"] for display in inherited] == [True, True] # "/Root": every block shown + assert [display["color_array"] for display in inherited] == ["SubOrgan.id"] * 2 # the representation's own + drawn = views.block_displays(layers) # drawn, then read back as drawn + state = {"BlockSelectors": drawn["selectors"], "Opacity": 1., + "BlockColorArrayNames": [value for selector, array in drawn["colourings"] + for value in (selector, 1, array)], + "colour_maps": {array: (None, False) for array in drawn["colour_maps"]}} + unchanged = [views.layer_from_display(layer, display) + for layer, display in zip(layers, views.displays_from_blocks(layers, state))] + assert unchanged == layers + + +# ── Selection by scale ─────────────────────────────────────────────────── + +def test_the_entities_containing_others_at_every_coarser_scale(): + frame = generate.root_system() + source = frame.data_structures[0] + assert views.ranked_sets(source) == ["Plant", "Axis", "SubOrgan"] + segments = source.entity_set("SubOrgan") + rows = np.array([0, segments.count - 1, -1]) # a tap root segment, the lateral's apex, none + found = views.ancestor_ids(source, "SubOrgan", rows) + np.testing.assert_array_equal(found["Axis"], [100, 101, -1]) + np.testing.assert_array_equal(found["Plant"], [1000, 1000, -1]) + assert views.ancestor_ids(source, "Plant", np.array([0])) == {} + assert views.ancestor_ids(generate.soil_and_roots().data_structures[1], "Voxel", np.array([0])) == {} + + +def test_picked_cells_grow_to_their_entity_and_what_it_contains(): + picked = [("plants", {"Axis.id": np.array([101., 101.]), "SubOrgan.id": np.array([60., 61.])}), + ("soil", {"Voxel.id": np.array([3.])})] + assert views.picked_entities(picked, "Axis") == {"plants": [101]} + assert views.picked_entities(picked, "SubOrgan") == {"plants": [60, 61]} + with pytest.raises(ValueError, match="'Compartment'.*Axis"): + views.picked_entities(picked, "Compartment") + with pytest.raises(ValueError, match="pick cells"): + views.picked_entities([], "Axis") + chosen = {"plants": [101]} + sections = {"Axis.id": np.array([100., 101., np.nan]), "Compartment.id": np.array([1., 2., 3.])} + np.testing.assert_array_equal(views.selection_mask("plants", sections, chosen, "Axis"), [False, True, False]) + np.testing.assert_array_equal(views.selection_mask("other", sections, chosen, "Axis"), [False] * 3) + np.testing.assert_array_equal(views.selection_mask("plants", {"Plant.id": np.ones(2)}, chosen, "Axis"), + [False] * 2) # a block without the scale: none of it + assert views.isolated({"roots": True, "shoot": True, "sections": False, "roots selection": True}, + ["roots selection"]) == ["roots", "shoot"] + blocks = [["Plant", "Axis", "SubOrgan"], [], ["Plant", "Axis", "Organ", "SubOrgan", "Compartment"]] + assert views.scale_names(blocks) == ["Plant", "Axis", "Organ", "SubOrgan", "Compartment"] # every block's + + +def test_views_are_checked_without_jsonschema(monkeypatch): + import builtins + original = builtins.__import__ + + def without_jsonschema(name, *arguments, **keywords): + if name == "jsonschema": + raise ImportError(name) + return original(name, *arguments, **keywords) + + monkeypatch.setattr(builtins, "__import__", without_jsonschema) + assert views.check_views(roots_views()).ok + broken = copy.deepcopy(roots_views()) + del broken["configurations"]["plain"]["layers"][0]["source"]["mesh"] + assert "names its mesh" in str(views.check_views(broken)) + assert not views.check_views({"encoding3d_views": 2, "configurations": {}}).ok diff --git a/conda/environment.yml b/conda/environment.yml index 32f55e5..d1bb8f5 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -1,19 +1,14 @@ -name: fspm_tf +name: encoding3d channels: - openalea3 - conda-forge dependencies: - python + - pip + - pandoc # the documentation's notebooks (nbsphinx) - numpy - jsonschema - - pip - pytest - # the documentation (docs/, also built by readthedocs from this file) - - sphinx - - pydata-sphinx-theme - - myst-parser - - sphinx-favicon - - sphinx-copybutton - # the package itself, from the sources (its dependencies are the conda ones above) + # the package from the sources, with the documentation's and the tests' tools (also what readthedocs builds with) - pip: - - -e .. + - -e ..[doc,test] diff --git a/conda/meta.yaml b/conda/meta.yaml index 88511a4..51c9f9f 100644 --- a/conda/meta.yaml +++ b/conda/meta.yaml @@ -27,7 +27,7 @@ build: # --ignore-installed ensure that compiled files (accidentally present in sources or uncleaned locally) will be overwritten script: {{ PYTHON }} -m pip install . --no-deps --ignore-installed --no-build-isolation -vv entry_points: - - fspm-tf = openalea.fspm_tf.cli:main + - encoding3d = openalea.encoding3d.cli:main requirements: build: @@ -57,7 +57,7 @@ test: - conformance/** # the reference files the tests check, and their generator commands: - - fspm-tf --help + - encoding3d --help - cd bindings/python/test - pytest -v diff --git a/conformance/README.md b/conformance/README.md index fa9c75a..cc8bd2a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -12,36 +12,52 @@ numbers, the same bytes on every run; the CI checks that the committed files are | `soil_and_roots` | a root system, a regular soil grid with z pointing down, and their mapping | | `adaptive_soil` | an adaptive grid's leaf cells (`level`, `index`) | | `parameter_set` | a `table` data structure: parameters per plant and in the scalars, references, an infinite value | +| `missing_values` | integer, text, boolean and float columns with missing values; a date column (`format`); `unknown` edge types and a `sibling_order` column; a `mixed` entity set; a labels map; a parent in another axis; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system, ids kept, and their index | +## Coupling sessions + +`coupling/` holds the coupling conformance sessions (specification §9, §11), which `coupling/generate.py` writes: each +session's peer (a sequence the coordinator replays), its subject's declaration and updates, and the expected values of +what the subject reads (`expected/turn_.json`) and of the session's record (`expected/record/`). A codec's +participant runs them with `encoding3d couple-conformance conformance/coupling --participant COMMAND`. The frame tools +(`compare-all`, each codec's conformance run) leave this folder out. + ## The expected values -`x.expected.json` is the frame `x.glb` holds, as plain JSON (written by `openalea.fspm_tf.dump.frame_to_dict`): +`x.expected.json` is the frame `x.glb` holds, as plain JSON (written by `openalea.encoding3d.dump.frame_to_dict`): ```text {"time": 0.0, "time_unit": "d", "step": 0, "data_structures": [ {"id": "...", "kind": "graph", "origin": [[4 numbers] x 4, row by row], "platform": "...", "model": "...", "extras": {...}, (when present) - "entity_sets": [{"name": "SubOrgan", "rank": 6, "count": 10, + "labels": {"Internode": "StemInternode", ...}, (when present) + "entity_sets": [{"name": "SubOrgan", "rank": 6, "scale": "SubOrgan", "count": 10, (scale when present) "columns": {"id": {"dtype": "int64", "values": [...]}, "label": {"dtype": "string", "values": [...]}, (labels and enums as their names) "end": {"dtype": "float64", "values": [[x, y, z], ...]}, ...}, "variables": {"length": {"unit": "m", "role": "state", "standard_name": "length", - "kind": "extensive", "description": "..."}, ...}}], + "kind": "extensive", "description": "...", + "format": "date"}, ...}}], (format when present) "relations": [... like entity sets, with "source_set" and "target_set"], "scalars": {... like an entity set, one row}, "grid": {"kind", "shape", "origin", "spacing", "axes", "periodic", "z_up", "max_level", "cells"}, "meshes": [{"name": "...", "positions_float32": [[x, y, z], ...], "feature_rows": {"SubOrgan": [...]}, "indices": [...], "display": {"name": [...]}}]}], - "mappings": [... like entity sets, with "source", "target" ([data structure, entity set]) and "method"]} + "mappings": [... like entity sets, with "source", "target" ([data structure, entity set]) and "method"], + "update": {"participant": "...", (a partial frame only, §11.1) + "removed": [{"data_structure": "...", "entity_set": "...", "ids": [...]}, ...]}} (sorted) ``` - **Floats** are the shortest decimals giving the same double, compared exactly. -- **NaN** is `null`; +∞ and −∞ are the strings `"Infinity"` and `"-Infinity"`. +- **NaN and missing values** are `null`, whatever the column's type; +∞ and −∞ are the strings `"Infinity"` and + `"-Infinity"`. - **Positions** are the FLOAT32 values of the file, widened to doubles. - **`dtype`** is the column's numpy type name (`float64`, `int64`, `uint8`, `bool`...), or `string` for text and enums. -- Keys only appear when the frame has them (no `extras`, `relations`, `scalars`, `grid` or `meshes` when absent). + A boolean column with missing values (stored as UINT8) is `bool`. +- Keys only appear when the frame has them (no `extras`, `labels`, `relations`, `scalars`, `grid`, `meshes`, + `scale` or `format` when absent). ## Checking a codec @@ -50,8 +66,8 @@ numbers, the same bytes on every run; the CI checks that the committed files are 3. Then: ```bash - fspm-tf compare-all DUMPS conformance # reading: every dump equals its expected values - fspm-tf validate WRITTEN/*.glb # writing: valid fspm-TF (and Khronos' glTF-Validator: valid glTF) + encoding3d compare-all DUMPS conformance # reading: every dump equals its expected values + encoding3d validate WRITTEN/*.glb # writing: valid encoding3d (and Khronos' glTF-Validator: valid glTF) ``` In GitHub Actions, the composite action `.github/actions/conformance` runs both. The Python codec's own run is diff --git a/conformance/adaptive_soil.expected.json b/conformance/adaptive_soil.expected.json index 5e17210..bc130d3 100644 --- a/conformance/adaptive_soil.expected.json +++ b/conformance/adaptive_soil.expected.json @@ -141,7 +141,7 @@ "rank": null } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy adaptive soil", "grid": { "kind": "adaptive", diff --git a/conformance/adaptive_soil.glb b/conformance/adaptive_soil.glb index 80dd7a0..4f60f3e 100644 Binary files a/conformance/adaptive_soil.glb and b/conformance/adaptive_soil.glb differ diff --git a/conformance/coupling/generate.py b/conformance/coupling/generate.py new file mode 100644 index 0000000..783925d --- /dev/null +++ b/conformance/coupling/generate.py @@ -0,0 +1,217 @@ +""" +Writes the coupling conformance sessions (specification §9, §11): each session's peer (a recorded sequence), its +subject's declaration and updates, and the expected dumps of what the subject reads and of the session's record +(expected/turn_.json, expected/record/frame_.json: not *.expected.json, which are the frames'). + +The subject's updates and expected values come from running the session once with a reference subject (the +behaviour each session describes, in Python), over a folder. Every run writes the same bytes. + + python conformance/coupling/generate.py +""" +import json +import os +import shutil +import sys +import tempfile +import threading + +import numpy as np + +from openalea.encoding3d import (DataStructure, EntitySet, Frame, Mapping, NO_ID, Variable, read, read_index, to_bytes, + write_sequence) +from openalea.encoding3d.coupling import (Coordinator, Declaration, EntitySetSpec, Link, MappingSpec, Participant, + VariableSpec) +from openalea.encoding3d.dump import frame_to_dict + +DIRECTORY = os.path.dirname(os.path.abspath(__file__)) +STEPS = 4 +OFFSET = 2 ** 32 # the subject's id offset: the first participant expected +PEER_OFFSET = 2 * 2 ** 32 # the peer's: the replay, after it + + +def _json(value) -> str: + return json.dumps(value, indent=1, ensure_ascii=False) + "\n" + + +# ── two_scales: the subject refines the peer's organs into segments ────────────────────────────────────── +def two_scales_peer() -> list: + """The grower: one organ more each step, in a chain, each 2 cm longer than the previous; the first removed at + step 3.""" + frames, organs, next_id = [], [], 0 + for step in range(STEPS + 1): + organs.append(next_id) + next_id += 1 + if step == 3: + organs.pop(0) + count = len(organs) + organ_set = EntitySet("Organ", rank=5, columns={ + "id": np.array(organs, dtype=np.int64), "label": np.array(["Leaf"] * count, dtype=object), + "parent": np.array([NO_ID] + organs[:-1], dtype=np.int64), + "edge_type": np.array(["none"] + ["successor"] * (count - 1), dtype=object), + "length": 2. * np.arange(1, count + 1)}, + variables={"length": Variable(unit="cm", standard_name="length", kind="extensive")}) + frames.append(Frame([DataStructure("plant", entity_sets=[organ_set])], time=float(step), step=step, + writer={"platform": "conformance"})) + return frames + + +def two_scales_declaration() -> Declaration: + return Declaration("subject", "conformance", [EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights="refines")], [ + VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol", kind="extensive")]) + + +def two_scales_subject(): + """Two segments under each new organ; each segment's sugar 1e-6 mol per m of its length.""" + segments = {} + + def answer(turn, offset): + plant = turn.frame.data_structure("plant") + for segment in turn.frame.update.removed.get(("plant", "SubOrgan"), np.zeros(0)).tolist(): + segments.pop(segment, None) + lengths = {} + if any(item.name == "SubOrgan" for item in plant.entity_sets): + known = plant.entity_set("SubOrgan") + lengths = dict(zip(known.columns["id"].tolist(), np.asarray(known.columns["segment_length"]).tolist())) + with_segments = {organ for organ, _ in segments.values()} + for organ in plant.entity_set("Organ").columns["id"].tolist(): + if organ not in with_segments: + first = offset + 1000 * turn.step + len(segments) + segments[first], segments[first + 1] = (organ, NO_ID), (organ, first) + ids = list(segments) + columns = {"id": np.array(ids, dtype=np.int64), + "complex": np.array([segments[segment][0] for segment in ids], dtype=np.int64), + "parent": np.array([segments[segment][1] for segment in ids], dtype=np.int64), + "edge_type": np.array(["none" if segments[segment][1] == NO_ID else "successor" + for segment in ids], dtype=object), + "label": np.array(["Segment"] * len(ids), dtype=object), + "sugar": np.array([1e-6 * np.nan_to_num(lengths.get(segment, 0.)) for segment in ids])} + return Frame([DataStructure("plant", entity_sets=[EntitySet("SubOrgan", rank=6, columns=columns)])]) + return answer + + +# ── two_data_structures: the subject's soil linked to the peer's plant by the mapping it provides ──────────── +def two_data_structures_peer() -> list: + """The plant: two organs, then three from step 2, each taking up 1 mol per step.""" + frames = [] + for step in range(STEPS + 1): + count = 2 if step < 2 else 3 + organ_set = EntitySet("Organ", rank=5, columns={"id": np.arange(count, dtype=np.int64), + "uptake": np.ones(count)}, + variables={"uptake": Variable(unit="mol", kind="extensive")}) + frames.append(Frame([DataStructure("plant", entity_sets=[organ_set])], time=float(step), step=step, + writer={"platform": "conformance"})) + return frames + + +def two_data_structures_declaration() -> Declaration: + return Declaration("subject", "conformance", [EntitySetSpec("soil", "Cell", rank=1, rights="owns"), + EntitySetSpec("plant", "Organ")], [ + VariableSpec("soil", "Cell", "water_content", "provides", "m3/m3", kind="intensive"), + VariableSpec("soil", "Cell", "uptake", "requires", "mol", kind="extensive")], + mappings=[MappingSpec("organs_in_soil", ("plant", "Organ"), ("soil", "Cell"))]) + + +def two_data_structures_subject(): + """Three cells; organ i in cells i % 3 (weight 1) and (i + 1) % 3 (weight 3).""" + water = np.array([0.1, 0.2, 0.3]) + + def answer(turn, offset): + cells = offset + np.arange(3) + organs = [] + if any(item.id == "plant" for item in turn.frame.data_structures): + organs = turn.frame.data_structure("plant").entity_set("Organ").columns["id"].tolist() + frame = Frame([DataStructure("soil", kind="table", entity_sets=[ + EntitySet("Cell", rank=1, columns={"id": cells, "water_content": water})])]) + frame.mappings.append(Mapping("organs_in_soil", source=("plant", "Organ"), target=("soil", "Cell"), columns={ + "source": np.repeat(np.array(organs, dtype=np.int64), 2), + "target": np.array([cells[(i + k) % 3] for i in range(len(organs)) for k in (0, 1)], dtype=np.int64), + "weight": np.tile([1., 3.], len(organs))})) + return frame + return answer + + +SESSIONS = { + "two_scales": dict( + peer=two_scales_peer, declaration=two_scales_declaration, subject=two_scales_subject, + links=[Link("subject", "segment_length", "peer", "length", factor=0.5, aggregation="copy")], + covers="a subject refining a replayed peer's organs into segments: new entities, removals cascading to the " + "subject's segments, a unit conversion and a factor, a copy to finer entities"), + "two_data_structures": dict( + peer=two_data_structures_peer, declaration=two_data_structures_declaration, + subject=two_data_structures_subject, + links=[Link("subject", "uptake", "peer", "uptake", mapping="organs_in_soil")], + covers="a subject owning another data structure, linked to the peer's by the mapping it provides: a sum " + "through a mapping, the subject's mappings in its updates"), +} + + +def generate(name: str, spec: dict, directory: str = DIRECTORY) -> None: + folder = os.path.join(directory, name) + for part in ("peer", "subject", "expected"): + if os.path.isdir(os.path.join(folder, part)): + shutil.rmtree(os.path.join(folder, part)) + os.makedirs(os.path.join(folder, "subject")) + os.makedirs(os.path.join(folder, "expected", "record")) + write_sequence(os.path.join(folder, "peer"), spec["peer"]()) + declaration = spec["declaration"]() + with open(os.path.join(folder, "subject", "declaration.json"), "w", encoding="utf-8") as file: + file.write(_json(declaration.to_json())) + settings = {"steps": STEPS, "interval": 1., "timeUnit": "d", "links": [link.to_json() for link in spec["links"]], + "covers": spec["covers"]} + with open(os.path.join(folder, "session.json"), "w", encoding="utf-8") as file: + file.write(_json(settings)) + + answer = spec["subject"]() + + def reference(endpoint): + participant = Participant.connect(spec["declaration"](), endpoint, timeout=60) + for turn in participant.turns(): + with open(os.path.join(folder, "expected", f"turn_{turn.step:04d}.json"), "w", + encoding="utf-8") as file: + file.write(_json(frame_to_dict(turn.frame))) + update = answer(turn, participant.id_offset) + update.update = None + with open(os.path.join(folder, "subject", f"update_{turn.step:04d}.glb"), "wb") as file: + file.write(to_bytes(update)) + participant.send(update) + + with tempfile.TemporaryDirectory() as temporary: + coordinator = Coordinator("files:" + os.path.join(temporary, "messages"), steps=STEPS, interval=1., + expect=["subject"], links=spec["links"], timeout=60, + replays={"peer": os.path.join(folder, "peer", "scene.fspm.json")}, + record=os.path.join(temporary, "record")) + thread = threading.Thread(target=reference, args=(coordinator.endpoint,), daemon=True) + thread.start() + coordinator.run() + thread.join(60) + for entry in read_index(os.path.join(temporary, "record", "scene.fspm.json"))["frames"]: + path = os.path.join(folder, "expected", "record", entry["file"].replace(".glb", ".json")) + with open(path, "w", encoding="utf-8") as file: + file.write(_json(frame_to_dict(read(entry["path"])))) + assert participant_offset_ok(folder) + + +def participant_offset_ok(folder: str) -> bool: + """The subject's ids in its updates are in its range (the first participant expected's).""" + for name in sorted(os.listdir(os.path.join(folder, "subject"))): + if name.endswith(".glb"): + frame = read(os.path.join(folder, "subject", name)) + for data_structure in frame.data_structures: + for entity_set in data_structure.entity_sets: + ids = entity_set.columns["id"] + if ids.size and not ((ids >= OFFSET) & (ids < OFFSET + 2 ** 32)).all(): + return False + return True + + +def main() -> int: + for name, spec in SESSIONS.items(): + generate(name, spec) + print(name) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00000.json b/conformance/coupling/two_data_structures/expected/record/frame_00000.json new file mode 100644 index 0000000..5a3d304 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00000.json @@ -0,0 +1,219 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 0, + "columns": { + "source": { + "dtype": "int64", + "values": [] + }, + "target": { + "dtype": "int64", + "values": [] + }, + "weight": { + "dtype": "float64", + "values": [] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00001.json b/conformance/coupling/two_data_structures/expected/record/frame_00001.json new file mode 100644 index 0000000..7b0e0a1 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00001.json @@ -0,0 +1,234 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 4, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00002.json b/conformance/coupling/two_data_structures/expected/record/frame_00002.json new file mode 100644 index 0000000..013a26c --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00002.json @@ -0,0 +1,240 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 4, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00003.json b/conformance/coupling/two_data_structures/expected/record/frame_00003.json new file mode 100644 index 0000000..410ddb7 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00003.json @@ -0,0 +1,246 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 6, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298, + 4294967298, + 4294967296 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00004.json b/conformance/coupling/two_data_structures/expected/record/frame_00004.json new file mode 100644 index 0000000..9dc2df3 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00004.json @@ -0,0 +1,246 @@ +{ + "time": 4.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 6, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298, + 4294967298, + 4294967296 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0000.json b/conformance/coupling/two_data_structures/expected/turn_0000.json new file mode 100644 index 0000000..e3d558a --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0000.json @@ -0,0 +1,11 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0001.json b/conformance/coupling/two_data_structures/expected/turn_0001.json new file mode 100644 index 0000000..9182413 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0001.json @@ -0,0 +1,181 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + null, + null, + null + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0002.json b/conformance/coupling/two_data_structures/expected/turn_0002.json new file mode 100644 index 0000000..03e5f83 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0002.json @@ -0,0 +1,181 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 0.25, + 1.0, + 0.75 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0003.json b/conformance/coupling/two_data_structures/expected/turn_0003.json new file mode 100644 index 0000000..32fa5f4 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0003.json @@ -0,0 +1,186 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 0.25, + 1.0, + 0.75 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0004.json b/conformance/coupling/two_data_structures/expected/turn_0004.json new file mode 100644 index 0000000..d87bf54 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0004.json @@ -0,0 +1,186 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/peer/frame_000000.glb b/conformance/coupling/two_data_structures/peer/frame_000000.glb new file mode 100644 index 0000000..6979655 Binary files /dev/null and b/conformance/coupling/two_data_structures/peer/frame_000000.glb differ diff --git a/conformance/coupling/two_data_structures/peer/frame_000001.glb b/conformance/coupling/two_data_structures/peer/frame_000001.glb new file mode 100644 index 0000000..ccdad2f Binary files /dev/null and b/conformance/coupling/two_data_structures/peer/frame_000001.glb differ diff --git a/conformance/coupling/two_data_structures/peer/frame_000002.glb b/conformance/coupling/two_data_structures/peer/frame_000002.glb new file mode 100644 index 0000000..11c3393 Binary files /dev/null and b/conformance/coupling/two_data_structures/peer/frame_000002.glb differ diff --git a/conformance/coupling/two_data_structures/peer/frame_000003.glb b/conformance/coupling/two_data_structures/peer/frame_000003.glb new file mode 100644 index 0000000..580ef84 Binary files /dev/null and b/conformance/coupling/two_data_structures/peer/frame_000003.glb differ diff --git a/conformance/coupling/two_data_structures/peer/frame_000004.glb b/conformance/coupling/two_data_structures/peer/frame_000004.glb new file mode 100644 index 0000000..f05e317 Binary files /dev/null and b/conformance/coupling/two_data_structures/peer/frame_000004.glb differ diff --git a/conformance/coupling/two_data_structures/peer/scene.fspm.json b/conformance/coupling/two_data_structures/peer/scene.fspm.json new file mode 100644 index 0000000..ee7ae02 --- /dev/null +++ b/conformance/coupling/two_data_structures/peer/scene.fspm.json @@ -0,0 +1,31 @@ +{ + "fspmIndex": "0.3", + "timeUnit": "d", + "frames": [ + { + "time": 0.0, + "step": 0, + "file": "frame_000000.glb" + }, + { + "time": 1.0, + "step": 1, + "file": "frame_000001.glb" + }, + { + "time": 2.0, + "step": 2, + "file": "frame_000002.glb" + }, + { + "time": 3.0, + "step": 3, + "file": "frame_000003.glb" + }, + { + "time": 4.0, + "step": 4, + "file": "frame_000004.glb" + } + ] +} diff --git a/conformance/coupling/two_data_structures/session.json b/conformance/coupling/two_data_structures/session.json new file mode 100644 index 0000000..c70b963 --- /dev/null +++ b/conformance/coupling/two_data_structures/session.json @@ -0,0 +1,15 @@ +{ + "steps": 4, + "interval": 1.0, + "timeUnit": "d", + "links": [ + { + "receiver": "subject", + "variable": "uptake", + "provider": "peer", + "source": "uptake", + "mapping": "organs_in_soil" + } + ], + "covers": "a subject owning another data structure, linked to the peer's by the mapping it provides: a sum through a mapping, the subject's mappings in its updates" +} diff --git a/conformance/coupling/two_data_structures/subject/declaration.json b/conformance/coupling/two_data_structures/subject/declaration.json new file mode 100644 index 0000000..9e2d768 --- /dev/null +++ b/conformance/coupling/two_data_structures/subject/declaration.json @@ -0,0 +1,49 @@ +{ + "participant": "subject", + "entitySets": [ + { + "dataStructure": "soil", + "name": "Cell", + "rank": 1, + "rights": "owns" + }, + { + "dataStructure": "plant", + "name": "Organ", + "rank": 5, + "rights": "follows" + } + ], + "variables": [ + { + "dataStructure": "soil", + "entitySet": "Cell", + "name": "water_content", + "direction": "provides", + "unit": "m3/m3", + "kind": "intensive" + }, + { + "dataStructure": "soil", + "entitySet": "Cell", + "name": "uptake", + "direction": "requires", + "unit": "mol", + "kind": "extensive" + } + ], + "platform": "conformance", + "mappings": [ + { + "name": "organs_in_soil", + "source": { + "dataStructure": "plant", + "entitySet": "Organ" + }, + "target": { + "dataStructure": "soil", + "entitySet": "Cell" + } + } + ] +} diff --git a/conformance/coupling/two_data_structures/subject/update_0000.glb b/conformance/coupling/two_data_structures/subject/update_0000.glb new file mode 100644 index 0000000..59f2551 Binary files /dev/null and b/conformance/coupling/two_data_structures/subject/update_0000.glb differ diff --git a/conformance/coupling/two_data_structures/subject/update_0001.glb b/conformance/coupling/two_data_structures/subject/update_0001.glb new file mode 100644 index 0000000..64ac050 Binary files /dev/null and b/conformance/coupling/two_data_structures/subject/update_0001.glb differ diff --git a/conformance/coupling/two_data_structures/subject/update_0002.glb b/conformance/coupling/two_data_structures/subject/update_0002.glb new file mode 100644 index 0000000..64ac050 Binary files /dev/null and b/conformance/coupling/two_data_structures/subject/update_0002.glb differ diff --git a/conformance/coupling/two_data_structures/subject/update_0003.glb b/conformance/coupling/two_data_structures/subject/update_0003.glb new file mode 100644 index 0000000..72ab8e4 Binary files /dev/null and b/conformance/coupling/two_data_structures/subject/update_0003.glb differ diff --git a/conformance/coupling/two_data_structures/subject/update_0004.glb b/conformance/coupling/two_data_structures/subject/update_0004.glb new file mode 100644 index 0000000..72ab8e4 Binary files /dev/null and b/conformance/coupling/two_data_structures/subject/update_0004.glb differ diff --git a/conformance/coupling/two_scales/expected/record/frame_00000.json b/conformance/coupling/two_scales/expected/record/frame_00000.json new file mode 100644 index 0000000..502af8f --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00000.json @@ -0,0 +1,149 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 1, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00001.json b/conformance/coupling/two_scales/expected/record/frame_00001.json new file mode 100644 index 0000000..dcbe0cf --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00001.json @@ -0,0 +1,167 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00002.json b/conformance/coupling/two_scales/expected/record/frame_00002.json new file mode 100644 index 0000000..fe1c345 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00002.json @@ -0,0 +1,185 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592, + 8589934593 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299, + 4294969300, + 4294969301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298, + -1, + 4294969300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00003.json b/conformance/coupling/two_scales/expected/record/frame_00003.json new file mode 100644 index 0000000..b39e4f5 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00003.json @@ -0,0 +1,185 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00004.json b/conformance/coupling/two_scales/expected/record/frame_00004.json new file mode 100644 index 0000000..5a8eda5 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00004.json @@ -0,0 +1,203 @@ +{ + "time": 4.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595, + 8589934596 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594, + 8589934595 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0, + 8.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 8, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301, + 4294971302, + 4294971303 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595, + 8589934596, + 8589934596 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300, + -1, + 4294971302 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 3e-08, + 3e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/turn_0000.json b/conformance/coupling/two_scales/expected/turn_0000.json new file mode 100644 index 0000000..93154bf --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0000.json @@ -0,0 +1,82 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 1, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0001.json b/conformance/coupling/two_scales/expected/turn_0001.json new file mode 100644 index 0000000..d92b526 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0001.json @@ -0,0 +1,144 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0002.json b/conformance/coupling/two_scales/expected/turn_0002.json new file mode 100644 index 0000000..e0f05f1 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0002.json @@ -0,0 +1,161 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592, + 8589934593 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0003.json b/conformance/coupling/two_scales/expected/turn_0003.json new file mode 100644 index 0000000..7c0cd9b --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0003.json @@ -0,0 +1,177 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [ + { + "data_structure": "plant", + "entity_set": "Organ", + "ids": [ + 8589934592 + ] + }, + { + "data_structure": "plant", + "entity_set": "SubOrgan", + "ids": [ + 4294967296, + 4294967297 + ] + } + ] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0004.json b/conformance/coupling/two_scales/expected/turn_0004.json new file mode 100644 index 0000000..19d69d1 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0004.json @@ -0,0 +1,178 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595, + 8589934596 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594, + 8589934595 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02, + 0.03, + 0.03 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/peer/frame_000000.glb b/conformance/coupling/two_scales/peer/frame_000000.glb new file mode 100644 index 0000000..a7691c0 Binary files /dev/null and b/conformance/coupling/two_scales/peer/frame_000000.glb differ diff --git a/conformance/coupling/two_scales/peer/frame_000001.glb b/conformance/coupling/two_scales/peer/frame_000001.glb new file mode 100644 index 0000000..26cc0b3 Binary files /dev/null and b/conformance/coupling/two_scales/peer/frame_000001.glb differ diff --git a/conformance/coupling/two_scales/peer/frame_000002.glb b/conformance/coupling/two_scales/peer/frame_000002.glb new file mode 100644 index 0000000..a39c12f Binary files /dev/null and b/conformance/coupling/two_scales/peer/frame_000002.glb differ diff --git a/conformance/coupling/two_scales/peer/frame_000003.glb b/conformance/coupling/two_scales/peer/frame_000003.glb new file mode 100644 index 0000000..daa99f8 Binary files /dev/null and b/conformance/coupling/two_scales/peer/frame_000003.glb differ diff --git a/conformance/coupling/two_scales/peer/frame_000004.glb b/conformance/coupling/two_scales/peer/frame_000004.glb new file mode 100644 index 0000000..8099951 Binary files /dev/null and b/conformance/coupling/two_scales/peer/frame_000004.glb differ diff --git a/conformance/coupling/two_scales/peer/scene.fspm.json b/conformance/coupling/two_scales/peer/scene.fspm.json new file mode 100644 index 0000000..ee7ae02 --- /dev/null +++ b/conformance/coupling/two_scales/peer/scene.fspm.json @@ -0,0 +1,31 @@ +{ + "fspmIndex": "0.3", + "timeUnit": "d", + "frames": [ + { + "time": 0.0, + "step": 0, + "file": "frame_000000.glb" + }, + { + "time": 1.0, + "step": 1, + "file": "frame_000001.glb" + }, + { + "time": 2.0, + "step": 2, + "file": "frame_000002.glb" + }, + { + "time": 3.0, + "step": 3, + "file": "frame_000003.glb" + }, + { + "time": 4.0, + "step": 4, + "file": "frame_000004.glb" + } + ] +} diff --git a/conformance/coupling/two_scales/session.json b/conformance/coupling/two_scales/session.json new file mode 100644 index 0000000..193f2ce --- /dev/null +++ b/conformance/coupling/two_scales/session.json @@ -0,0 +1,16 @@ +{ + "steps": 4, + "interval": 1.0, + "timeUnit": "d", + "links": [ + { + "receiver": "subject", + "variable": "segment_length", + "provider": "peer", + "source": "length", + "aggregation": "copy", + "factor": 0.5 + } + ], + "covers": "a subject refining a replayed peer's organs into segments: new entities, removals cascading to the subject's segments, a unit conversion and a factor, a copy to finer entities" +} diff --git a/conformance/coupling/two_scales/subject/declaration.json b/conformance/coupling/two_scales/subject/declaration.json new file mode 100644 index 0000000..4a1ba1c --- /dev/null +++ b/conformance/coupling/two_scales/subject/declaration.json @@ -0,0 +1,36 @@ +{ + "participant": "subject", + "entitySets": [ + { + "dataStructure": "plant", + "name": "Organ", + "rank": 5, + "rights": "follows" + }, + { + "dataStructure": "plant", + "name": "SubOrgan", + "rank": 6, + "rights": "refines" + } + ], + "variables": [ + { + "dataStructure": "plant", + "entitySet": "SubOrgan", + "name": "segment_length", + "direction": "requires", + "unit": "m", + "standardName": "length" + }, + { + "dataStructure": "plant", + "entitySet": "SubOrgan", + "name": "sugar", + "direction": "provides", + "unit": "mol", + "kind": "extensive" + } + ], + "platform": "conformance" +} diff --git a/conformance/coupling/two_scales/subject/update_0000.glb b/conformance/coupling/two_scales/subject/update_0000.glb new file mode 100644 index 0000000..273a3c8 Binary files /dev/null and b/conformance/coupling/two_scales/subject/update_0000.glb differ diff --git a/conformance/coupling/two_scales/subject/update_0001.glb b/conformance/coupling/two_scales/subject/update_0001.glb new file mode 100644 index 0000000..943b4ea Binary files /dev/null and b/conformance/coupling/two_scales/subject/update_0001.glb differ diff --git a/conformance/coupling/two_scales/subject/update_0002.glb b/conformance/coupling/two_scales/subject/update_0002.glb new file mode 100644 index 0000000..ebe68b0 Binary files /dev/null and b/conformance/coupling/two_scales/subject/update_0002.glb differ diff --git a/conformance/coupling/two_scales/subject/update_0003.glb b/conformance/coupling/two_scales/subject/update_0003.glb new file mode 100644 index 0000000..7d6169d Binary files /dev/null and b/conformance/coupling/two_scales/subject/update_0003.glb differ diff --git a/conformance/coupling/two_scales/subject/update_0004.glb b/conformance/coupling/two_scales/subject/update_0004.glb new file mode 100644 index 0000000..4a7b60f Binary files /dev/null and b/conformance/coupling/two_scales/subject/update_0004.glb differ diff --git a/conformance/generate.py b/conformance/generate.py index 1759799..bb33089 100644 --- a/conformance/generate.py +++ b/conformance/generate.py @@ -1,6 +1,6 @@ """ -The conformance files of fspm-TF: small reference scenes covering the specification, each written as a GLB with its -expected values as JSON (openalea.fspm_tf.dump). A reader in any language passes when what it reads from each GLB equals the +The conformance files of encoding3d: small reference scenes covering the specification, each written as a GLB with its +expected values as JSON (openalea.encoding3d.dump). A reader in any language passes when what it reads from each GLB equals the JSON; a writer passes when the validator accepts what it writes for the same scenes. python conformance/generate.py (re)writes the files next to this script @@ -14,12 +14,12 @@ import numpy as np -from openalea.fspm_tf import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, Table, Variable, +from openalea.encoding3d import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, Table, Variable, to_bytes, write_index) -from openalea.fspm_tf.dump import frame_to_dict +from openalea.encoding3d.dump import frame_to_dict DIRECTORY = os.path.dirname(os.path.abspath(__file__)) -WRITER = {"platform": "fspm-tf", "model": "conformance"} +WRITER = {"platform": "encoding3d", "model": "conformance"} def _segment_triangles(start, end, radius, sides: int = 6): @@ -108,8 +108,8 @@ def root_system(n_segments: int = 6, lateral_from: int = 2, ids_from: int = 10, data_structure = DataStructure(plant_id, kind="graph", entity_sets=[plants, axes, segments], meshes=[mesh], scalars=Table("scalars", columns=dict(time_step=[3600.]), variables=_declare(time_step=dict(unit="s", role="setting"))), - platform="fspm-tf", model="toy root system") - return Frame([data_structure], time=time, time_unit="d", step=step, writer=WRITER, vocabulary_version="0.1") + platform="encoding3d", model="toy root system") + return Frame([data_structure], time=time, time_unit="d", step=step, writer=WRITER, vocabulary_version="0.2") def wheat_shoot() -> Frame: @@ -152,10 +152,10 @@ def wheat_shoot() -> Frame: organ_rows = np.full(rows.size, 2) mesh = Mesh("leaves", positions=quads, feature_rows={"SubOrgan": rows, "Organ": organ_rows}) data_structure = DataStructure("shoot", entity_sets=[plants, axes, organs, elements], meshes=[mesh], - platform="fspm-tf", model="toy shoot", + platform="encoding3d", model="toy shoot", origin=np.array([[1., 0., 0., 0.5], [0., 1., 0., 0.25], [0., 0., 1., 0.], [0., 0., 0., 1.]])) - return Frame([data_structure], time=12.5, step=300, writer=WRITER, vocabulary_version="0.1") + return Frame([data_structure], time=12.5, step=300, writer=WRITER, vocabulary_version="0.2") def section_cells() -> Frame: @@ -187,9 +187,9 @@ def section_cells() -> Frame: rows += [row] * 6 mesh = Mesh("cells", positions=squares, feature_rows={"Compartment": rows}) data_structure = DataStructure("section", entity_sets=[cells], relations=[connections], meshes=[mesh], - platform="fspm-tf", model="toy cell network", - extras={"fspm-tf": {"note": "extras are the writer's, kept by readers"}}) - return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + platform="encoding3d", model="toy cell network", + extras={"encoding3d": {"note": "extras are the writer's, kept by readers"}}) + return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.2") def soil_and_roots() -> Frame: @@ -210,7 +210,7 @@ def soil_and_roots() -> Frame: soil = DataStructure("soil", kind="regular_grid", entity_sets=[voxels], grid=Grid(kind="regular", shape=shape, origin=(-0.02, -0.02, 0.), spacing=(0.02, 0.02, 0.02), axes=("x", "y", "z"), periodic=(True, True, False), z_up=False, cells="Voxel"), - platform="fspm-tf", model="toy soil") + platform="encoding3d", model="toy soil") segments = roots.entity_set("SubOrgan") middle_depth = -(segments.columns["z1"] + segments.columns["z2"]) / 2 middle_x = (segments.columns["x1"] + segments.columns["x2"]) / 2 @@ -238,8 +238,8 @@ def adaptive_soil() -> Frame: soil = DataStructure("soil", kind="adaptive_grid", entity_sets=[cells], grid=Grid(kind="adaptive", shape=(2, 1, 1), origin=(0., 0., 0.), spacing=(0.1, 0.1, 0.1), axes=("x", "y", "z"), periodic=(False, False, False), z_up=False, max_level=1, - cells="Cell"), platform="fspm-tf", model="toy adaptive soil") - return Frame([soil], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + cells="Cell"), platform="encoding3d", model="toy adaptive soil") + return Frame([soil], time=0., step=0, writer=WRITER, vocabulary_version="0.2") def parameter_set() -> Frame: @@ -262,8 +262,54 @@ def parameter_set() -> Frame: T_ref=dict(unit="K", role="parameter", kind="intensive"), method=dict(unit="1", role="setting"))) data_structure = DataStructure("archisimple_parameters", kind="table", entity_sets=[plants], scalars=scalars, - platform="fspm-tf", model="RootArchitecture parameters") - return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + platform="encoding3d", model="RootArchitecture parameters") + return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.2") + + +def missing_values() -> Frame: + """A plant as platforms with per-class fields write it (MTG.jl, PlantGraphs, GroIMP): integer, text, boolean and + float columns with missing values, a date column, an edge type the writer does not know, the order of children, + an entity set mixing base scales (a whole root axis among organs), the platform's labels mapped to the + vocabulary's, a branch's first internode whose parent is in another axis, bearer relations between two pairs of + sets.""" + plant = EntitySet("Plant", rank=1, scale="Plant", columns=dict(id=np.array([0], dtype=np.int64), + label=np.array(["Wheat"], dtype=object))) + axes = EntitySet("Axis", rank=2, scale="Axis", columns=dict( + id=np.array([1, 2], dtype=np.int64), label=np.array(["Axis", "Axis"], dtype=object), + complex=np.array([0, 0], dtype=np.int64), parent=np.array([-1, -1], dtype=np.int64), + edge_type=np.array(["none", "none"], dtype=object))) + organs = EntitySet("Organ", rank=5, scale="mixed", columns=dict( + id=np.arange(10, 17, dtype=np.int64), + label=np.array(["Internode", "Internode", "Leaf", "Leaf", "Internode", "Bud", "Root"], dtype=object), + complex=np.array([1, 1, 1, 1, 2, 2, 0], dtype=np.int64), + parent=np.array([-1, 10, 10, 11, 11, 14, -1], dtype=np.int64), + edge_type=np.array(["none", "successor", "branch", "unknown", "branch", "successor", "none"], dtype=object), + sibling_order=np.array([0, 1, 0, 0, 0, 0, 1], dtype=np.int64), + length=np.array([0.12, 0.08, 0.2, 0.15, 0.05, np.nan, 0.3]), + rank=np.ma.MaskedArray(np.array([1, 2, 1, 2, 1, 0, 0], dtype=np.int32), + mask=[False, False, False, False, False, True, True]), + tissue=np.array(["stem", "stem", "blade", None, "stem", None, "root"], dtype=object), + alive=np.ma.MaskedArray(np.array([True, True, True, False, True, False, False]), + mask=[False, False, False, False, False, True, False]), + emergence=np.array(["2026-03-01", "2026-03-04", "2026-03-02", "2026-03-06", None, None, "2026-02-28"], + dtype=object)), + variables=_declare(length=dict(unit="m", standard_name="length", kind="extensive"), + rank=dict(unit="1", kind="descriptor", description="the organ's rank on its axis"), + tissue=dict(unit="1", kind="descriptor"), + alive=dict(unit="1", kind="descriptor"), + emergence=dict(unit="1", kind="descriptor", format="date", + description="the organ's emergence date"))) + bearer_axes = Relation("bearer_Axis_Organ", source_set="Axis", target_set="Organ", columns=dict( + source=np.array([2], dtype=np.int64), target=np.array([11], dtype=np.int64), + relation=np.array(["branch"], dtype=object))) + bearer_buds = Relation("bearer_Organ_Axis", source_set="Organ", target_set="Axis", columns=dict( + source=np.array([15], dtype=np.int64), target=np.array([1], dtype=np.int64), + relation=np.array(["decomposition"], dtype=object))) + data_structure = DataStructure("plant", entity_sets=[plant, axes, organs], relations=[bearer_axes, bearer_buds], + platform="encoding3d", model="toy plant with per-class fields", + labels={"Internode": "StemInternode", "Leaf": "Leaf", "Bud": "Meristem", + "Root": "RootAxis", "Axis": "ShootAxis"}) + return Frame([data_structure], time=3., time_unit="d", step=72, writer=WRITER, vocabulary_version="0.2") def growing_sequence() -> list: @@ -272,7 +318,8 @@ def growing_sequence() -> list: SCENES = {"root_system": root_system, "wheat_shoot": wheat_shoot, "section_cells": section_cells, - "soil_and_roots": soil_and_roots, "adaptive_soil": adaptive_soil, "parameter_set": parameter_set} + "soil_and_roots": soil_and_roots, "adaptive_soil": adaptive_soil, "parameter_set": parameter_set, + "missing_values": missing_values} def files() -> dict: diff --git a/conformance/growing_sequence/frame_000000.expected.json b/conformance/growing_sequence/frame_000000.expected.json index 707f85f..74ec746 100644 --- a/conformance/growing_sequence/frame_000000.expected.json +++ b/conformance/growing_sequence/frame_000000.expected.json @@ -392,7 +392,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy root system", "scalars": { "name": "scalars", diff --git a/conformance/growing_sequence/frame_000000.glb b/conformance/growing_sequence/frame_000000.glb index 7d09190..6cbc787 100644 Binary files a/conformance/growing_sequence/frame_000000.glb and b/conformance/growing_sequence/frame_000000.glb differ diff --git a/conformance/growing_sequence/frame_000001.expected.json b/conformance/growing_sequence/frame_000001.expected.json index 9053f7c..8a69827 100644 --- a/conformance/growing_sequence/frame_000001.expected.json +++ b/conformance/growing_sequence/frame_000001.expected.json @@ -424,7 +424,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy root system", "scalars": { "name": "scalars", diff --git a/conformance/growing_sequence/frame_000001.glb b/conformance/growing_sequence/frame_000001.glb index ecb3a69..3b9371f 100644 Binary files a/conformance/growing_sequence/frame_000001.glb and b/conformance/growing_sequence/frame_000001.glb differ diff --git a/conformance/growing_sequence/frame_000002.expected.json b/conformance/growing_sequence/frame_000002.expected.json index aa2c46b..67695a5 100644 --- a/conformance/growing_sequence/frame_000002.expected.json +++ b/conformance/growing_sequence/frame_000002.expected.json @@ -472,7 +472,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy root system", "scalars": { "name": "scalars", diff --git a/conformance/growing_sequence/frame_000002.glb b/conformance/growing_sequence/frame_000002.glb index 5af5667..2ab9df6 100644 Binary files a/conformance/growing_sequence/frame_000002.glb and b/conformance/growing_sequence/frame_000002.glb differ diff --git a/conformance/growing_sequence/scene.fspm.json b/conformance/growing_sequence/scene.fspm.json index d56186c..a76f338 100644 --- a/conformance/growing_sequence/scene.fspm.json +++ b/conformance/growing_sequence/scene.fspm.json @@ -1,5 +1,5 @@ { - "fspmIndex": "0.1", + "fspmIndex": "0.3", "timeUnit": "d", "frames": [ { diff --git a/conformance/missing_values.expected.json b/conformance/missing_values.expected.json new file mode 100644 index 0000000..0e6b93d --- /dev/null +++ b/conformance/missing_values.expected.json @@ -0,0 +1,337 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 72, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Plant", + "count": 1, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 0 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Wheat" + ] + } + }, + "rank": 1, + "scale": "Plant" + }, + { + "name": "Axis", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 1, + 2 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Axis", + "Axis" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 0, + 0 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + } + }, + "rank": 2, + "scale": "Axis" + }, + { + "name": "Organ", + "count": 7, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 10, + 11, + 12, + 13, + 14, + 15, + 16 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Internode", + "Internode", + "Leaf", + "Leaf", + "Internode", + "Bud", + "Root" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 1, + 1, + 1, + 1, + 2, + 2, + 0 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 10, + 10, + 11, + 11, + 14, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "branch", + "unknown", + "branch", + "successor", + "none" + ] + }, + "sibling_order": { + "dtype": "int64", + "values": [ + 0, + 1, + 0, + 0, + 0, + 0, + 1 + ] + }, + "length": { + "dtype": "float64", + "values": [ + 0.12, + 0.08, + 0.2, + 0.15, + 0.05, + null, + 0.3 + ] + }, + "rank": { + "dtype": "int32", + "values": [ + 1, + 2, + 1, + 2, + 1, + null, + null + ] + }, + "tissue": { + "dtype": "string", + "values": [ + "stem", + "stem", + "blade", + null, + "stem", + null, + "root" + ] + }, + "alive": { + "dtype": "bool", + "values": [ + true, + true, + true, + false, + true, + null, + false + ] + }, + "emergence": { + "dtype": "string", + "values": [ + "2026-03-01", + "2026-03-04", + "2026-03-02", + "2026-03-06", + null, + null, + "2026-02-28" + ] + } + }, + "variables": { + "length": { + "unit": "m", + "role": "state", + "standard_name": "length", + "kind": "extensive" + }, + "rank": { + "unit": "1", + "role": "state", + "kind": "descriptor", + "description": "the organ's rank on its axis" + }, + "tissue": { + "unit": "1", + "role": "state", + "kind": "descriptor" + }, + "alive": { + "unit": "1", + "role": "state", + "kind": "descriptor" + }, + "emergence": { + "unit": "1", + "role": "state", + "kind": "descriptor", + "description": "the organ's emergence date", + "format": "date" + } + }, + "rank": 5, + "scale": "mixed" + } + ], + "platform": "encoding3d", + "model": "toy plant with per-class fields", + "labels": { + "Internode": "StemInternode", + "Leaf": "Leaf", + "Bud": "Meristem", + "Root": "RootAxis", + "Axis": "ShootAxis" + }, + "relations": [ + { + "name": "bearer_Axis_Organ", + "count": 1, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 2 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 11 + ] + }, + "relation": { + "dtype": "string", + "values": [ + "branch" + ] + } + }, + "source_set": "Axis", + "target_set": "Organ" + }, + { + "name": "bearer_Organ_Axis", + "count": 1, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 15 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 1 + ] + }, + "relation": { + "dtype": "string", + "values": [ + "decomposition" + ] + } + }, + "source_set": "Organ", + "target_set": "Axis" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/missing_values.glb b/conformance/missing_values.glb new file mode 100644 index 0000000..37f8bac Binary files /dev/null and b/conformance/missing_values.glb differ diff --git a/conformance/parameter_set.expected.json b/conformance/parameter_set.expected.json index b3e3d21..b1e420e 100644 --- a/conformance/parameter_set.expected.json +++ b/conformance/parameter_set.expected.json @@ -101,7 +101,7 @@ "rank": 1 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "RootArchitecture parameters", "scalars": { "name": "scalars", diff --git a/conformance/parameter_set.glb b/conformance/parameter_set.glb index 8ac363c..0920739 100644 Binary files a/conformance/parameter_set.glb and b/conformance/parameter_set.glb differ diff --git a/conformance/root_system.expected.json b/conformance/root_system.expected.json index c026960..6a2da45 100644 --- a/conformance/root_system.expected.json +++ b/conformance/root_system.expected.json @@ -440,7 +440,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy root system", "scalars": { "name": "scalars", diff --git a/conformance/root_system.glb b/conformance/root_system.glb index 3b9b857..3413759 100644 Binary files a/conformance/root_system.glb and b/conformance/root_system.glb differ diff --git a/conformance/section_cells.expected.json b/conformance/section_cells.expected.json index 63c1aac..2d2fa10 100644 --- a/conformance/section_cells.expected.json +++ b/conformance/section_cells.expected.json @@ -133,10 +133,10 @@ "rank": 9 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy cell network", "extras": { - "fspm-tf": { + "encoding3d": { "note": "extras are the writer's, kept by readers" } }, diff --git a/conformance/section_cells.glb b/conformance/section_cells.glb index 2272cc3..c389032 100644 Binary files a/conformance/section_cells.glb and b/conformance/section_cells.glb differ diff --git a/conformance/soil_and_roots.expected.json b/conformance/soil_and_roots.expected.json index a4bfb33..13d09b5 100644 --- a/conformance/soil_and_roots.expected.json +++ b/conformance/soil_and_roots.expected.json @@ -440,7 +440,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy root system", "scalars": { "name": "scalars", @@ -2889,7 +2889,7 @@ "rank": null } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy soil", "grid": { "kind": "regular", diff --git a/conformance/soil_and_roots.glb b/conformance/soil_and_roots.glb index 8a17a94..88562f5 100644 Binary files a/conformance/soil_and_roots.glb and b/conformance/soil_and_roots.glb differ diff --git a/conformance/wheat_shoot.expected.json b/conformance/wheat_shoot.expected.json index e0edbe0..0f9b2f0 100644 --- a/conformance/wheat_shoot.expected.json +++ b/conformance/wheat_shoot.expected.json @@ -256,7 +256,7 @@ "rank": 6 } ], - "platform": "fspm-tf", + "platform": "encoding3d", "model": "toy shoot", "meshes": [ { diff --git a/conformance/wheat_shoot.glb b/conformance/wheat_shoot.glb index 6c059ce..79607c9 100644 Binary files a/conformance/wheat_shoot.glb and b/conformance/wheat_shoot.glb differ diff --git a/docs/_static/images/archisimple.png b/doc/_static/images/archisimple.png similarity index 100% rename from docs/_static/images/archisimple.png rename to doc/_static/images/archisimple.png diff --git a/docs/_static/images/granap_section.png b/doc/_static/images/granap_section.png similarity index 100% rename from docs/_static/images/granap_section.png rename to doc/_static/images/granap_section.png diff --git a/docs/_static/images/mecha_section.png b/doc/_static/images/mecha_section.png similarity index 100% rename from docs/_static/images/mecha_section.png rename to doc/_static/images/mecha_section.png diff --git a/docs/_static/images/root_system.png b/doc/_static/images/root_system.png similarity index 100% rename from docs/_static/images/root_system.png rename to doc/_static/images/root_system.png diff --git a/doc/_static/images/views_layers.png b/doc/_static/images/views_layers.png new file mode 100644 index 0000000..3ce1f5a Binary files /dev/null and b/doc/_static/images/views_layers.png differ diff --git a/docs/_static/images/wheat_shoot.png b/doc/_static/images/wheat_shoot.png similarity index 100% rename from docs/_static/images/wheat_shoot.png rename to doc/_static/images/wheat_shoot.png diff --git a/docs/_static/openalea_web.svg b/doc/_static/openalea_web.svg similarity index 100% rename from docs/_static/openalea_web.svg rename to doc/_static/openalea_web.svg diff --git a/doc/api.md b/doc/api.md new file mode 100644 index 0000000..84615c6 --- /dev/null +++ b/doc/api.md @@ -0,0 +1,92 @@ +# Python API + +`openalea.encoding3d` is the reference codec: reading and writing frames and sequences, the validator, display copies, +the vocabulary and units. + +## Reading and writing + +```{eval-rst} +.. automodule:: openalea.encoding3d.codec + :members: read, write, from_bytes, to_bytes, write_sequence, read_sequence, write_index, read_index +``` + +## The data model + +```{eval-rst} +.. automodule:: openalea.encoding3d.model + :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable, Update, LazyColumns, LazyList +``` + +## Validation + +```{eval-rst} +.. automodule:: openalea.encoding3d.validate + :members: validate_file, validate_bytes, validate_sequence, Report, Issue +``` + +## Display copies + +```{eval-rst} +.. automodule:: openalea.encoding3d.display + :members: add_display, remove_display +``` + +## Views + +```{eval-rst} +.. automodule:: openalea.encoding3d.views + :members: read_views, write_views, views_path, check_views, configuration, configuration_names, selected, + display_settings, layer_from_display, save_configuration, block_selector +``` + +## Vocabulary and units + +```{eval-rst} +.. automodule:: openalea.encoding3d.vocabulary + :members: standard_names, platform_names, labels, enums, version + +.. automodule:: openalea.encoding3d.ucum + :members: dimension, is_valid, convertible, conversion_factor +``` + +## Comparing with expected values + +```{eval-rst} +.. automodule:: openalea.encoding3d.compare + :members: compare, compare_files, compare_all +``` + +## Expected values + +```{eval-rst} +.. automodule:: openalea.encoding3d.dump + :members: frame_to_dict +``` + +## Coupling + +```{eval-rst} +.. automodule:: openalea.encoding3d.coupling.messages + :members: Declaration, EntitySetSpec, VariableSpec, Link, CouplingError + +.. automodule:: openalea.encoding3d.coupling.participant + :members: Participant, Turn + +.. automodule:: openalea.encoding3d.coupling.coordinator + :members: Coordinator + +.. automodule:: openalea.encoding3d.coupling.resolution + :members: resolve, stages, Coupling + +.. automodule:: openalea.encoding3d.coupling.replay + :members: Replay + +.. automodule:: openalea.encoding3d.coupling.conformance + :members: run_session, run_all, subject +``` + +## Command line + +```{eval-rst} +.. automodule:: openalea.encoding3d.cli +``` diff --git a/docs/conf.py b/doc/conf.py similarity index 58% rename from docs/conf.py rename to doc/conf.py index c06c8d0..9d34cd2 100644 --- a/docs/conf.py +++ b/doc/conf.py @@ -1,60 +1,89 @@ # -*- coding: utf-8 -*- -"""Sphinx configuration of fspm-TF's documentation. The vocabulary's pages are generated from spec/vocabulary at each +"""Sphinx configuration of encoding3d's documentation. The vocabulary's pages are generated from spec/vocabulary at each build (_generated/), so that they never differ from the agreed files.""" import csv import json import os import sys -from importlib.metadata import PackageNotFoundError, metadata +from importlib.metadata import metadata -import pydata_sphinx_theme # noqa: F401 (Pydata theme: https://pydata-sphinx-theme.readthedocs.io) +import pydata_sphinx_theme # noqa: F401 (Pydata theme: https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html) HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) -sys.path.insert(0, os.path.join(ROOT, "bindings", "python", "src")) # the package, installed or not -project = "fspm-TF" -try: - release = metadata("openalea.fspm_tf").get("version") -except PackageNotFoundError: - release = "0.1" +# The package's metadata (pyproject.toml), as every OpenAlea documentation reads it +package = "openalea.encoding3d" +meta = metadata(package) +project = "encoding3d" +release = meta.get("version") version = ".".join(release.split(".")[:3]) -author = "Tristan Gérault" -copyright = "Cecill-C INRAE / INRIA / CIRAD" +author = meta["Author-email"].split(" <")[0] +desc = meta["Summary"] +urls = {key.strip(): value.strip() for key, value in (item.split(",") for item in meta.get_all("Project-URL"))} + +# The package's root, for autodoc when it is not installed (its sources are in bindings/python/src) +sys.path.insert(0, os.path.join(ROOT, "bindings", "python", "src")) +# -- General configuration ------------------------------------------------ extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", - "sphinx.ext.intersphinx", - "sphinx_copybutton", - "sphinx_favicon", - "myst_parser", + "sphinx.ext.autodoc", # support for automatic inclusion of docstrings + "sphinx.ext.autosummary", # generates autodoc summaries + "sphinx.ext.doctest", # inclusion and testing of doctest code snippets + "sphinx.ext.intersphinx", # support for linking to other projects + "sphinx.ext.mathjax", # support for math equations + "sphinx.ext.ifconfig", # support for conditional content + "sphinx.ext.viewcode", # support for links to source code + "sphinx.ext.coverage", # includes doc coverage stats in the documentation + "sphinx.ext.todo", # support for todo items + "sphinx.ext.napoleon", # support for numpy and google style docstrings + "sphinx_favicon", # support for favicon + "sphinx_copybutton", # support for copybutton in code blocks + "nbsphinx", # for integrating jupyter notebooks + "myst_parser", # for parsing .md files ] -myst_enable_extensions = ["colon_fence", "deflist"] -myst_heading_anchors = 3 templates_path = ["_templates"] -exclude_patterns = ["_build", "_templates"] +autosummary_generate = True +exclude_patterns = ["build", "_build", "_templates", "**.ipynb_checkpoints"] source_suffix = {".rst": "restructuredtext", ".md": "markdown"} master_doc = "index" +copyright = "Cecill-C INRAE / INRIA / CIRAD" language = "en" pygments_style = "sphinx" +todo_include_todos = False +myst_enable_extensions = ["colon_fence", "deflist"] +myst_heading_anchors = 3 autodoc_member_order = "bysource" -autodoc_default_options = {"members": True, "undoc-members": False} +nbsphinx_execute = "always" # the notebooks run at each build, as nbmake runs them in the tests +# -- Options for HTML output ---------------------------------------------- html_theme = "pydata_sphinx_theme" html_theme_options = { "header_links_before_dropdown": 6, - "icon_links": [{"name": "GitHub", "url": "https://github.com/openalea/fspm-TF", "icon": "fa-brands fa-github"}], + "icon_links": [{"name": "GitHub", "url": urls["Repository"], "icon": "fa-brands fa-github"}], + "show_version_warning_banner": True, "footer_start": ["copyright"], "footer_center": ["sphinx-version"], - "secondary_sidebar_items": {"**/*": ["page-toc", "edit-this-page", "sourcelink"]}, + "secondary_sidebar_items": {"**/*": ["page-toc", "edit-this-page", "sourcelink"], "examples/no-sidebar": []}, "use_edit_page_button": True, } html_static_path = ["_static"] html_logo = "_static/openalea_web.svg" html_favicon = "_static/openalea_web.svg" -html_context = {"github_user": "openalea", "github_repo": "fspm-TF", "github_version": "main", "doc_path": "docs"} +html_domain_indices = True +html_use_index = True +html_split_index = False +html_show_sourcelink = True +html_show_sphinx = True +html_show_copyright = True +htmlhelp_basename = "encoding3d_documentation" +html_context = {"github_user": "openalea", "github_repo": "encoding3d", "github_version": "main", "doc_path": "doc"} + +# -- Options for LaTeX, manual page and Texinfo output --------------------- +latex_elements = {} +latex_documents = [(master_doc, "encoding3d.tex", "encoding3d Documentation", "INRAE / inria / CIRAD", "manual")] +man_pages = [(master_doc, "encoding3d", "encoding3d Documentation", [author], 1)] +texinfo_documents = [(master_doc, "encoding3d", "encoding3d Documentation", author, "encoding3d", desc, "Miscellaneous")] intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} diff --git a/doc/conformance.md b/doc/conformance.md new file mode 100644 index 0000000..5da5ab7 --- /dev/null +++ b/doc/conformance.md @@ -0,0 +1,2 @@ +```{include} ../conformance/README.md +``` diff --git a/doc/coupling.md b/doc/coupling.md new file mode 100644 index 0000000..633d75a --- /dev/null +++ b/doc/coupling.md @@ -0,0 +1,158 @@ +# Coupling platforms + +encoding3d also runs models of several platforms together, one time step after the other. This is a **session**: + +- each platform's model is a **participant**, running in its own process and its own language; +- a **coordinator** (encoding3d's, in Python) starts them, passes each one the values it needs from the others, and + keeps the plant they share. + +Two platforms can work on the same plant: one grows the organs, another splits them into segments and computes a +function on them, a third reads the plant from a recorded run. Each one only says what it works on, what it +provides and what it needs; the coordinator does the rest. + +The rules are in the specification's §11 (version 0.3, a draft the other platforms' teams are to review). Python, +Julia, C++ and Java have a participant; the coordinator is Python's. + +## What a participant says + +A participant **declares**: + +- the entity sets it works on (`Organ`, `SubOrgan`, …) and what it may do with their entities: `owns` (it creates + and removes them), `refines` (it adds its own entities under others', e.g. segments under organs), `follows` (it + reads them); +- the variables it **provides** (its outputs) and **requires** (its inputs), each with its unit and, if the + vocabulary has one, its standard name. + +A **link** says which output feeds which input, when the names differ: a factor, and a change of scale (a sum or a +mean of finer entities, a copy of a coarser one's value). Without a link, an input is fed by the output with the +same standard name, else the same name. Units are converted. + +Two **data structures** (a plant and a soil grid, say) are linked through a **mapping**, the weights between their +entities, which one participant provides with its outputs (declared in its `mappings`, sent as the frame's mapping). +The link names it: `Link("plant", "water", "soil", "water_content", mapping="roots_in_soil")`. Through it, `sum` +shares each source's amount among its targets in proportion to the weights (the total is kept), and `mean` gives each +target the weighted mean of its sources. + +A participant declares only what it works on. A model growing the organs needs to know nothing of the segments +another platform refines them into: it is never sent them, and when it removes an organ, the coordinator removes its +segments and tells their refiner. + +## A session in Python + +A participant needs the encoding3d codec, and nothing else of the other platforms: + +```python +from openalea import encoding3d +from openalea.encoding3d.coupling import Declaration, EntitySetSpec, Participant, VariableSpec + +declaration = Declaration("function", entity_sets=[EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights="refines")], + variables=[VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol")]) +participant = Participant.connect(declaration) # the session's address and its name come from the coordinator +for turn in participant.turns(): # until the coordinator stops the session + plant = turn.frame.data_structure("plant") # the organs and segments, the segments' lengths + ... # the model, from turn.time to turn.until + participant.send(frame) # its new segments and their sugar +``` + +Each turn is a frame holding the participant's entity sets and inputs, and the answer is a frame holding its new +entities and outputs (only those: the coordinator keeps the rest). The entities it creates take ids from +`participant.id_offset`. + +## A session in Julia, C++ or Java + +The same participant in Julia (encoding3d.jl; ZeroMQ needs `using ZMQ`): + +```julia +using encoding3d, ZMQ + +participant = Participant(Declaration("function"; entity_sets = [...], variables = [...])) +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + # the model + send_update(participant, frame) +end +``` + +In C++ (`encoding3d/coupling.hpp`; ZeroMQ when built with libzmq) and Java (`org.openalea.encoding3d.coupling`; ZeroMQ with +JeroMQ on the class path), the declaration is its JSON: + +```cpp +encoding3d::coupling::Participant participant(declaration); // a JSON declaration (nlohmann) +while (auto turn = participant.next_turn()) { + // the model, on turn->frame + participant.send_update(frame); +} +``` + +```java +try (Participant participant = new Participant(declaration, null, null, null)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + // the model, on turn.frame() + participant.sendUpdate(frame); + } +} +``` + +## Running it + +The coordinator starts each participant's command, and waits for the ones started otherwise (`--expect`): + +```bash +encoding3d couple --steps 30 --interval 1 --time-unit d \ + --participant growth="julia --project=vpl growth.jl" \ + --participant function="python function.py" \ + --links links.json --record run/ +``` + +- `--links`: a JSON list of links, e.g. + `[{"receiver": "growth", "variable": "supply", "provider": "function", "source": "sugar"}]`. +- `--record`: the whole session written as an ordinary sequence of frames, to read in ParaView or to replay. +- `--replay name=run/scene.fspm.json`: a recorded run takes part as a participant, without its platform running. + A platform's routine log can thus feed another platform's model. CPlantBox's and GroIMP's logs are tested this way. +- `--endpoint files:folder`: the participants exchange files in a folder instead of ZeroMQ messages. It is slower, + but needs nothing but a file system. Each message is deleted once read; `--keep-messages` keeps them all, as the + session's log (a turn of 100,000 segments with five variables is about 5 MB: plan the disk space). The usual log + is `--record`. +- `--stages "growth;function"`: who runs first in each step (by default, the participants creating entities, then + the others). Each step, a stage sees what the previous stages did in that step. + +In Python, `openalea.encoding3d.coupling.Coordinator` does the same from a script. + +## In metafspm + +metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for another platform's model inside a metafspm +model: a component computing nothing, whose state variables are the other model's outputs and whose inputs are sent +to it. Its tests couple metafspm's root growth to a carbon model written in VirtualPlantLab. + +## How fast it is + +The exchange's own cost per step, with the coordinator and two participants in separate processes, each sending five +variables on every segment: + +| Segments | ZeroMQ | A folder | +|---|---|---| +| 1,000 | 3 ms | 12 ms | +| 10,000 | 9–11 ms | 21 ms | +| 100,000 | 99–108 ms | 110–122 ms | + +ZeroMQ here goes through the local network interface (`tcp://127.0.0.1`), its messages held in memory; the folder's +messages are files. The folder costs about 10 ms more per step (its polling and file writes). + +Encoding and decoding the frames is about a tenth of it (10 ms for 100,000 segments); the rest is the coordinator's +work on the shared structure, in Python. + +## Checking a participant + +`conformance/coupling/` holds conformance sessions: a peer the coordinator replays, and what the participant under +test must read and send. `encoding3d couple-conformance conformance/coupling --participant COMMAND` runs them over both +transports; each codec's CI runs its own participant (Python, Julia, C++, Java) this way. + +## How it behaves + +- The coupling is explicit: a participant's inputs at a step are what the others gave at their previous turn. +- Entities removed by their owner are removed, with their finer entities, from every participant that follows them. +- A participant receiving new entities gives its own variables on them their initial values, and shares its own + quantities when an entity splits, by its own rules. +- A participant's error stops the session; every participant is told why. diff --git a/doc/cpp.md b/doc/cpp.md new file mode 100644 index 0000000..e4a3b14 --- /dev/null +++ b/doc/cpp.md @@ -0,0 +1,126 @@ +# Reading and writing in C++, and CPlantBox + +`bindings/cpp/` holds the C++ codec (C++17). It reads and writes encoding3d files and passes the conformance check; its +tests run in `.github/workflows/cpp-codec.yml` on Linux, macOS and Windows. In `bindings/cpp/cplantbox/` is CPlantBox's +adapter, a proposal for CPlantBox, tested by hand (below). + +Licence: GPL-3.0, as CPlantBox (the rest of encoding3d is CeCILL-C). + +## Building + +```bash +cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +cmake --install build --prefix /some/prefix # then: find_package(encoding3d), encoding3d::encoding3d +``` + +The one dependency, nlohmann_json, is used when installed and fetched otherwise. + +## The format's own tables + +```cpp +#include + +auto frame = encoding3d::read_frame("scene.glb"); +const auto &segments = frame.data_structure("RootPopulation").entity_set("SubOrgan"); +const auto &length = segments.column("length").as(); // one value per segment +const auto unit = segments.variables.at("length").unit; // its declared unit, e.g. "m" + +encoding3d::EntitySet organs("Organ", 5); +organs.set("id", std::vector{1, 2}) + .set("parent", std::vector{-1, 1}) + .set("edge_type", {"none", "branch"}) + .set("length", std::vector{0.12, 0.05}); +encoding3d::Variable declaration; +declaration.unit = "m"; +declaration.standard_name = "length"; +organs.declare("length", declaration); +encoding3d::DataStructure plants("plants"); +plants.entity_sets.push_back(organs); +encoding3d::Frame written; +written.data_structures.push_back(plants); +encoding3d::write_frame("plants.glb", written); +``` + +A column holds a vector of numbers (any integer or float type), `bool` or `std::string`. With a `width` of 2 or +more, it holds a fixed-size vector per row. NaN and infinities are written with the specification's stand-ins and +read back as they were. + +## Coupling with other platforms + +`encoding3d/coupling.hpp` is a participant of the coupling sessions encoding3d's coordinator runs +([Coupling platforms](coupling.md)): `encoding3d::coupling::Participant(declaration)` joins one (the declaration as its +JSON; the address and name from the environment when launched by the coordinator), `next_turn()` gives each turn's +inputs until the session stops, `send_update(frame)` answers. Its folder transport needs nothing; its ZeroMQ transport +is built when CMake finds libzmq (`ENCODING3D_WITH_ZMQ`, on by default; `has_zeromq()` tells). The tool +`encoding3d_coupling_conformance` is its program for the conformance sessions. + +## CPlantBox + +```{note} +**A proposal for CPlantBox.** This adapter is a first proposal for CPlantBox's team to take over; it lives here until +they adopt it, then moves into CPlantBox. encoding3d does not depend on it: its requirements and CI never install +CPlantBox. To build and test it by hand, see `bindings/cpp/cplantbox/README.md`. +``` + +Build the adapter against CPlantBox's sources and its library: + +```bash +cmake -S CPlantBox -B cpb -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF -DCPB_BUILD_SHARED=OFF && cmake --build cpb --target CPlantBox +cmake -S bindings/cpp -B build -DENCODING3D_CPLANTBOX_SOURCE_DIR=$PWD/CPlantBox \ + -DENCODING3D_CPLANTBOX_LIBRARY=$PWD/cpb/src/libCPlantBox.a +``` + +```cpp +#include + +auto plant = std::make_shared(1); +plant->readParameters("fspm2023.xml"); +plant->initialize(); +plant->simulate(28., false); + +auto frame = encoding3d::cplantbox::to_frame(*plant); // Plant, Organ, SubOrgan; time in days +encoding3d::cplantbox::add_geometry(frame.data_structures[0], plant); // leaves and stems, triangles +plant->setRectangularGrid({-10, -10, -40}, {10, 10, 0}, {4, 4, 8}, false); +auto soil = encoding3d::cplantbox::soil_grid(*plant); +frame.mappings.push_back(encoding3d::cplantbox::segments_in_soil(*plant, frame.data_structures[0], soil)); +frame.data_structures.push_back(soil); +encoding3d::write_frame("plant.glb", frame); + +// A soil solver's values per cell (DuMux, or Python), CPlantBox's hydraulics per node and per segment +// (PlantHydraulicModel: rx, radial fluxes), each declared with its unit and standard name +encoding3d::Variable potential; +potential.unit = "cm[H2O]"; +potential.standard_name = "soil_water_potential"; +encoding3d::cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", sx, potential}}); +potential.standard_name = "water_potential"; +encoding3d::cplantbox::add_node_values(frame.data_structures[0], *plant, {{"xylem_potential", rx, potential}}); +encoding3d::Variable uptake; +uptake.unit = "cm3/d"; +uptake.standard_name = "root_water_uptake"; +encoding3d::cplantbox::add_segment_values(frame.data_structures[0], *plant, {{"radial_flux", fluxes, uptake}}); + +// A frame (CPlantBox's, or another platform's) as a static CPlantBox root system +auto roots = encoding3d::cplantbox::to_root_system(encoding3d::read_frame("roots.glb").data_structures[0]); +``` + +| encoding3d | CPlantBox | +|---|---| +| Plant (rank 1), id 0 | the organism | +| Organ (rank 5, scale `mixed`), id 1 + organ id | a seed, a root or a stem (labelled `RootAxis`, `ShootAxis`: a whole axis), a leaf | +| SubOrgan (rank 6), id 2^32 + distal node id | a segment; its parent is the segment ending at its proximal node | +| `x1` … `z2` (`proximal_end_*`, `distal_end_*`) | the segment's nodes, in cm (declared; readers convert, so a rebuilt root system has them bit for bit) | +| other variables | CPlantBox's names and units (`length` in cm, `age` in d, `order`, `subType`, …), with their standard names | +| `extras["cplantbox"]` | the id offsets, the organism's parameters as CPlantBox's XML | + +CPlantBox's core has no soil model: the soil's values come from a solver outside it, which hands them to CPlantBox +per cell. `add_cell_values` writes them in the soil grid's cells, reordered from CPlantBox's cell order (x fastest) to +encoding3d's (C order of x, y, z). `add_node_values` gives each segment its distal node's value. + +CPlantBox has two scales below the plant. `to_root_system` reads frames with more scales by what they hold: the +segments are the entity set with segment ends, and the organs are the set their `complex` points to. For example, +metafspm's Plant, Axis and SubOrgan become CPlantBox root organs. CPlantBox has static roots only, so stems and +leaves are not rebuilt. + +The adapter's tests also write CPlantBox's log of a growing root system (`anagallis_log/`: a frame a day for six +days, and its index), which the CI then replays as a participant of a coupling session: a reader receives every +logged segment and value at each step. diff --git a/docs/design_note.md b/doc/design_note.md similarity index 88% rename from docs/design_note.md rename to doc/design_note.md index d82fea8..3465012 100644 --- a/docs/design_note.md +++ b/doc/design_note.md @@ -1,9 +1,10 @@ -# fspm-TF: design note +# encoding3d: design note *An exchange format for whole FSPM scenes, across platforms and languages.* -Status (2026-10-29): the decisions of §7 are taken on the metafspm side; P1 to P3 are done (§6). The specification -itself is `spec/specification.md`, version 0.1, a draft until the partner platforms have reviewed it. This note keeps +Status (2026-10-29; updated 2026-11-03): the decisions of §7 are taken on the metafspm side; P1 to P5 are done (§6). +The specification itself is `spec/specification.md`, now version 0.3, a draft until the partner platforms have +reviewed it. This note keeps the review of the existing formats and the reasons for the choices; where the two differ, the specification is the reference. Points marked **[to verify]** have not been checked on real files or code yet. @@ -19,7 +20,7 @@ Functional-structural plant models are written on several platforms, each in its These platforms share no common representation of a *scene*: plants with their multiscale structure, their state variables, their geometry, and their environment (soil grids, light, weather). -fspm-TF aims at two uses: +encoding3d aims at two uses: 1. **Static exchange:** one file, written by one platform and read by another. It holds a scene, or a sequence of time steps of it: a simulation's output, or an initial state given to another model. @@ -31,7 +32,7 @@ units, which each platform maps its own names to. Precedent: RSML ([openalea/RSML](https://github.com/openalea/RSML), Lobet et al. 2015) did this for root architectures. It is adopted by about ten tools, including CPlantBox, ArchiSimple, GroIMP's `rsmlreader` and -openalea's `rsml2mtg`. fspm-TF extends the idea to whole scenes, all scales, state variables and environments. +openalea's `rsml2mtg`. encoding3d extends the idea to whole scenes, all scales, state variables and environments. ## 2. Requirements @@ -281,7 +282,7 @@ The following are **not** exchanged; they stay in metafspm's own checkpoint: - derivation formulas and mask rules, since these are code; - the random streams' counters. -**As implemented in P2** (`openalea.metafspm.io.fspm_tf`): +**As implemented in P2** (`openalea.metafspm.io.encoding3d`): - **Every MPG scale holding vertices is an entity set**, Compartments included; the Connections are a relation, with their vids as ids. `complex` is the containing entity in any coarser set: a wall Compartment belongs to its @@ -305,7 +306,7 @@ One glTF asset is **one frame**. The binding uses only: - `EXT_structural_metadata`, property tables only: no property textures and no property attributes, because these can't hold float64; - `EXT_mesh_features`, attribute feature IDs only; -- one fspm-TF extension, `FSPM_scene` (decided, §7.1), holding what the +- one encoding3d extension, `FSPM_scene` (decided, §7.1), holding what the metadata extension cannot hold. #### Layout @@ -335,7 +336,7 @@ meshes TRIANGLES or LINES, POSITION in float32 relat - One glTF node per entity, as PlantConvert does, puts 10⁵ objects in the JSON. It also holds only one tree, so PlantConvert has to duplicate a scene per scale. - The MTG's relations are columns instead, where they are compact, typed and complete. -- **The relational columns are uint64 ids** whose meaning is fixed by fspm-TF, since the metadata extension has no +- **The relational columns are uint64 ids** whose meaning is fixed by encoding3d, since the metadata extension has no reference type. `FSPM_scene` declares, per column, which entity set the ids point to. - **float64 state lives in property tables only.** - Viewers that ignore the extensions still show the geometry, and the `_NAME` float32 display copies (ParaView @@ -358,10 +359,10 @@ pointing at them, capped at 4 GiB. #### Viewing - Any glTF viewer shows the geometry. -- ParaView, with the `openalea.fspm_tf` reader plugin, shows every variable at every scale as float64 cell data, with the +- ParaView, with the `openalea.encoding3d` reader plugin, shows every variable at every scale as float64 cell data, with the index's frames as time steps (§7.4). - ParaView without the plugin, and other viewers, show the display copies: float32 per-vertex copies of chosen - variables. Files have none by default; the writer's `display=` option or the `fspm-tf display` tool adds them. + variables. Files have none by default; the writer's `display=` option or the `encoding3d display` tool adds them. - Cesium and 3DTilesRendererJS show every table. ### 4.5 The vocabulary @@ -405,9 +406,17 @@ A coupling **protocol** shaped like BMI: This is a later phase (§6). The data model is specified first, so that the coupling inherits it. +**Update (2026-11-03): built differently.** The coupling is the specification's §11. A coordinator (encoding3d's, in +Python) holds the shared structure and applies the links; each platform's participant declares what it works on, +provides and requires, then answers turns with updates. The messages carry **partial frames**, the same GLB as a time +step, not Arrow batches: every codec already reads and writes GLB, Arrow would have been a large new dependency in +all four (heavy in Java and C++), and measurements showed the format is not the cost (encoding 10 ms of a 100 ms step +at 100,000 segments; the rest is the coordinator's work). Transports: ZeroMQ, and a folder of files. A BMI-like +in-process interface is not part of it. + ### 4.7 VTKHDF (dropped) -A VTKHDF export was considered for viewing long simulations in ParaView. It is dropped, to keep fspm-TF free of an +A VTKHDF export was considered for viewing long simulations in ParaView. It is dropped, to keep encoding3d free of an HDF5 dependency (§7.8). ParaView views the glTF files themselves, through display copies. ### 4.8 Read and write cost in metafspm, and the glTF-backed DataStructure question @@ -433,7 +442,7 @@ A DataStructure *backed* by a glTF buffer would have to keep the JSON and the bu speed. **Recommendation: no backing DataStructure.** Provide instead (all done in P2): - `export_frame(path, data structures)`, `export_scene(path, scene)` and `import_frame(path) -> data structures`, in - metafspm's `io` module, on top of the `openalea.fspm_tf` codec (§5.1); + metafspm's `io` module, on top of the `openalea.encoding3d` codec (§5.1); - a `SceneRecorder` option writing frames every N steps: `Scene(frame_period=N)`; - the round-trip test as the contract: export → import gives the same ids, topology, values (bit for bit), metadata and mappings, for every DataStructure type and location; @@ -444,15 +453,15 @@ speed. **Recommendation: no backing DataStructure.** Provide instead (all done i This repository is platform-neutral, separate from metafspm, like RSML. ``` -fspm-TF/ +encoding3d/ README.md - docs/design_note.md this note + doc/design_note.md this note spec/ the agreement, the only folder discussed with the platforms: specification.md the specification schema/ JSON Schema of FSPM_scene and of the index file vocabulary/ standard_names.csv (with one column of names per platform), labels.csv, enums.json bindings/python/ the Python reference package (§5.1): codec, validator, display tool, command line, and - its tests; it reads spec/ and ships it (openalea.fspm_tf.spec) + its tests; it reads spec/ and ships it (openalea.encoding3d.spec) bindings// the other languages' codecs, by their teams (adapters stay in the platforms) paraview/ the ParaView reader plugin conformance/ the reference files: a root system, a wheat shoot, a cell network, a soil grid with @@ -466,7 +475,7 @@ fspm-TF/ - **The repository holds:** the specification, the schema, the vocabulary, the conformance files, and a small Python package, which is generic and knows no platform. - **Each platform's own mapping lives in that platform:** metafspm's in metafspm, CPlantBox's in CPlantBox, and so on. -- **fspm-TF is separate from RSML and PlantConvert** (decided, §7.6). Converters from OPF, `.mtg` or RSML can come +- **encoding3d is separate from RSML and PlantConvert** (decided, §7.6). Converters from OPF, `.mtg` or RSML can come later, in PlantConvert. - **Rules:** - semantic versioning of the specification; @@ -479,19 +488,19 @@ fspm-TF/ Everything maintained in this repository is Python. The other platforms take part through files and a command line, not code: no C++, Java or Julia has to be maintained here. -**The Python package `openalea.fspm_tf`.** Its only dependencies are numpy and jsonschema; GLB is a 12-byte header plus a JSON +**The Python package `openalea.encoding3d`.** Its only dependencies are numpy and jsonschema; GLB is a 12-byte header plus a JSON chunk and a binary chunk, so no glTF library is needed. It contains: -- `openalea.fspm_tf.codec`: +- `openalea.encoding3d.codec`: - `write(path, frame)` and `read(path) -> frame`, where a frame is plain Python: data structures, entity sets as `{column: numpy array}`, variable declarations as dicts, meshes as arrays with feature IDs; - `write_index` and `read_index` for sequences. -- `openalea.fspm_tf.validate`: JSON Schema checks of `FSPM_scene` and the index; the checks the schema cannot express (ids +- `openalea.encoding3d.validate`: JSON Schema checks of `FSPM_scene` and the index; the checks the schema cannot express (ids unique and never reused across frames, `complex` and `parent` pointing to existing entities, units parsable as UCUM, kinds and standard names in the vocabulary, tables 8-byte aligned, display copies flagged). -- `openalea.fspm_tf.display`: adds or removes display copies (§7.4). -- `openalea.fspm_tf.vocabulary`: the vocabulary table, shipped as package data. -- A command line: `fspm-tf validate file.glb`, `fspm-tf display file.glb name ...`, `fspm-tf info file.glb`. +- `openalea.encoding3d.display`: adds or removes display copies (§7.4). +- `openalea.encoding3d.vocabulary`: the vocabulary table, shipped as package data. +- A command line: `encoding3d validate file.glb`, `encoding3d display file.glb name ...`, `encoding3d info file.glb`. **Tests and CI**, the same way as metafspm: @@ -505,7 +514,7 @@ chunk and a binary chunk, so no glTF library is needed. It contains: **How another platform tests its implementation**, in its own CI: -- **Writing:** it writes the conformance scenes with its own code and runs `fspm-tf validate` (conda or pip, or +- **Writing:** it writes the conformance scenes with its own code and runs `encoding3d validate` (conda or pip, or a container). - **Reading:** it reads the conformance GLBs with its own code and compares the values with the expected JSON. @@ -514,14 +523,14 @@ claims. **How metafspm uses it:** -- `openalea.metafspm.io.fspm_tf`, a new module, maps metafspm's DataStructures, mappings and declarations to - `fspm_tf` frames and back (§4.3), and calls the codec. It is the only metafspm code concerned. -- metafspm's environment and recipe add `openalea.fspm_tf`, pinned to the specification version it implements - (`openalea.fspm_tf >=0.1,<0.2`). -- During development it is installed with `pip install -e ../fspm-TF`, as metafspm itself is in the ported packages. -- metafspm's tests use the round trip (§4.8), `openalea.fspm_tf.validate` on what it writes, and the conformance files on +- `openalea.metafspm.io.encoding3d`, a new module, maps metafspm's DataStructures, mappings and declarations to + `encoding3d` frames and back (§4.3), and calls the codec. It is the only metafspm code concerned. +- metafspm's environment and recipe add `openalea.encoding3d`, pinned to the specification version it implements + (`openalea.encoding3d >=0.1,<0.2`). +- During development it is installed with `pip install -e ../encoding3d`, as metafspm itself is in the ported packages. +- metafspm's tests use the round trip (§4.8), `openalea.encoding3d.validate` on what it writes, and the conformance files on what it reads. -- A new release of the specification is a new `openalea.fspm_tf` version. metafspm moves to it with a pin change, and its +- A new release of the specification is a new `openalea.encoding3d` version. metafspm moves to it with a pin change, and its tests tell what has to follow. ### 5.2 What other languages need @@ -535,7 +544,7 @@ in `schema/`, `vocabulary/` and `conformance/` (and as release assets): | `schema/fspm_scene.schema.json`, `schema/index.schema.json` | the structure of `FSPM_scene` and of the index | any JSON Schema validator: valijson or json-schema-validator (C++), networknt json-schema-validator (Java), JSONSchema.jl (Julia) | | `vocabulary/standard_names.csv`, `vocabulary/enums.json` | standard names with units and kinds; labels, edge types, relations, kinds, roles | any CSV or JSON reader | | `conformance/*.glb` with `*.expected.json` | the reference scenes and their values | the platform's own codec, against the JSON | -| `fspm-tf validate` | the full checks, as a command line | its CI (conda, pip, or a container image) | +| `encoding3d validate` | the full checks, as a command line | its CI (conda, pip, or a container image) | What a platform writes is **its codec**: - **The GLB container:** a 12-byte header, a JSON chunk and a binary chunk. @@ -544,24 +553,29 @@ What a platform writes is **its codec**: That is a few hundred lines in any language, and existing libraries can help: - **C++ (CPlantBox):** cesium-native reads and writes `EXT_structural_metadata` fully. CPlantBox can also begin - through its Python layer, with `openalea.fspm_tf` directly. + through its Python layer, with `openalea.encoding3d` directly. - **Java (GroIMP):** JglTF has the extension's JSON classes. GroIMP's existing glTF plugin is the place to extend. - **Julia (VPL):** GLTF.jl handles `.gltf` JSON only, so a small GLB reader has to be written. That is the part above, the header and the two chunks. These codecs live in the platforms' repositories, maintained by their teams. `implementations.md` lists them with -the specification version each one passes. fspm-TF itself maintains only Python. +the specification version each one passes. encoding3d itself maintains only Python. + +**Update (2026-11-03): decided otherwise.** The codecs live in this repository (`bindings//`: Python, Julia, +C++, Java), each passing the conformance files; the adapters (a platform's model to and from the codec's tables) +belong to the platforms, as Julia package extensions where the platform's library is a plain dependency. +`implementations.md` lists them. ## 6. Plan | Phase | Content | Done when | Status | |---|---|---|---| | P0 | This note, discussed with the partner platforms | the [decision] points settled | decisions taken on the metafspm side; review by the partners to come | -| P1 | Specification v0.1: data model, glTF binding, JSON Schema, first vocabulary; the `openalea.fspm_tf` package (codec, validator, display tool), the conformance files, CI | the validator and glTF-Validator accept the conformance files | done | +| P1 | Specification v0.1: data model, glTF binding, JSON Schema, first vocabulary; the `openalea.encoding3d` package (codec, validator, display tool), the conformance files, CI | the validator and glTF-Validator accept the conformance files | done | | P2 | metafspm reference implementation: `export_frame` and `import_frame` for MPG, Array and AdaptiveGrid DataStructures, mappings and triangles | round-trip tests bit-exact on the ported models' scenes (ArchiSimple, Adel, Caribu, GRANAP and MECHA, soil) | done, with `Scene(frame_period=)`; the continuation after an import is exact too | -| P3 | The ParaView reader plugin (Python, on the `openalea.fspm_tf` codec: every variable as cell data, frames as time steps); viewer checks of display copies: ParaView without the plugin, three.js/3DTilesRenderer, Blender | the example files viewed with their variables | ParaView done (6.0.1, the plugin and ParaView's own glTF reader); three.js and Blender skipped | -| P4 | A second platform: CPlantBox through its Python layer (the easiest), or VPL through PlantGeom.jl | a file written by one platform and read by the other, with the same values | not started | -| P5 | Arrow binding and coupling protocol; a two-platform coupled run | one component in metafspm and one in another platform exchanging tables at each step | not started | +| P3 | The ParaView reader plugin (Python, on the `openalea.encoding3d` codec: every variable as cell data, frames as time steps); viewer checks of display copies: ParaView without the plugin, three.js/3DTilesRenderer, Blender | the example files viewed with their variables | ParaView done (6.0.1, the plugin and ParaView's own glTF reader); three.js and Blender skipped | +| P4 | A second platform: CPlantBox through its Python layer (the easiest), or VPL through PlantGeom.jl | a file written by one platform and read by the other, with the same values | done, and more: the Julia (VirtualPlantLab, MTG.jl), C++ (CPlantBox) and Java (GroIMP) codecs and adapters ([Implementations](implementations.md)) | +| P5 | Arrow binding and coupling protocol; a two-platform coupled run | one component in metafspm and one in another platform exchanging tables at each step | done without Arrow (§4.6's update): the coupling sessions of the specification's §11; metafspm coupled with VirtualPlantLab | ## 7. Open decisions @@ -589,7 +603,7 @@ answer: I don't understand the question. **Proposal:** - Files hold no display copies by default: they are exact and lean. - The writer takes an option `display=[names]` to add some. - - The repository's tool also adds them to an existing file afterwards (`fspm-tf display file.glb + - The repository's tool also adds them to an existing file afterwards (`encoding3d display file.glb water_potential`), computed from the tables. A file can then be prepared for ParaView at any time, and no platform has to implement this. - Display copies are flagged as such, and readers ignore them as state. @@ -604,10 +618,10 @@ answer: I don't understand the question. show its values, and colour by a property through a custom shader. But they are geospatial and web-graphics tools: there is no plot, no filter, and no notion of our index of time steps. - The best-suited option is a **ParaView reader plugin written in Python**, shipped with `openalea.fspm_tf`: + The best-suited option is a **ParaView reader plugin written in Python**, shipped with `openalea.encoding3d`: - ParaView loads Python readers (a `VTKPythonAlgorithmBase` with its decorators, since ParaView 5.6). It runs - them in its own Python, which has numpy, so the pure-Python `openalea.fspm_tf` codec runs inside ParaView. - (Checked with ParaView 6.0.1: nothing to install, the plugin finds `openalea.fspm_tf` in the clone's `bindings/python/src/`.) + them in its own Python, which has numpy, so the pure-Python `openalea.encoding3d` codec runs inside ParaView. + (Checked with ParaView 6.0.1: nothing to install, the plugin finds `openalea.encoding3d` in the clone's `bindings/python/src/`.) - The plugin reads the GLB (or the index file) and turns each triangle's entity value into float64 *cell data* (one value per triangle, from the feature IDs). Every variable, at every scale, becomes available for colouring, filters and plots. The index's frames become ParaView's time steps, with a growing topology. @@ -624,18 +638,18 @@ answer: also parameters, that would enable also to exchange them between platfor - one value per entity type, as a column of that scale. - A file holding only parameters, with no entities, is valid: a **parameter set** that can be exchanged on its own (§4.2). -6. Coordination with PlantConvert and with the RSML maintainers: one family of formats, or fspm-TF absorbing RSML's +6. Coordination with PlantConvert and with the RSML maintainers: one family of formats, or encoding3d absorbing RSML's use cases. answer: should be a separated initiative. In this regard, how is the test / source management and CI to be handled, since I am only used to python? In the example of metafspm, how will this repo or its release be used to build metafspm's io? - → **Decided:** fspm-TF is an initiative separate from RSML and PlantConvert. Converters to and from them can come + → **Decided:** encoding3d is an initiative separate from RSML and PlantConvert. Converters to and from them can come later, in PlantConvert. On tests, sources, CI and metafspm's use of the repository: see §5.1. In short: - - The repository holds the specification and a **Python** package, `openalea.fspm_tf`, which depends only on numpy and + - The repository holds the specification and a **Python** package, `openalea.encoding3d`, which depends only on numpy and jsonschema. It is the reference codec: it reads and writes GLB files, their property tables and the index, and knows nothing of metafspm. It also contains the validator and the conformance files. - Other platforms need no Python of yours. They use the conformance files (GLBs plus their expected values as JSON) as test fixtures in their own CI, and run the validator on what they write. The validator is installed by `pip` or `conda` and run as a command line. - - metafspm depends on `openalea.fspm_tf` like on any package, pinned to a specification version. Its `io` module maps + - metafspm depends on `openalea.encoding3d` like on any package, pinned to a specification version. Its `io` module maps DataStructures to the generic model and calls the codec. answer: ok for the strategy, I see how the python package will make it easily available in python, but for other languages / platforms what does it imply to make the format's conventions available? @@ -646,7 +660,7 @@ answer: should be a separated initiative. In this regard, how is the test / sour conventions. - Each platform writes **its own codec**: GLB's three parts, the property tables, and the mapping to its own model. - The validator checks every platform's output from the command line. - - CPlantBox can also start from its Python layer with `openalea.fspm_tf` directly. + - CPlantBox can also start from its Python layer with `openalea.encoding3d` directly. 7. The first vocabulary: who proposes it, and the 30 names to start from. The first vocabulary can be inspired from metafspm's current version, but if metafspm lacks some aspects, you can inspire from other platforms. → **Decided:** the first vocabulary is drafted from the declarations of metafspm and of its ported packages diff --git a/docs/governance.md b/doc/governance.md similarity index 82% rename from docs/governance.md rename to doc/governance.md index 85be9af..0156538 100644 --- a/docs/governance.md +++ b/doc/governance.md @@ -43,5 +43,7 @@ The CI then runs every codec against the new conformance files, and shows which ## Status -Version 0.1 is a draft. It was written with metafspm as the first platform; its partners' reviews come next. Until -0.1 is released, the draft may still change in ways that make earlier files invalid. +Version 0.3 is a draft. 0.1 was written with metafspm as the first platform; 0.2 adds what the C++ (CPlantBox), Julia +(VirtualPlantLab) and Java (GroIMP) bindings needed (its "Changes from 0.1"); 0.3 adds coupling sessions (§11, "Changes +from 0.2"), which the other platforms' teams are to review. Until 0.3 is released, the draft may still change in ways +that make earlier files invalid. diff --git a/docs/implementations.md b/doc/implementations.md similarity index 100% rename from docs/implementations.md rename to doc/implementations.md diff --git a/docs/index.md b/doc/index.md similarity index 65% rename from docs/index.md rename to doc/index.md index 960faa5..9fcee9f 100644 --- a/docs/index.md +++ b/doc/index.md @@ -1,9 +1,9 @@ -# fspm-TF +# encoding3d **One file format for whole plant-model scenes, shared by the FSPM platforms.** Functional-structural plant models (FSPM) are written on several platforms: openalea.metafspm (Python), CPlantBox -(C++), GroIMP (Java) and VirtualPlantLab (Julia). fspm-TF lets them exchange what they simulate. A file holds a whole +(C++), GroIMP (Java) and VirtualPlantLab (Julia). encoding3d lets them exchange what they simulate. A file holds a whole scene at one time step: - **the plants:** their organs and elements at every scale, from the plant down to the cell, and how they connect; @@ -12,38 +12,47 @@ scene at one time step: - **the environment:** soil grids, and which soil cells each root element lies in. A simulation writes one file per time step. Any platform can read them back. ParaView, Blender or any glTF viewer -can display them. +can display them. Platforms can also run their models together, step by step, exchanging their values through the +same format ([Coupling platforms](coupling.md)). ```{image} _static/images/archisimple.png -:alt: An ArchiSimple root system read from an fspm-TF file in ParaView, coloured by root order +:alt: An ArchiSimple root system read from an encoding3d file in ParaView, coloured by root order :width: 360px :align: center ``` The files are glTF 2.0 binaries (`.glb`): the 3D format of the web and of most viewers, extended with typed tables. -## Using fspm-TF +## Using encoding3d For those who run models and want to look at, keep or share their results. ```{toctree} :maxdepth: 1 -fspm-TF in five minutes +encoding3d in five minutes +Tutorials Viewing and checking files Reading and writing in Python +A notebook: encoding3d in Python +Reading and writing in Julia +Reading and writing in C++, and CPlantBox +Reading and writing in Java, and GroIMP +Coupling platforms ``` -## Implementing fspm-TF +## Implementing encoding3d For platform developers: how a platform reads and writes the format, and how the format evolves. ```{toctree} :maxdepth: 1 +Reading the specification Implementing a platform Changing the format Implementations +The conformance files ``` ## Reference diff --git a/doc/java.md b/doc/java.md new file mode 100644 index 0000000..c3ecd74 --- /dev/null +++ b/doc/java.md @@ -0,0 +1,90 @@ +# Reading and writing in Java, and GroIMP + +`bindings/java/` holds the Java codec (Java 17, Maven; one dependency, Jackson). It reads and writes encoding3d files and +passes the conformance check. In `bindings/java/groimp/` is GroIMP's adapter. Both have tests, run in +`.github/workflows/java-codec.yml`. + +Licence: GPL-3.0-or-later, as GroIMP (the rest of encoding3d is CeCILL-C). + +## The format's own tables + +```java +import org.openalea.encoding3d.*; + +Frame frame = Codec.read(Path.of("scene.glb")); +EntitySet segments = frame.dataStructure("RootPopulation").entitySet("SubOrgan"); +double[] length = segments.column("length").doubles(); // one value per segment +String unit = segments.variables.get("length").unit; // its declared unit, e.g. "m" + +EntitySet organs = new EntitySet("Organ", 5L); +organs.set("id", Column.of(1L, 2L)).set("parent", Column.of(-1L, 1L)) + .set("edge_type", Column.of("none", "branch")).set("length", Column.of(0.12, 0.05)); +organs.declare("length", new Variable("m", "length", "extensive", "state")); +DataStructure plants = new DataStructure("plants"); +plants.entitySets.add(organs); +Frame written = new Frame(); +written.dataStructures.add(plants); +Codec.write(Path.of("plants.glb"), written); +``` + +A `Column` holds a Java array of its type (`long[]`, `double[]`, `boolean[]`, `String[]`, …; unsigned integers in +the array of their width), with a `width` of 2 or more for a fixed-size vector per row, and its missing rows +(`withMissing(...)`, `isMissing(row)`; NaN in a float column). NaN and infinities are written +with the specification's stand-ins and read back as they were. + +## Coupling with other platforms + +`org.openalea.encoding3d.coupling.Participant` is a participant of the coupling sessions encoding3d's coordinator runs +([Coupling platforms](coupling.md)): `new Participant(declaration, endpoint, links, timeout)` joins one (the declaration +as its JSON; a null endpoint, the environment's when launched by the coordinator), `nextTurn()` gives each turn's +inputs until the session stops, `sendUpdate(frame)` answers. Its folder transport needs nothing; its ZeroMQ transport +needs JeroMQ (pure Java, an optional dependency) on the class path. `ConformanceParticipant` is its program for the +conformance sessions. + +## GroIMP + +```{note} +**A proposal for GroIMP.** This adapter is a first proposal for GroIMP's team to take over; it lives here until +they adopt it, then moves into GroIMP. encoding3d does not depend on it: its requirements and CI never install +GroIMP. To build and test it by hand, see `bindings/java/groimp/README.md`. +``` + +`GroImp.toDataStructure(root, options)` writes the graph below a GroIMP root: + +| encoding3d | GroIMP | +|---|---| +| one entity set per scale | the nodes at one refinement depth (a successor or branch edge keeps the scale, a refinement edge adds one) | +| `id`, `label` | `Node.getId()`, the node's class's simple name | +| `complex` | the source of its incoming refinement edge, else its parent's complex | +| `parent`, `edge_type` | the source of its incoming successor (`successor`) or branch (`branch`) edge at the same scale | +| relations | the other edge bits (containment, mark, notifies, user edges), one relation per bit and pair of sets | +| variables | the managed fields of the model's classes (numbers, booleans, text; vecmath tuples as vectors) | + +`options.scaleNames`, `options.scaleRanks` and `options.scaleScales` give each scale its entity set's name, rank and +base scale, e.g. the vocabulary's `Organ`, rank 5. `options.labels` maps the model's class names to the vocabulary's +labels, and `options.variables` declares the fields' units and standard names (GroIMP's fields have no units). A field +that other classes of the same scale lack is missing in their rows. +`GroImp.toGraph(dataStructure, label -> new Internode(), ids)` builds the nodes back: their fields, refinement, +successor and branch edges, and edge bits. It does this for any graph, e.g. metafspm's Plant, Axis and SubOrgan +become three scales linked by refinement edges. + +The adapter depends on GroIMP's Graph module only. + +## GroIMP's plugin + +`bindings/java/groimp-plugin/` is a GroIMP 3.0 plugin. In GroIMP, **File > Export** of the 3D view to a `.glb` file +writes the project as an encoding3d frame, and the CLI's `export3d plant.glb` does the same headless: + +- the graph, by the adapter above; +- the scene's geometry as one mesh, each triangle carrying the rows of its node and of its complexes (an instance's + geometry: the node instantiating it). + +GroIMP's glTF plugin, which it requires, meshes the shapes: cylinders, boxes, spheres, cones, frusta, planes, +parallelograms, NURBS, meshes and polygons. Its folder (`target/plugin/encoding3d`) is copied into GroIMP's `plugins/`. +`test/run_in_groimp.sh` installs both plugins in GroIMP 3.0.0, grows GroIMP's `simpleRapeseed` example headless and +exports it (456,900 vertices, each with its node), then checks the frame. It then writes GroIMP's log, a frame after +each of four runs (GroIMP has no simulation clock: each frame's time and step are its run count), which a coupling +session replays. Like the adapter, the plugin is a proposal for GroIMP, run by hand +(`bindings/java/groimp-plugin/README.md`), not by encoding3d's CI. + +Not done yet: an import filter (frames into a GroIMP project), and units for GroIMP's fields, which have none. diff --git a/doc/julia.md b/doc/julia.md new file mode 100644 index 0000000..3b8adc3 --- /dev/null +++ b/doc/julia.md @@ -0,0 +1,153 @@ +# Reading and writing in Julia + +encoding3d.jl, in `bindings/julia/encoding3d/`, reads and writes encoding3d files in Julia (1.10 or later). It passes the +conformance check like the Python codec. Two VirtualPlantLab models are covered: + +- with MultiScaleTreeGraph.jl loaded, it turns an MTG into a data structure, and back. This is the multiscale graph + of PlantSimEngine and PlantGeom; +- with PlantGraphs.jl loaded, it does the same for VPL's own graphs, and with PlantGeomTurtle for their geometry. + +## Installing + +encoding3d.jl is MIT-licensed, as VirtualPlantLab (the rest of encoding3d is CeCILL-C). It is not registered yet. Add it from the repository's subfolder: + +```julia +using Pkg +Pkg.add(url = "https://github.com/openalea/encoding3d", subdir = "bindings/julia/encoding3d") +``` + +## An MTG, written and read back + +```{note} +**A proposal for MTG.jl.** This adapter is a first proposal for MTG.jl's team to take over; it lives here until +they adopt it, then moves to MTG.jl's packages. encoding3d does not depend on it: its requirements and CI never install +MTG.jl. To build and test it by hand, see `bindings/julia/README.md`. +``` + +```julia +using encoding3d, MultiScaleTreeGraph + +mtg = read_mtg("plant.mtg") +plants = from_mtg(mtg; id = "plant", + scales = Dict(1 => "Plant", 2 => "Axis", 3 => "Organ"), # entity set names, per scale + variables = Dict("Length" => Variable(unit = "cm", standard_name = "length", + kind = "extensive"))) +write_frame("plant.glb", Frame(data_structures = [plants], time = 0.0, time_unit = "d")) + +frame = read_frame("plant.glb") +same_mtg = to_mtg(frame.data_structures[1]) # the same ids, symbols, links, attributes and order +``` + +`to_mtg` also reads a graph another platform wrote, e.g. a root system simulated by metafspm or a CPlantBox plant. +Its MTG has one scale per entity set and its variables as attributes, under a `Scene` root that holds the scalars. +Two things differ from the other platform's graph, and are undone by `from_mtg`: + +- **Ids.** MTG.jl's node ids start at 1 and size its attribute store. When ids start at 0 or are sparse (CPlantBox's + segments: 2^32 + node id), the nodes are numbered 1, 2, … in id order, and each keeps its id in the attribute + `encoding3d_id`. +- **A parent in another complex**, such as a lateral root's first segment. MTG.jl takes a node's complex from its + ancestors, so the node stays under its own axis (`/`) and keeps its parent in the attributes `encoding3d_parent` and + `encoding3d_edge_type`. + +| encoding3d | MultiScaleTreeGraph | +|---|---| +| one entity set per scale ≥ 1, `rank` the scale | the nodes of that scale | +| `id`, `label` | `node_id`, `symbol` | +| `complex` | the first ancestor at a smaller scale | +| `parent`, `edge_type` | the parent, at the same scale, by `<` (successor) or `+` (branch) | +| relation `bearer` | the parent otherwise, e.g. an axis borne by an internode | +| `sibling_order` | the node's position among its parent's children | +| `mtg_index` (standard name `mtg_index`) | the node's `index` | +| variables | the attributes, declared by `variables` (MTG.jl has no units); missing where a node has no such attribute; dates and times as ISO 8601 text with their `format` | +| scalars | the attributes of the scale 0 root (`Scene`) | + +`ranks` gives a scale another rank than its number, e.g. the vocabulary's (Organ 5), and `labels` maps the +symbols to the vocabulary's labels. MTG.jl's own metadata (its classes and description, the scene node, each set's +MTG scale) is kept in the data structure's `extras["virtualplantlab"]`, so that `to_mtg` rebuilds the MTG exactly. +A float attribute a node lacks is NaN in its column, so `to_mtg` reads a NaN as no attribute. + +## A VirtualPlantLab graph + +```{note} +**A proposal for VirtualPlantLab.** This adapter is a first proposal for VirtualPlantLab's team to take over; it lives here until +they adopt it, then moves into VirtualPlantLab. encoding3d does not depend on it: its requirements and CI never install +VirtualPlantLab. To build and test it by hand, see `bindings/julia/README.md`. +``` + +VPL's own graphs (PlantGraphs.jl) have one scale, and their nodes are the model's types (`Internode`, `Leaf`, …). +With PlantGraphs loaded (Julia 1.12 or later), `from_graph` writes one entity set, `Organ` by default, with: + +- each node's type as its label, and its fields as variables; +- the graph's data as scalars; +- its parent as the nearest kept node. + +The turtle's movements (`RU`, `T`, `F`, …) are left out. PlantGraphs stores no edge type, so the edges are written as +`unknown`, and the children's order as the `sibling_order` column. A field a node's type lacks is missing. With +PlantGeomTurtle loaded too, `add_geometry!` adds the graph's turtle mesh, each triangle carrying the row of the node +that made it. + +```julia +using encoding3d, PlantGraphs, PlantGeomTurtle, PlantGeomPrimitives + +plants = from_graph(graph; id = "plant", + variables = Dict("length" => Variable(unit = "m", standard_name = "length"))) +add_geometry!(plants, graph) # the mesh of PlantGeomPrimitives.Mesh(graph), node by node (both export Mesh) +write_frame("plant.glb", Frame(data_structures = [plants])) + +static, ids = to_graph(read_frame("plant.glb").data_structures[1]; + types = Dict("Internode" => Internode, "Leaf" => Leaf)) # nodes rebuilt from their fields +graph = Graph(axiom = static, rules = rules) +``` + +## The format's own tables + +```julia +using encoding3d + +frame = read_frame("scene.glb") +frame.time, frame.time_unit, frame.step + +roots = data_structure(frame, "RootPopulation") +segments = entity_set(roots, "SubOrgan") +segments.columns["length"] # one value per segment +segments.variables["length"].unit # its declared unit, e.g. "m" +entity_ids(segments) + +for frame in read_sequence("outputs/frames/scene.fspm.json") # a sequence, frame by frame + println(frame.step) +end +``` + +Writing builds the same objects: + +```julia +segments = EntitySet("SubOrgan"; rank = 6, + columns = Dict("id" => [10, 11], "parent" => [-1, 10], + "edge_type" => ["none", "successor"], "length" => [0.01, 0.012]), + variables = Dict("length" => Variable(unit = "m", standard_name = "length", + kind = "extensive"))) +write_frame("roots.glb", Frame(data_structures = [DataStructure("roots"; entity_sets = [segments])])) +write_sequence("frames", frames) # one GLB per frame, and scene.fspm.json +``` + +A column is a `Vector` (numbers, `Bool`, `String`) or a `Matrix` (one row per entity: a fixed-size vector per +entity). NaN and infinities are written with the specification's stand-ins and read back as they were. + +## Coupling with other platforms + +encoding3d.jl takes part in coupling sessions run by encoding3d's coordinator ([Coupling platforms](coupling.md)): +`Participant(declaration)` joins one (its address and name from the environment when the coordinator launches it), +`turns(participant)` gives each turn's inputs, `send_update(participant, frame)` answers with its outputs. The folder +transport needs nothing more; ZeroMQ needs `using ZMQ` (the `encoding3dZMQExt` extension). A declaration can be built +(`Declaration`, `EntitySetSpec`, `VariableSpec`, `MappingSpec`) or read from its JSON (`declaration_from_json`). +`bindings/julia/coupling_conformance.jl` is its program for the conformance sessions. + +## Checking it + +```bash +julia --project=bindings/julia/encoding3d -e 'using Pkg; Pkg.test()' +julia --project=bindings/julia/encoding3d bindings/julia/conformance_run.jl conformance out +encoding3d compare-all out/dumps conformance && encoding3d validate out/written/*.glb +``` + +The workflow `.github/workflows/julia-codec.yml` runs both, on Julia 1.10 and the latest release. diff --git a/doc/notebooks/encoding3d_in_python.ipynb b/doc/notebooks/encoding3d_in_python.ipynb new file mode 100644 index 0000000..3a6a07c --- /dev/null +++ b/doc/notebooks/encoding3d_in_python.ipynb @@ -0,0 +1,293 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "514c77a2", + "metadata": {}, + "source": [ + "# encoding3d in Python\n", + "\n", + "This notebook reads, inspects and writes encoding3d files with `openalea.encoding3d`, the format's Python reference codec.\n", + "It uses the repository's conformance files, small reference scenes in `conformance/`, as examples.\n", + "\n", + "- a **frame** is one time step of a scene: one `.glb` file;\n", + "- it holds **data structures** (a plant population, a soil grid...), each made of **entity sets** (one per scale: Plant,\n", + " Axis, SubOrgan...), which are tables of numpy columns;\n", + "- every variable column comes with a **declaration**: unit, standard name, kind, role." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "438f5812", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import encoding3d\n", + "\n", + "CONFORMANCE = os.path.join(\"..\", \"..\", \"conformance\") # the repository's reference scenes" + ] + }, + { + "cell_type": "markdown", + "id": "31adf4b2", + "metadata": {}, + "source": [ + "## Reading a frame\n", + "\n", + "`encoding3d.read` gives a `Frame`: its time, and its data structures." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dd1105d", + "metadata": {}, + "outputs": [], + "source": [ + "frame = encoding3d.read(os.path.join(CONFORMANCE, \"soil_and_roots.glb\"))\n", + "print(frame.time, frame.time_unit, frame.step)\n", + "for data_structure in frame.data_structures:\n", + " print(data_structure.id, data_structure.kind, [entity_set.name for entity_set in data_structure.entity_sets])" + ] + }, + { + "cell_type": "markdown", + "id": "024ae15f", + "metadata": {}, + "source": [ + "Each entity set is a table: one row per entity, one numpy column per variable. Pandas shows it directly (vector\n", + "columns, one per row, aside)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cded09d7", + "metadata": {}, + "outputs": [], + "source": [ + "roots = frame.data_structure(\"root_system\")\n", + "segments = roots.entity_set(\"SubOrgan\")\n", + "pd.DataFrame({name: values for name, values in segments.columns.items() if np.ndim(values) == 1})" + ] + }, + { + "cell_type": "markdown", + "id": "1562b906", + "metadata": {}, + "source": [ + "The columns `id`, `label`, `complex`, `parent` and `edge_type` are the multiscale graph: `complex` is the entity\n", + "containing a segment one scale up (its axis), `parent` the segment it follows, and `edge_type` how (in line, or as a\n", + "branch)." + ] + }, + { + "cell_type": "markdown", + "id": "dcfbaf6c", + "metadata": {}, + "source": [ + "## What the variables mean\n", + "\n", + "Each variable is declared by its writer: its unit (a UCUM code), its standard name in the shared vocabulary, how it\n", + "combines (its kind) and its role." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e8d75be", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame({name: vars(variable) for name, variable in segments.variables.items()}).T[\n", + " [\"unit\", \"standard_name\", \"kind\", \"role\", \"description\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2b30a2d", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.encoding3d import vocabulary\n", + "\n", + "vocabulary.standard_names()[\"water_potential\"][\"definition\"], vocabulary.standard_names()[\"water_potential\"][\"canonical_unit\"]" + ] + }, + { + "cell_type": "markdown", + "id": "9a180b10", + "metadata": {}, + "source": [ + "## The soil, and which cells each root segment lies in\n", + "\n", + "A grid is a data structure too, and a **mapping** says which soil cell each segment is in, with a weight." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "970cdb4d", + "metadata": {}, + "outputs": [], + "source": [ + "soil = frame.data_structure(\"soil\")\n", + "print(soil.grid)\n", + "mapping = frame.mappings[0]\n", + "pd.DataFrame(mapping.columns)" + ] + }, + { + "cell_type": "markdown", + "id": "34d6e326", + "metadata": {}, + "source": [ + "## Geometry\n", + "\n", + "A mesh's vertices carry the row of their entity at every scale they represent: here each triangle knows its segment,\n", + "its axis and its plant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7a4bb1c", + "metadata": {}, + "outputs": [], + "source": [ + "mesh = roots.meshes[0]\n", + "print(len(mesh.positions) // 3, \"triangles; rows by scale:\", list(mesh.feature_rows))\n", + "first_triangle_segment = segments.columns[\"id\"][mesh.feature_rows[\"SubOrgan\"][0]]\n", + "first_triangle_segment" + ] + }, + { + "cell_type": "markdown", + "id": "a5089917", + "metadata": {}, + "source": [ + "## A sequence of frames\n", + "\n", + "A simulation writes a frame every so many steps, with an index, `scene.fspm.json`. Entities keep their ids from\n", + "frame to frame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c18ea1c8", + "metadata": {}, + "outputs": [], + "source": [ + "index = os.path.join(CONFORMANCE, \"growing_sequence\", \"scene.fspm.json\")\n", + "rows = []\n", + "for growing in encoding3d.read_sequence(index):\n", + " elements = growing.data_structures[0].entity_set(\"SubOrgan\")\n", + " rows.append({\"time (d)\": growing.time, \"segments\": elements.count, \"root length (m)\": elements.columns[\"length\"].sum()})\n", + "pd.DataFrame(rows)" + ] + }, + { + "cell_type": "markdown", + "id": "851aa18e", + "metadata": {}, + "source": [ + "## Writing and checking a frame\n", + "\n", + "Writing builds the same objects. The validator checks a file against the specification: ids, links, units against\n", + "the vocabulary, tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f88d3c6b", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.encoding3d.validate import validate_bytes\n", + "\n", + "plant = encoding3d.EntitySet(\"Plant\", rank=1, columns=dict(id=np.array([1]), label=np.array([\"Wheat\"], dtype=object)))\n", + "segments = encoding3d.EntitySet(\"SubOrgan\", rank=6, columns=dict(\n", + " id=np.array([10, 11, 12]), label=np.array([\"RootSegment\"] * 3, dtype=object), complex=np.array([1, 1, 1]),\n", + " parent=np.array([-1, 10, 11]), edge_type=np.array([\"none\", \"successor\", \"successor\"], dtype=object),\n", + " length=np.array([0.01, 0.01, 0.004])),\n", + " variables={\"length\": encoding3d.Variable(unit=\"m\", standard_name=\"length\", kind=\"extensive\")})\n", + "written = encoding3d.Frame([encoding3d.DataStructure(\"roots\", entity_sets=[plant, segments])], time=0., time_unit=\"d\")\n", + "data = encoding3d.to_bytes(written)\n", + "print(len(data), \"bytes;\", validate_bytes(data))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b4ffd9e", + "metadata": {}, + "outputs": [], + "source": [ + "again = encoding3d.from_bytes(data)\n", + "again.data_structure(\"roots\").entity_set(\"SubOrgan\").columns[\"length\"]" + ] + }, + { + "cell_type": "markdown", + "id": "077e016f", + "metadata": {}, + "source": [ + "A wrong unit is caught: a length in seconds does not convert to the standard name's metres." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e9e76d8", + "metadata": {}, + "outputs": [], + "source": [ + "segments.variables[\"length\"] = encoding3d.Variable(unit=\"s\", standard_name=\"length\", kind=\"extensive\")\n", + "print(validate_bytes(encoding3d.to_bytes(written)))" + ] + }, + { + "cell_type": "markdown", + "id": "1ebb1400", + "metadata": {}, + "source": [ + "## For other viewers: display copies\n", + "\n", + "Viewers that do not read the tables (Blender, a web viewer, ParaView without encoding3d's plugin) colour shapes by\n", + "per-vertex attributes. A display copy adds one, from the tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff9605db", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.encoding3d.display import add_display\n", + "\n", + "shown = add_display(encoding3d.read(os.path.join(CONFORMANCE, \"root_system.glb\")), [\"water_potential\"])\n", + "mesh = shown.data_structures[0].meshes[0]\n", + "mesh.display[\"water_potential\"][:6], mesh.display_sources" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/overview.md b/doc/overview.md similarity index 94% rename from docs/overview.md rename to doc/overview.md index b575ec9..068c1dd 100644 --- a/docs/overview.md +++ b/doc/overview.md @@ -1,10 +1,10 @@ -# fspm-TF in five minutes +# encoding3d in five minutes -This page explains, in plain words, what an fspm-TF file contains. No programming is needed to follow it. +This page explains, in plain words, what an encoding3d file contains. No programming is needed to follow it. ## A scene, one time step at a time -A simulation runs a **scene**: plants growing in an environment (soil, light, weather), step after step. fspm-TF saves +A simulation runs a **scene**: plants growing in an environment (soil, light, weather), step after step. encoding3d saves the scene as it is at one step: a **frame**. A simulation saves a frame every so many steps, so a run gives a **sequence** of frames: diff --git a/docs/platforms.md b/doc/platforms.md similarity index 61% rename from docs/platforms.md rename to doc/platforms.md index ae671e0..395ac86 100644 --- a/docs/platforms.md +++ b/doc/platforms.md @@ -1,6 +1,6 @@ # Implementing a platform -A platform reads and writes fspm-TF through two pieces of code, which live in different places. +A platform reads and writes encoding3d through two pieces of code, which live in different places. | | What it does | Depends on | Lives in | |---|---|---|---| @@ -26,18 +26,18 @@ What it has to handle: This is a few hundred lines in any language. Existing libraries help: -- **C++:** cesium-native reads and writes `EXT_structural_metadata`. -- **Java:** JglTF has its JSON classes. -- **Julia:** a small GLB reader is enough; `GLTF.jl` handles the JSON. +- **C++:** done, `bindings/cpp/` ([C++ and CPlantBox](cpp.md)): hand-written, with nlohmann_json. +- **Java:** done, `bindings/java/` ([Java and GroIMP](java.md)): hand-written, with Jackson. +- **Julia:** done, `bindings/julia/encoding3d/` ([encoding3d.jl](julia.md)): a hand-written GLB container and JSON.jl. Each codec goes in its own folder, with its usual build files and tests: ``` bindings/ - python/ the reference (openalea.fspm_tf) + python/ the reference (openalea.encoding3d) cpp/ CMakeLists.txt, include/, src/, test/ java/ pom.xml or build.gradle, src/main/, src/test/ - julia/FspmTF/ Project.toml, src/, test/ + julia/encoding3d/ Project.toml, src/, test/ ``` A codec is released by its team on its own channel (conda, Maven, the Julia registry), under its own tag (e.g. @@ -55,7 +55,7 @@ What the platform needs to rebuild its own model exactly, beyond the shared data structure's `extras` (under the platform's name) and in columns of role `setting`. Other readers keep them without interpreting them. -**Example:** metafspm's adapter, `openalea.metafspm.io.fspm_tf`, rebuilds its multiscale graphs bit for bit from its +**Example:** metafspm's adapter, `openalea.metafspm.io.encoding3d`, rebuilds its multiscale graphs bit for bit from its own files, and a simulation continues from them exactly as from the original. It reads its standard names from its column of the vocabulary. @@ -73,15 +73,15 @@ mapping, an adaptive grid, a parameter set, and a growing sequence. Each GLB com - Floats are compared exactly. - NaN is `null`; infinities are the strings `"Infinity"` and `"-Infinity"`. - Positions are the float32 values. -- **Writing:** what it writes, for the same scenes, passes `fspm-tf validate` with no error. It also passes Khronos' +- **Writing:** what it writes, for the same scenes, passes `encoding3d validate` with no error. It also passes Khronos' glTF-Validator with no error, so that every viewer opens it. `conformance/README.md` gives the layout of the expected JSON. A codec dumps what it reads from each conformance GLB in that layout (`x.json` for `x.glb`), and writes the same scenes back as GLB files. Then: ```bash -fspm-tf compare-all dumps/ conformance/ # reading: every dump equals its expected values, or the differences -fspm-tf validate written/*.glb # writing +encoding3d compare-all dumps/ conformance/ # reading: every dump equals its expected values, or the differences +encoding3d validate written/*.glb # writing ``` In GitHub Actions, the composite action `.github/actions/conformance` runs both, plus Khronos' glTF-Validator, @@ -99,3 +99,30 @@ after the codec's own steps: The Python codec does exactly this (`bindings/python/conformance_run.py`, the `python-codec-conformance` job). A codec that passes is listed in [Implementations](implementations.md) with the version it passes. + +## Taking part in a coupling session + +To couple its models with other platforms' (specification §11), a codec adds a participant's side; +the coordinator is encoding3d's own (`encoding3d couple`), so no platform implements it. A participant: + +1. reads and writes **partial frames**: `FSPM_scene.update` (its sender, and a property table of the removed ids per + entity set), tables holding some rows and columns only; +2. sends its **declaration** and receives the welcome (its id offset, its inputs' providers), as JSON headers + (`spec/schema/coupling.schema.json`); +3. answers each **turn** (a partial frame of its inputs and the entities added or removed since its last turn) with + an **update** (its outputs and its own topology changes), until `stop`; +4. speaks both transports: a folder of numbered JSON headers and GLB payloads (files only), and ZeroMQ (a DEALER + socket named after the participant; one or two message frames). + +The four codecs have a participant: Python (`openalea.encoding3d.coupling.participant`), Julia (`Participant`, `turns`, +`send_update`), C++ (`encoding3d/coupling.hpp`) and Java (`org.openalea.encoding3d.coupling`). A new one proves itself on the +conformance sessions: a small program run as `COMMAND SESSION OUT` that joins with the session's declaration, dumps +each turn it reads and sends the session's updates, as the four codecs' programs do +(`coupling/conformance.py`, `bindings/julia/coupling_conformance.jl`, `bindings/cpp/tools/coupling_conformance.cpp`, +`ConformanceParticipant.java`): + +```bash +encoding3d couple-conformance conformance/coupling --participant "" +``` + +An adapter then maps the turns to the platform's model, as metafspm's `RemoteComponent` does. diff --git a/docs/python.md b/doc/python.md similarity index 55% rename from docs/python.md rename to doc/python.md index c4d9190..bf42ac3 100644 --- a/docs/python.md +++ b/doc/python.md @@ -5,24 +5,25 @@ A platform writes and reads its own simulations through its adapter. In openalea.metafspm: ```python -from openalea.metafspm.io.fspm_tf import export_scene, import_frame +from openalea.metafspm.io.encoding3d import export_scene, import_frame export_scene("scene.glb", scene) # the scene now data_structures = import_frame("scene.glb") # metafspm's DataStructures, rebuilt bit for bit -scene = Scene(planting, ..., output_dirpath="outputs", frame_period=24) # a frame every 24 steps +recorder = SceneRecorder("outputs", frame_period=24) # openalea.metafspm.scene.recorder: a frame every 24 steps +scene = Scene(planting, ..., recorder=recorder) ``` See [metafspm's documentation](https://github.com/openalea/metafspm) for the options. ## The format's own tables -`openalea.fspm_tf` reads any fspm-TF file, whichever platform wrote it, into plain Python objects holding numpy arrays. +`openalea.encoding3d` reads any encoding3d file, whichever platform wrote it, into plain Python objects holding numpy arrays. ```python -from openalea import fspm_tf +from openalea import encoding3d -frame = fspm_tf.read("scene.glb") +frame = encoding3d.read("scene.glb") print(frame.time, frame.time_unit, frame.step) roots = frame.data_structure("RootPopulation") @@ -51,29 +52,45 @@ Writing builds the same objects: ```python import numpy as np -from openalea import fspm_tf +from openalea import encoding3d -segments = fspm_tf.EntitySet("SubOrgan", rank=6, columns=dict( +segments = encoding3d.EntitySet("SubOrgan", rank=6, columns=dict( id=np.array([1, 2, 3]), label=np.array(["RootSegment"] * 3, dtype=object), parent=np.array([-1, 1, 2]), edge_type=np.array(["none", "successor", "successor"], dtype=object), length=np.array([0.01, 0.01, 0.004])), - variables={"length": fspm_tf.Variable(unit="m", standard_name="length", kind="extensive")}) -plant = fspm_tf.DataStructure("roots", entity_sets=[segments]) -fspm_tf.write("roots.glb", fspm_tf.Frame([plant], time=0., time_unit="d")) + variables={"length": encoding3d.Variable(unit="m", standard_name="length", kind="extensive")}) +plant = encoding3d.DataStructure("roots", entity_sets=[segments]) +encoding3d.write("roots.glb", encoding3d.Frame([plant], time=0., time_unit="d")) ``` A sequence is a folder of frames with its index: ```python -fspm_tf.write_sequence("outputs/frames", frames) # frame_000000.glb, ..., scene.fspm.json -for frame in fspm_tf.read_sequence("outputs/frames/scene.fspm.json"): +encoding3d.write_sequence("outputs/frames", frames) # frame_000000.glb, ..., scene.fspm.json +for frame in encoding3d.read_sequence("outputs/frames/scene.fspm.json"): ... ``` +## Large frames: reading lazily + +`encoding3d.read(path, lazy=True)` maps the file in memory and parses only its JSON: names, declarations, ranks and +counts are there at once, and each column (and each mesh) is decoded when first read. A reader using a few columns of +a large frame decodes only those (a GroIMP frame of 7.4 MB with a 456,000-vertex mesh: 17 ms read whole, 0.8 ms for +one column). The frame holds the file until it is closed: + +```python +with encoding3d.read("plant.glb", lazy=True) as frame: + segments = frame.data_structure("plant").entity_set("SubOrgan") + lengths = segments.columns["length"] # decoded now; the other columns are not +``` + +The ParaView reader and a coupling session's replay read frames this way (a replay decodes only the columns some +link uses). + ## The vocabulary ```python -from openalea.fspm_tf import vocabulary +from openalea.encoding3d import vocabulary vocabulary.standard_names()["water_potential"] # its unit, kind, definition, platform names vocabulary.platform_names("metafspm") # {"water_potential": "water_potential", "x1": "proximal_end_x", ...} diff --git a/doc/reading_the_specification.md b/doc/reading_the_specification.md new file mode 100644 index 0000000..c0919ce --- /dev/null +++ b/doc/reading_the_specification.md @@ -0,0 +1,136 @@ +# Reading the specification + +This page is for a platform's developers, before they read the specification itself. It explains: + +- what the `spec/` folder holds; +- in which order to read it; +- how a file is built; +- what each word means; +- how a codec proves that it follows the format. + +## What is agreed, and where + +Everything the platforms agree on is in the repository's `spec/` folder, together with the conformance files: + +| File | What it fixes | How it is checked | +|---|---|---| +| `spec/specification.md` | the rules, in words (MUST, SHOULD, MAY as in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)) | by the validator, `encoding3d validate` | +| `spec/schema/fspm_scene.schema.json` | the shape of the `FSPM_scene` JSON (its fields and their types) | by any JSON Schema validator | +| `spec/schema/index.schema.json` | the shape of a sequence's index, `scene.fspm.json` | by any JSON Schema validator | +| `spec/schema/coupling.schema.json` | the shape of a coupling session's messages (their JSON headers, §11) | by any JSON Schema validator | +| `spec/schema/views.schema.json` | the shape of a sequence's optional views, `views.json` (how to look at it) | by any JSON Schema validator; names against a frame by `openalea.encoding3d.views.check_views` | +| `spec/vocabulary/standard_names.csv` | the shared quantities: canonical unit, kind, definition, and each platform's names for them | by the validator (units, kinds) | +| `spec/vocabulary/labels.csv` | the entity labels, with their scale and Plant Ontology term | by the validator (warnings) | +| `spec/vocabulary/enums.json` | the allowed values: edge types, kinds, roles, base scales, formats… | by the schemas and the validator | +| `conformance/` | the specification as test cases: reference files and what a reader must find in them | `encoding3d compare-all` | + +When they disagree, `specification.md` prevails, then the schemas, then the conformance files. A disagreement is a +defect of the lower one. A change of the format is a pull request to `spec/`, with its conformance scene +([Changing the format](governance.md)). + +## A reading path + +1. **§1, the data model**, all of it: frames, data structures, entity sets, relations, mappings, meshes, variables. + Everything else describes how these are written. +2. **§4.2, the reserved columns.** `id`, `label`, `complex`, `parent`, `edge_type` and `sibling_order` carry the plant's + topology (an MTG's decomposition `/`, successor `<` and branch `+`). +3. **§5 and §6, variables and the vocabulary**: units, kinds, roles, standard names. This is what lets two platforms + recognise the same quantity. +4. **§2, §3, §4.1, §7, the encoding.** These come with the example below; a codec's writer needs them in full. +5. **§8 and §9, sequences and conformance.** +6. **§11, coupling**, for a platform whose models run with other platforms' in a session: partial frames, the + messages, the shared structure, the transports. Its conformance sessions are in §9. + +## The three layers of a file + +A frame is a standard glTF 2.0 binary file (`.glb`): a JSON part and a binary part. Any glTF viewer opens it. Three +layers hold the scene: + +| Layer | Defined by | Holds | +|---|---|---| +| Property tables | the glTF extension [`EXT_structural_metadata`](https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata) | every table: one column per variable, with its values in the binary part | +| Feature IDs | the glTF extension [`EXT_mesh_features`](https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features) | for each vertex of a mesh, the row of its entity in one or several tables | +| `FSPM_scene` | encoding3d (`spec/schema/fspm_scene.schema.json`) | what glTF cannot say: the time, the data structures, which table is which scale, relations, grids, mappings, the variables' units and standard names | + +Here is the JSON part of `conformance/root_system.glb`, abridged: + +```text +"extensions": { + "FSPM_scene": { + "version": "0.3", "vocabulary": "0.2", + "frame": {"time": 0.0, "timeUnit": "d", "step": 0}, + "dataStructures": [{ + "id": "root_system", "kind": "graph", "origin": [1, 0, 0, 0, ...], a 4×4 matrix, column by column + "entitySets": [{"name": "Plant", "rank": 1, "propertyTable": 0}, one table per scale + {"name": "Axis", "rank": 2, "propertyTable": 1}, + {"name": "SubOrgan", "rank": 6, "propertyTable": 2}], + "scalars": 3, a one-row table for the whole plant + "meshes": [{"name": "roots", "mesh": 0}]}], + "variables": {"root_system__SubOrgan": { + "length": {"unit": "m", "standardName": "length", "kind": "extensive", "role": "state"}, ...}}}, + "EXT_structural_metadata": { + "schema": {"classes": {"root_system__SubOrgan": {"properties": { + "id": {"type": "SCALAR", "componentType": "INT64", "required": true}, + "label": {"type": "ENUM", "enumType": "root_system__SubOrgan__label"}, + "complex": {"type": "SCALAR", "componentType": "INT64", "noData": -1}, the axis it belongs to + "parent": {"type": "SCALAR", "componentType": "INT64", "noData": -1}, the segment it follows + "edge_type": {"type": "ENUM", "enumType": "FSPM_edge_type"}, none, successor, branch, unknown + "length": {"type": "SCALAR", "componentType": "FLOAT64"}, ...}}}}, + "propertyTables": [..., {"class": "root_system__SubOrgan", "count": 10, + "properties": {"id": {"values": 9}, "length": {"values": 14}, ...}}]}} buffer views +"meshes": [{"primitives": [{"attributes": {"POSITION": 0, "_FEATURE_ID_0": 1, "_FEATURE_ID_1": 2, "_FEATURE_ID_2": 3}, + "extensions": {"EXT_mesh_features": {"featureIds": [ + {"attribute": 0, "propertyTable": 2, "label": "SubOrgan"}, each vertex's segment, + {"attribute": 1, "propertyTable": 1, "label": "Axis"}, its axis, + {"attribute": 2, "propertyTable": 0, "label": "Plant"}]}}}]}] and its plant +``` + +To read the segments' lengths, a reader: + +1. finds the `SubOrgan` entity set in `FSPM_scene`, and its `propertyTable` (2); +2. finds that table's `length` property and its buffer view (14); +3. reads `count` (10) FLOAT64 values from the binary part; +4. finds their unit (`m`) and standard name in `variables`. + +`encoding3d info file.glb` lists what a frame holds: its data structures, their tables and their variables. + +## Words + +| Word | Meaning | +|---|---| +| **frame** | the scene at one time step: one `.glb` file | +| **sequence** | a folder of frames and their index, `scene.fspm.json` | +| **data structure** | one plant population's graph, or one environment model's grid (metafspm's word). Not glTF's *node*: each data structure has a glTF node, which only places it for viewers. | +| **entity set** | a table of entities of one kind: the entities of one scale of a graph, or a grid's cells | +| **scale** | a level of description: plant, axis, organ, segment, cell… The vocabulary lists base scales (`enums.json`); an entity set may declare its own (`scale`). | +| **rank** | an entity set's order among the scales, coarser first. It SHOULD be the base scale's position: Plant 1, Axis 2, Organ 5, SubOrgan 6. | +| **entity** | one row of an entity set: a plant, a root axis, a segment, a cell | +| **relation** | an edge table between two entity sets: a cell network, a branch's bearer | +| **mapping** | weights between the entities of two data structures, e.g. root segments and soil voxels | +| **class** | `EXT_structural_metadata`'s word for a table's type: its columns. encoding3d names them `__`. | +| **variable** | a column that is not reserved, with its declaration: unit, standard name, kind, role | +| **standard name** | the vocabulary's name for a quantity, the same on every platform (`length`, `water_potential`) | +| **codec** | a language's reading and writing of the format (`bindings//`) | +| **adapter** | a platform's model to and from the codec's tables | + +## How a codec proves it follows the format + +`conformance/` holds reference scenes, each a GLB with what a reader must find in it (`*.expected.json`, in the layout +of [the conformance files](conformance.md)). A codec: + +1. reads each GLB and writes what it read as JSON, in that layout; +2. writes the same scenes back as GLB files. + +`encoding3d compare-all` then compares its JSON with the expected values, exactly. `encoding3d validate` and Khronos' +[glTF-Validator](https://github.com/KhronosGroup/glTF-Validator) check what it wrote. In GitHub Actions, the +composite action `.github/actions/conformance` runs these checks. The Python, Julia, C++ and Java codecs all pass them +([Implementations](implementations.md)). + +## Standards it builds on + +- glTF 2.0: +- `EXT_structural_metadata` and `EXT_mesh_features`: the links above (Cesium's 3D Tiles vendor extensions) +- UCUM, the units: +- I-ADOPT, the decomposition of the vocabulary's names: +- the Plant Ontology, the labels' terms: +- RFC 2119, MUST and SHOULD: diff --git a/docs/specification.md b/doc/specification.md similarity index 100% rename from docs/specification.md rename to doc/specification.md diff --git a/doc/tutorials.md b/doc/tutorials.md new file mode 100644 index 0000000..5f0e294 --- /dev/null +++ b/doc/tutorials.md @@ -0,0 +1,50 @@ +# Tutorials + +The tutorials are in the repository's `tutorials/` folder, each with its code and a `run.sh` running it from start to +end. This page gathers their explanations. + +```{include} ../tutorials/README.md +:start-line: 2 +``` + +## Tutorial 1: encoding3d on its own + +```{include} ../tutorials/01_encoding3d_alone/README.md +:start-line: 2 +``` + +## Tutorial 2a: metafspm + +```{include} ../tutorials/02a_metafspm/README.md +:start-line: 2 +``` + +## Tutorial 2b: VirtualPlantLab + +```{include} ../tutorials/02b_virtualplantlab/README.md +:start-line: 2 +``` + +## Tutorial 2c: CPlantBox + +```{include} ../tutorials/02c_cplantbox/README.md +:start-line: 2 +``` + +## Tutorial 2d: GroIMP + +```{include} ../tutorials/02d_groimp/README.md +:start-line: 2 +``` + +## Tutorial 3: metafspm and VirtualPlantLab, one plant + +```{include} ../tutorials/03_metafspm_and_virtualplantlab/README.md +:start-line: 2 +``` + +## Tutorial 4: four platforms, one plant + +```{include} ../tutorials/04_four_platforms/README.md +:start-line: 2 +``` diff --git a/doc/using.md b/doc/using.md new file mode 100644 index 0000000..3c021d2 --- /dev/null +++ b/doc/using.md @@ -0,0 +1,202 @@ +# Viewing and checking files + +## Installing + +The Python package `openalea.encoding3d` brings the command line, the validator and the reader used by ParaView's +plugin. Until it is published on the `openalea3` channel, install it from the sources: + +```bash +git clone https://github.com/openalea/encoding3d.git +cd encoding3d +mamba env create -f conda/environment.yml +mamba activate encoding3d +``` + +To use it in another environment, e.g. a model's, install its dependencies there (`mamba install numpy jsonschema`), +then `pip install -e path/to/encoding3d`. + +## In ParaView + +encoding3d comes with a ParaView plugin, one file to load: `paraview/plugin_encoding3d.py` (the *encoding3d reader* +and the *encoding3d Select at scale* filter; no macros). It works with ParaView 5.11 and later, and has been checked +with 6.2.0. `paraview/README.md` is the step-by-step how-to, with ParaView 6.2's menu names. + +1. In ParaView, open **Tools → Manage Plugins...**, click **Load New ...**, choose `plugin_encoding3d.py`, and tick + **Auto Load** to keep it loaded. +2. **File → Open...** a frame (`.glb`), or a sequence's `scene.fspm.json` to play its frames as time steps, choose + **encoding3d reader** if ParaView asks, and click **Apply**. + +The plugin needs nothing installed in ParaView: it reads files with the code of the encoding3d folder it comes from. + +What you see: + +- **One block per plant population or environment model** (without a views file, below). Each holds its meshes, its + root segments as lines (apply a *Tube* filter on `radius` to give them volume), and its grid. +- **Every variable of every scale is a colouring choice**, named `.`: `SubOrgan.water_potential`, + `Axis.branching_order`, `Voxel.soil_water_potential`. Values are at full precision. Where a triangle has no entity + at a scale, the value is empty (NaN). +- **Labels and other text become integer codes**, `.label_code`; the names behind the codes are in the + block's field data. +- **Every cell knows its plant, axis, organ ...**: besides its own entity's `.id`, a cell has the ids of the + entities containing it (`Axis.id`, `Plant.id` on a root segment), which selection by scale uses (below). +- **Show Settings**, a reader option, adds the columns a platform keeps for its own bookkeeping, and the tables' + links (`complex`, `parent`, `edge_type`). They are hidden by default. + +```{image} _static/images/root_system.png +:alt: A small root system coloured by water potential +:width: 260px +``` + +*The conformance root system, coloured by `SubOrgan.water_potential`.* + +In a `pvpython` script: + +```python +from paraview import simple +simple.LoadPlugin("encoding3d/paraview/plugin_encoding3d.py", ns=globals()) +roots = encoding3dreader(FileName="outputs/frames/scene.fspm.json") +roots.UpdatePipeline(roots.TimestepValues[-1]) # the last frame +``` + +### Views: layers ready to look at + +A sequence may come with a `views.json` next to its `scene.fspm.json` (metafspm writes one with +`SceneRecorder(frame_views="default")`). It names **configurations**, each a list of **layers**: a part of a data +structure (a mesh, root segments, a grid), the entities it keeps (e.g. the root segments, or the cells where the +soil is wet), the variable it is coloured by, its colour map and range, its opacity, whether it is shown at first, +and tubes for segments. The specification describes the file (§8.1). + +With a views file, the reader gives **one block per layer** of the chosen configuration instead of one per data +structure: each block holds only the layer's entities, already coloured by its variable, a segments layer with a +`tube` as tubes. Hidden layers are there too, so showing one is instant, and a layer is empty in the frames that lack +what it shows (a shoot not yet drawn). Two reader options: + +- **Views:** the views file (empty: the `views.json` next to the opened file; without one, the blocks are as above). +- **Configuration:** a drop-down of the file's configurations (empty: the file's active one). + +The plugin draws the layers itself, with no macro: when you click **Apply** (or change the Configuration and Apply), +each layer is drawn as its block with its own colour variable, colour map, range, log scale or solid colour, +opacity and visibility, with its own colour legend. The soil can be translucent while the roots and the shoot have +their own colour maps. + +- **View → MultiBlock Inspector** lists the layers by name: tick one to show or hide it, click one to change its + colouring (*Block Coloring*: the variable, the colour map, *Rescale to Data Range*, *Use Separate Color Map*) or its + *Opacity*. In the view, right-click a layer for *Hide Block*, *Show Only Block*, *Set Block Opacity*. +- **Save Views** (a button of the reader) writes the layers' display back into `views.json`, over the configuration + shown; with a name in **Save As** (then Apply), as a new configuration, made the active one. + +A colour map's preset name is not read back from ParaView, so a saved layer keeps its `colormap`; layers coloured by +the same variable share one colour map (*Use Separate Color Map* gives a layer its own). One representation draws +all layers: a layer's own `representation` is kept in the file, not drawn apart. + +```{image} _static/images/views_layers.png +:alt: A wheat seedling and its root system in a translucent soil grid, each layer with its own colour map +:width: 420px +``` + +*metafspm's use case 1 (day 8), as the plugin draws it on Apply, no macro: the soil grid translucent +(`pressure_head`, Viridis), the roots (`type`, Inferno) and the shoot (`organ_rank`, Cool to Warm), a configuration +giving each its own colour map.* + +In a script, the same views without ParaView: `frame_to_vtk(frame, views="views.json", configuration="default")` from +`paraview/plugin_encoding3d.py`, and `openalea.encoding3d.views` to read, check and write views: + +```python +from openalea import encoding3d +from openalea.encoding3d import views + +document = views.read_views("outputs/frames/views.json") +report = views.check_views(document, encoding3d.read("outputs/frames/frame_000000.glb")) # every name in the frame +print(report) +``` + +### Selecting a plant, an axis, an organ + +The plugin has a filter that selects by scale: pick an element, and get its whole plant, axis, organ or segment, +with everything it contains at finer scales, in every layer, hidden ones included (a segment's section drawings, a +hidden layer by default, come with the selection). + +1. Pick cells in the view (**Select Cells On (s)**, or `s`), on the reader's item or a selection shown before. The + item picked on becomes the active one in the Pipeline Browser. +2. **Filters → encoding3d → encoding3d Select at scale**, then **Apply**. The filter takes ParaView's current selection + when it is made, as *Extract Selection* does (its **Selection** property: **Copy Active Selection** takes the + selection made since, on the same item). +3. Choose the **Scale** in its Properties: a drop-down of the frames' scales, coarser first (`Plant`, `Axis`, + `Organ`, `SubOrgan`, … `Compartment`), `Plant` at first. **Apply** grows the picked cells again. +4. Play the time steps: the picked entities are kept, so the plant or organ is followed as it grows. + +The filter's output is one block per layer, named and filled like the reader's, each holding the selection's part +of the layer (empty when it has none), and it is drawn as the layers (the same colour maps), the hidden ones shown. +A scale finer than the picked element (a Compartment of a leaf drawn without sections) is reported in the Output +Messages, with the scales the picked cells have. + +The filter's buttons show it: + +- **Isolate**: the active view shows only the selection, the other items hidden. +- **Show In Panel**: a new view beside the active one shows the selection alone, its camera fit to it, the first view + untouched. +- **Put Back**: one step back, the hidden items shown again or the panel closed. Then delete the filter to drop the + selection. + +To narrow a selection (an axis of the plant shown), pick on it, apply another encoding3d Select at scale to it, and +click its Isolate: it is shown over the first, and its Put Back returns to the plant. The buttons work with +ParaView's built-in server (the default); which cells are selected is `openalea.encoding3d.views.picked_entities` +and `selection_mask`, tested without ParaView. + +`paraview/check_in_paraview.py` checks all this inside ParaView. `pvpython paraview/check_in_paraview.py +conformance` runs on the conformance files (the binding tests run it when `ENCODING3D_PVPYTHON` names a `pvpython`); +`pvbatch paraview/check_in_paraview.py --design /frames/scene.fspm.json ` is a check by hand, on a +copy of a sequence with its `views.json`: it shows the layers as the GUI's Apply does, switches configurations, +saves the views back, selects at several scales with the filter (switching its Scale, stepping in time), shows the +selections isolated, in a panel and one over another, puts them back, and saves a screenshot of each step in the +folder, to look at. + +## In other viewers + +Any glTF viewer (Blender, a web viewer, ParaView without the plugin) shows the geometry, and ignores the tables. +To colour shapes there by a variable, add a **display copy**: a per-vertex copy of the variable, in single precision, +which viewers read as a colouring attribute. + +```bash +encoding3d display scene.glb water_potential # adds _WATER_POTENTIAL to the meshes +encoding3d display scene.glb --remove # removes the display copies +``` + +Display copies are optional and flagged as such. Readers never take them for the variable itself. + +## Checking a file + +```bash +encoding3d validate scene.glb # one frame +encoding3d validate outputs/frames/scene.fspm.json # a sequence: every frame, and the rules between frames +``` + +The validator reports **errors** (the file does not follow the specification) and **warnings** (it does, but something +is likely missing, such as a variable without a declared unit). It checks, among other things, that: + +- the ids are unique and, in a sequence, never reused; +- every `complex` and `parent` points to an existing entity, and the parents form no cycle; +- the units are valid UCUM codes that convert to their standard names' units; +- the grids and the mappings agree with their entities; +- the binary tables have the length their types imply. + +The command exits with 1 when a file has errors, so it can run in a CI. + +## Looking inside a file + +```bash +encoding3d info scene.glb +``` + +For example, for a frame of a metafspm scene with roots in a soil grid: + +``` +scene.glb: time 86400.0 s, step 24, writer {'platform': 'openalea.metafspm'} + RootPopulation (graph, MPGDataStructure) + Plant rank 1: 3 entities, variables x, y, z, ... + SubOrgan rank 6: 412 entities, variables length, radius, struct_mass, ... + mesh triangles: 9888 vertices, entity sets SubOrgan, Axis, Plant + Soil (regular_grid, ArrayDataStructure) + cell: 1024 entities, variables water_content, soil_temperature, ... + mapping RootPopulation_to_Soil: RootPopulation/SubOrgan -> Soil/cell, 530 pairs +``` diff --git a/docs/vocabulary.md b/doc/vocabulary.md similarity index 100% rename from docs/vocabulary.md rename to doc/vocabulary.md diff --git a/docs/api.md b/docs/api.md deleted file mode 100644 index fc35111..0000000 --- a/docs/api.md +++ /dev/null @@ -1,62 +0,0 @@ -# Python API - -`openalea.fspm_tf` is the reference codec: reading and writing frames and sequences, the validator, display copies, -the vocabulary and units. - -## Reading and writing - -```{eval-rst} -.. automodule:: openalea.fspm_tf.codec - :members: read, write, from_bytes, to_bytes, write_sequence, read_sequence, write_index, read_index -``` - -## The data model - -```{eval-rst} -.. automodule:: openalea.fspm_tf.model - :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable -``` - -## Validation - -```{eval-rst} -.. automodule:: openalea.fspm_tf.validate - :members: validate_file, validate_bytes, validate_sequence, Report, Issue -``` - -## Display copies - -```{eval-rst} -.. automodule:: openalea.fspm_tf.display - :members: add_display, remove_display -``` - -## Vocabulary and units - -```{eval-rst} -.. automodule:: openalea.fspm_tf.vocabulary - :members: standard_names, platform_names, labels, enums, version - -.. automodule:: openalea.fspm_tf.ucum - :members: dimension, is_valid, convertible -``` - -## Comparing with expected values - -```{eval-rst} -.. automodule:: openalea.fspm_tf.compare - :members: compare, compare_files, compare_all -``` - -## Expected values - -```{eval-rst} -.. automodule:: openalea.fspm_tf.dump - :members: frame_to_dict -``` - -## Command line - -```{eval-rst} -.. automodule:: openalea.fspm_tf.cli -``` diff --git a/docs/using.md b/docs/using.md deleted file mode 100644 index 8fd3bf3..0000000 --- a/docs/using.md +++ /dev/null @@ -1,106 +0,0 @@ -# Viewing and checking files - -## Installing - -The Python package `openalea.fspm_tf` brings the command line, the validator and the reader used by ParaView's -plugin. Until it is published on the `openalea3` channel, install it from the sources: - -```bash -git clone https://github.com/openalea/fspm-TF.git -cd fspm-TF -mamba env create -f conda/environment.yml -mamba activate fspm_tf -``` - -To use it in another environment, e.g. a model's, install its dependencies there (`mamba install numpy jsonschema`), -then `pip install -e path/to/fspm-TF`. - -## In ParaView - -fspm-TF comes with a ParaView reader plugin: `paraview/fspm_tf_reader.py`. It works with ParaView 5.10 and later, -and has been checked with 6.0.1. - -1. In ParaView, open **Tools → Manage Plugins → Load New…**, choose `fspm_tf_reader.py`, and tick **Auto Load** to - keep it loaded. -2. **File → Open** a frame (`.glb`), or a sequence's `scene.fspm.json` to play its frames as time steps, and choose - **fspm-TF reader**. - -The plugin needs nothing installed in ParaView: it reads files with the code of the fspm-TF folder it comes from. - -What you see: - -- **One block per plant population or environment model.** Each holds its meshes, its root segments as lines (apply - a *Tube* filter on `radius` to give them volume), and its grid. -- **Every variable of every scale is a colouring choice**, named `.`: `SubOrgan.water_potential`, - `Axis.branching_order`, `Voxel.soil_water_potential`. Values are at full precision. Where a triangle has no entity - at a scale, the value is empty (NaN). -- **Labels and other text become integer codes**, `.label_code`; the names behind the codes are in the - block's field data. -- **Show Settings**, a reader option, adds the columns a platform keeps for its own bookkeeping, and the tables' - links (`complex`, `parent`, `edge_type`). They are hidden by default. - -```{image} _static/images/root_system.png -:alt: A small root system coloured by water potential -:width: 260px -``` - -*The conformance root system, coloured by `SubOrgan.water_potential`.* - -In a `pvpython` script: - -```python -from paraview import simple -simple.LoadPlugin("fspm-TF/paraview/fspm_tf_reader.py", ns=globals()) -roots = fspmTFreader(FileName="outputs/frames/scene.fspm.json") -roots.UpdatePipeline(roots.TimestepValues[-1]) # the last frame -``` - -## In other viewers - -Any glTF viewer (Blender, a web viewer, ParaView without the plugin) shows the geometry, and ignores the tables. -To colour shapes there by a variable, add a **display copy**: a per-vertex copy of the variable, in single precision, -which viewers read as a colouring attribute. - -```bash -fspm-tf display scene.glb water_potential # adds _WATER_POTENTIAL to the meshes -fspm-tf display scene.glb --remove # removes the display copies -``` - -Display copies are optional and flagged as such. Readers never take them for the variable itself. - -## Checking a file - -```bash -fspm-tf validate scene.glb # one frame -fspm-tf validate outputs/frames/scene.fspm.json # a sequence: every frame, and the rules between frames -``` - -The validator reports **errors** (the file does not follow the specification) and **warnings** (it does, but something -is likely missing, such as a variable without a declared unit). It checks, among other things, that: - -- the ids are unique and, in a sequence, never reused; -- every `complex` and `parent` points to an existing entity, and the parents form no cycle; -- the units are valid UCUM codes that convert to their standard names' units; -- the grids and the mappings agree with their entities; -- the binary tables have the length their types imply. - -The command exits with 1 when a file has errors, so it can run in a CI. - -## Looking inside a file - -```bash -fspm-tf info scene.glb -``` - -For example, for a frame of a metafspm scene with roots in a soil grid: - -``` -scene.glb: time 86400.0 s, step 24, writer {'platform': 'openalea.metafspm'} - RootPopulation (graph, MPGDataStructure) - Plant rank 1: 3 entities, variables x, y, z, ... - SubOrgan rank 6: 412 entities, variables length, radius, struct_mass, ... - mesh triangles: 9888 vertices, entity sets SubOrgan, Axis, Plant - Soil (regular_grid, ArrayDataStructure) - cell: 1024 entities, variables water_content, soil_temperature, ... - mapping RootPopulation_to_Soil: RootPopulation/SubOrgan -> Soil/cell, 530 pairs -``` diff --git a/implementations.md b/implementations.md index 362268a..80e3bb5 100644 --- a/implementations.md +++ b/implementations.md @@ -2,13 +2,28 @@ | Platform | Language | Where | Reads | Writes | Specification version | |---|---|---|---|---|---| -| openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.1 | -| openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.1 | -| CPlantBox | C++ / Python | planned (phase P4) | | | | -| GroIMP | Java | | | | | -| VirtualPlantLab | Julia | | | | | +| openalea.encoding3d (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.3 | +| openalea.metafspm | Python | `openalea.metafspm.io.encoding3d` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.3 (through openalea.encoding3d) | +| encoding3d (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.3 | +| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`encoding3d::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.3 | +| encoding3d (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.3 | +| GroIMP | Java | `bindings/java/groimp/` (`GroImp`) and `bindings/java/groimp-plugin/` (a GroIMP 3.0 plugin: File > Export to `.glb`, CLI `export3d`) | yes: graphs and fields into the model's node classes (adapter) | yes: scales, topology, edge bits, managed fields, the scene's geometry by node | 0.3 | +| encoding3d.jl (Julia codec) | Julia | this repository, `bindings/julia/encoding3d/` | yes | yes | 0.3 | +| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | encoding3d.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.3 | +| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | encoding3d.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.3 | + +Coupling sessions (specification §11): the Python package holds the coordinator and a participant; encoding3d.jl, the C++ +and the Java codecs a participant each. All four pass the coupling conformance sessions over both transports (ZeroMQ: +pyzmq, ZMQ.jl, libzmq, JeroMQ). metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for another +platform's model in a metafspm model, tested with a VirtualPlantLab model (PlantGraphs). CPlantBox's and GroIMP's logs +are replayed as participants when their adapters are tested by hand. + +The platform adapters kept in this repository (CPlantBox's, GroIMP's adapter and plugin, the MTG.jl and VirtualPlantLab +extensions) are **first proposals for those platforms' teams**: they move into the platforms once adopted. encoding3d's +requirements and CI do not depend on any platform; each adapter's README says how to build and test it by hand. Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters -(a platform's model to and from the codec's tables) in the platforms' repositories. An implementation is listed with +(a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional +extension when the platform's library is a plain dependency (Julia's package extensions). An implementation is listed with the version it passes: it reads every conformance file as expected, and writes files that the validator accepts (specification, §9). diff --git a/paraview/README.md b/paraview/README.md new file mode 100644 index 0000000..26653ad --- /dev/null +++ b/paraview/README.md @@ -0,0 +1,142 @@ +# encoding3d in ParaView: how-to + +One file to load: **`plugin_encoding3d.py`**, the plugin. There are no macros to add. The other files here are not +plugins, so do not load them: + +| file | what it is | +|---|---| +| `plugin_encoding3d.py` | **the plugin**: the *encoding3d reader* and the *encoding3d Select at scale* filter | +| `encoding3d_displays.py` | the plugin's code that draws each layer and runs its buttons (the plugin loads it) | +| `check_in_paraview.py` | a check run with `pvpython` or `pvbatch`, for developers (end of this page) | + +Menu names below are ParaView 6.2's. The plugin works with ParaView 5.11 and later. + +## 1. Install the plugin (once) + +1. **Tools → Manage Plugins...** +2. Under *Local Plugins*, click **Load New ...** and choose `encoding3d/paraview/plugin_encoding3d.py`. +3. Unfold the plugin's line, tick **Auto Load**, and click **Close**. ParaView then loads the plugin each time it + starts. + +You do not need to install anything in ParaView: the plugin reads files with the code of the encoding3d folder it +sits in. Do not move the file out of that folder. A link to it, placed anywhere, works. + +## 2. Open a run's frames + +1. **File → Open...**, go to the run's `outputs/.../frames/` folder, and choose `scene.fspm.json` (all the frames, as + time steps) or a single `frame_XXXXXX.glb`. +2. If ParaView asks which reader to use, choose **encoding3d reader**. +3. Click **Apply** in the *Properties* panel. + +**Time Unit** (the reader's *Properties*): the unit of the times ParaView shows in the animation toolbar and the +*Time Inspector*, days by default (`d`, `h`, `min` or `s`). The frames' files keep their own unit (metafspm writes +seconds); only what ParaView shows is converted. To print the time in the view, add **Filters → Annotate Time Filter** on +the reader. + +Large runs: frames grow with the plants. To look around quickly, open an early `frame_XXXXXX.glb` first. In a +sequence, ParaView reads only the frame of the current time. + +## 3. What appears: the layers + +When the frames come with a `views.json` (metafspm writes one), its **default configuration** is drawn as soon as +you click Apply, with no macro to run. Each **layer** is drawn its own way: the shoot, the roots (as tubes), the soil +grid (translucent), and the root sections (hidden at first). Each layer has its own colour variable, colour map, +range, opacity and visibility, with its own colour legend. + +The layers are the blocks of the reader's item. To see them and change them one by one: + +- **View → MultiBlock Inspector** opens a panel that lists the layers by name. + - The **check box** of a layer shows or hides it, for example the sections. + - **Click a layer** (Ctrl-click for several) to edit only that layer, in the inspector's lower part: + - **Block Coloring**: its colour variable (the drop-down), **Rescale to Data Range**, the colour map (the + *Edit color map* button opens the **Color Map Editor**), **Toggle Color Legend Visibility** and **Use Separate + Color Map**. + - **Opacity**: for example 0.2 to make the soil more see-through. +- In the view, **right-click a part** (for example the soil) for **Block 'RichardsSoilModel'** → **Hide Block**, + **Show Only Block**, **Show All Blocks**, **Set Block Color**, **Set Block Opacity**. + +The colouring and opacity in the *Properties* panel apply to the whole item. They do not change a layer that has its +own colouring or opacity, which is every layer at the start. Change a layer in the MultiBlock Inspector instead. + +Layers coloured by the same variable share one colour map, as in ParaView in general. Tick **Use Separate Color Map** +to give a layer its own. + +Without a `views.json`, the reader shows one block per data structure (each plant population, the soil), with every +variable as a colouring choice. Colour them like any ParaView item. + +## 4. Switch configurations + +A `views.json` can hold several configurations, for example "default" or "roots and sections". On the reader's item, +in the *Properties* panel: + +- **Configuration**: a drop-down of the configurations in the file (empty: the file's active one). Choose one, then + click **Apply**. The layers are drawn again as that configuration says. +- **Views**: another views file to use (empty: the `views.json` next to the opened file). + +## 5. Save your changes into `views.json` + +After you change layers in the MultiBlock Inspector, select the reader's item. In its *Properties* panel: + +- **Save Views** writes each layer's display into `views.json`, over the configuration shown: visibility, colour + variable or solid colour, range (fixed, or following the data), log scale, opacity, and the item's representation. +- To keep the configuration and save a new one instead, type a name in **Save As**, click **Apply**, then + **Save Views**. The new configuration becomes the file's active one, so it opens next time, and the + **Configuration** drop-down shows it. + +A colour map's preset name is not read back from ParaView, so a saved layer keeps its `colormap`. + +## 6. Select a plant, an axis, an organ, a segment + +1. In the render view's toolbar, click **Select Cells On (s)** (or press `s`), then drag a small rectangle over a + part of the view (a root, a leaf). The picked cells are highlighted. +2. With the reader's item still active in the *Pipeline Browser*, choose **Filters → encoding3d → encoding3d Select at + scale** (or **Filters → Alphabetical**, or Ctrl+Space and type "Select at scale"), then **Apply**. +3. In the filter's *Properties*, choose the **Scale**: `Plant` (the default), `Axis`, `Organ`, `SubOrgan`, ... + `Compartment`. Click **Apply**. The selection is the picked element's whole plant, axis, organ or segment, with + everything it contains at finer scales, in every layer, the hidden ones included (a segment's section comes with + it). +4. Play the time steps: the selection follows the same plant or organ as it grows. + +The selection's item is drawn like the layers (the same colour maps). Its *Properties* panel has three buttons: + +- **Isolate**: the active view shows only the selection; the other items are hidden. +- **Show In Panel**: a new view opens beside the active one with the selection alone, its camera fitted to it; the + first view is not changed. +- **Put Back**: one step back. The items that Isolate hid are shown again, or the panel is closed. + +To narrow a selection (an axis of the plant shown), pick on the selection's item in the view or panel, apply another +*encoding3d Select at scale* to it, then click its **Isolate**. Its **Put Back** returns to the plant. + +To drop a selection, click **Put Back** first, then delete the filter (*Pipeline Browser*, Delete key). To pick +another element with the same filter, make the new selection, then click **Copy Active Selection** in the filter's +*Properties* and click **Apply**. + +A Scale finer than the picked element (a `Compartment` of a leaf drawn without sections) is reported in **View → +Output Messages**. + +## Limits + +- One representation draws all layers. Wireframe or Points apply to the whole item; a layer's own `representation` + is kept in `views.json` but not drawn separately. +- The buttons (**Save Views**, **Isolate**, **Show In Panel**, **Put Back**) work with ParaView's built-in server, + which is the default. With a remote server (*File → Connect*), the layers are still drawn and the filter works, + but the buttons do nothing (a message in Output Messages). +- Items made from the reader by other filters (*Clip*, *Slice*) are drawn by ParaView in the usual way, not as the + layers. + +## For developers + +- `pvpython paraview/check_in_paraview.py conformance` checks the plugin on the conformance files (the Python tests + run it when `ENCODING3D_PVPYTHON` names a `pvpython`). +- `pvbatch paraview/check_in_paraview.py --design /frames/scene.fspm.json ` is a check by hand on a + copy of a run's frames. It draws each configuration, saves the views back, selects at several scales, isolates, + opens a panel and puts back, and saves a screenshot of each step in ``. +- In a script: `simple.LoadPlugin("paraview/plugin_encoding3d.py", ns=globals())`, then + `reader = encoding3dreader(FileName=...)`. `simple.Show(reader)` draws the layers as the GUI's Apply does, and + `reader.InvokeCommand("SaveViews")` or `selection.InvokeCommand("Isolate")` press the buttons. +- How it works: the plugin watches the session (its `encoding3d_displays.install()`). It draws each new + representation of a reader with views, or of a selection drawn from one, by setting the representation's block + properties (`BlockSelectors`, `BlockColorArrayNames`, `BlockLookupTables`, `BlockColors`, `BlockOpacities`). It + draws them again when the configuration or the views file changes. The block properties come from + `openalea.encoding3d.views.block_displays`, and `displays_from_blocks` reads them back. Both are tested without + ParaView. diff --git a/paraview/check_in_paraview.py b/paraview/check_in_paraview.py index 74731c2..d06662c 100644 --- a/paraview/check_in_paraview.py +++ b/paraview/check_in_paraview.py @@ -1,88 +1,439 @@ """ -The reader inside ParaView: loads the plugin, opens conformance files and a sequence, checks their blocks, arrays and -time steps, and renders a picture of each. +The plugin inside ParaView: loads plugin_encoding3d.py, opens conformance files and a sequence, checks their blocks, +arrays and time steps, and renders a picture of each; then a frame with views (one block per layer, drawn by the +plugin as its layers when shown, the configurations' drop-down, "Save Views"), and selection by scale (the filter +"encoding3d Select at scale", its Scale drop-down, its Isolate and Put Back buttons). pvpython paraview/check_in_paraview.py conformance [output folder] + +A design check, by hand (no test runs it): a sequence with its views.json (e.g. metafspm's +SceneRecorder(frame_views="default", display_period=...) frames of plants with shoots, roots and sections, and a soil +grid) opened and shown as the GUI's Apply does (each layer its colouring and opacity, with no macro), a configuration +of distinct colour maps added and chosen in the drop-down, the display saved back unchanged and changed (the round +trip, "Save Views" and "Save As"), then selections by the filter "encoding3d Select at scale" (cells picked as a click +would, the filter made with ParaView's selection as the GUI makes it, its Scale switched, the time stepped), shown by +its buttons isolated, in a panel, one over another, and put back; a screenshot of each step in the output folder, to +look at (colours, opacities, tubes, hidden layers, isolation). Run it on a copy of the frames: the views file is put +back as it was, but the check writes it meanwhile. + + pvbatch paraview/check_in_paraview.py --design copy/frames/scene.fspm.json screenshots """ import os import sys -from paraview import simple +from paraview import servermanager, simple HERE = os.path.dirname(os.path.abspath(__file__)) -conformance = sys.argv[1] -output = sys.argv[2] if len(sys.argv) > 2 else None +DESIGN = len(sys.argv) > 1 and sys.argv[1] == "--design" +conformance = sys.argv[1] if not DESIGN else None +pictures = sys.argv[2 + DESIGN] if len(sys.argv) > 2 + DESIGN else None +# (not "output": a Programmable Filter removes that name from the main module once it has run) -simple.LoadPlugin(os.path.join(HERE, "fspm_tf_reader.py"), ns=globals()) +simple.LoadPlugin(os.path.join(HERE, "plugin_encoding3d.py"), ns=globals()) +import encoding3d_displays # noqa: E402 (on ParaView's path once the plugin is loaded) def arrays(source) -> set: - information = source.GetDataInformation().DataInformation - names = set() - for association in (0, 1): # points, cells - field = source.GetDataInformation().GetPointDataInformation() if association == 0 \ - else source.GetDataInformation().GetCellDataInformation() - for number in range(field.GetNumberOfArrays()): - names.add(field.GetArrayInformation(number).GetName()) - del information - return names + return set(source.PointData.keys()) | set(source.CellData.keys()) + + +def configuration_names(reader) -> list: + reader.UpdatePropertyInformation() + return list(reader.GetProperty("ConfigurationNames")) + + +def blocks(source) -> int: + return servermanager.Fetch(source).GetNumberOfBlocks() + + +def block_names(source) -> list: + """The names of *source*'s blocks, in order (its hierarchy's labels: Fetch renames them).""" + hierarchy = source.GetDataInformation().DataInformation.GetHierarchy() + return [hierarchy.GetAttributeOrDefault(node, "label", hierarchy.GetNodeName(node)) + for node in hierarchy.GetChildNodes(hierarchy.GetRootNode(), False)] def picture(source, name, colour_by, side=False): - if output is None: + if pictures is None: return - os.makedirs(output, exist_ok=True) + os.makedirs(pictures, exist_ok=True) + active = simple.GetActiveView() view = simple.CreateView("RenderView") view.ViewSize = [800, 600] display = simple.Show(source, view) - simple.ColorBy(display, ("CELLS", colour_by)) - display.RescaleTransferFunctionToDataRange(True) + if colour_by: + simple.ColorBy(display, ("CELLS", colour_by)) + display.RescaleTransferFunctionToDataRange(True) if side: # seen from the side: z up view.CameraPosition, view.CameraFocalPoint, view.CameraViewUp = [0., -1., 0.], [0., 0., 0.], [0., 0., 1.] view.ResetCamera() - simple.SaveScreenshot(os.path.join(output, f"{name}.png"), view) + simple.SaveScreenshot(os.path.join(pictures, f"{name}.png"), view) simple.Delete(view) + if active is not None: + simple.SetActiveView(active) # the view picked on afterwards + + +def drawn(display) -> dict: + """The block properties of a display, as the plugin set them.""" + return {name: [str(value) for value in display.GetProperty(name)] + for name in ("BlockSelectors", "BlockColorArrayNames", "BlockColors", "BlockOpacities")} + + +def pairs(values: list, size: int) -> dict: + return {values[start]: values[start + 1:start + size] for start in range(0, len(values), size)} + + +def check_drawn(display, layers: list, selection: bool = False) -> None: + """*display* drawn as *layers* say: shown blocks, colourings, solid colours, opacities.""" + from openalea.encoding3d import views + found = drawn(display) + if selection: + layers = [dict(layer, visible=True) for layer in layers] + expected = views.block_displays(layers, set(display.Input.CellData.keys())) + assert found["BlockSelectors"] == expected["selectors"], (found, expected) + colourings = {selector: values[1] for selector, values in pairs(found["BlockColorArrayNames"], 3).items()} + assert colourings == dict(expected["colourings"]), (colourings, expected) + opacities = {selector: float(values[0]) for selector, values in pairs(found["BlockOpacities"], 2).items()} + assert opacities == dict(expected["opacities"]), (opacities, expected) + assert set(pairs(found["BlockColors"], 4)) == {selector for selector, _ in expected["solid_colours"]} + if expected["main_array"]: + assert display.ColorArrayName[1] == expected["main_array"], display.ColorArrayName + + +def flat_index(source, name: str) -> int: + """The flat index of the reader's (or a selection's) block *name* (the root is 0).""" + return block_names(source).index(name) + 1 + + +def pick(item, block: str, array: str, value: float) -> int: + """Pick (select) the first cell of *item*'s *block* whose *array* is *value*, as a click would; its index.""" + import numpy as np + from vtkmodules.util.numpy_support import vtk_to_numpy + data = servermanager.Fetch(item) + leaf = data.GetBlock(block_names(item).index(block)) + cell = int(np.flatnonzero(vtk_to_numpy(leaf.GetCellData().GetArray(array)) == value)[0]) + simple.SelectCompositeDataIDs(IDs=[flat_index(item, block), 0, cell], FieldType="CELL", Source=item) + return cell + + +def cells(source, block: str) -> int: + data = servermanager.Fetch(source) + return data.GetBlock(block_names(source).index(block)).GetNumberOfCells() + + +def scale_names(selection) -> list: + selection.UpdatePropertyInformation() + return list(selection.GetProperty("ScaleNames")) + + +def visible(item, view) -> bool: + return bool(simple.GetDisplayProperties(item, view).Visibility) + + +def check_conformance() -> None: + roots = encoding3dreader(FileName=os.path.join(conformance, "soil_and_roots.glb")) # noqa: F821 + roots.UpdatePipeline() + found = arrays(roots) + for name in ("SubOrgan.water_potential", "Axis.branching_order", "Voxel.soil_water_potential"): + assert name in found, (name, sorted(found)) + roots_only = encoding3dreader(FileName=os.path.join(conformance, "root_system.glb")) # noqa: F821 + picture(roots_only, "root_system", "SubOrgan.water_potential", side=True) + + shoot = encoding3dreader(FileName=os.path.join(conformance, "wheat_shoot.glb")) # noqa: F821 + shoot.UpdatePipeline() + assert "SubOrgan.PARa" in arrays(shoot) + picture(shoot, "wheat_shoot", "SubOrgan.PARa") + + sequence = encoding3dreader(FileName=os.path.join(conformance, "growing_sequence", "scene.fspm.json")) # noqa: F821 + sequence.UpdatePipelineInformation() + assert list(sequence.TimestepValues) == [0., 1., 2.], list(sequence.TimestepValues) + counts = [] + for time in sequence.TimestepValues: + sequence.UpdatePipeline(time) + counts.append(sequence.GetDataInformation().GetNumberOfCells()) + assert counts[0] < counts[1] < counts[2], counts # the root system grows + print("cells per time step", counts) + sequence.TimeUnit = "h" # the times shown in hours: the files' days + sequence.UpdatePipelineInformation() + assert list(sequence.TimestepValues) == [0., 24., 48.], list(sequence.TimestepValues) + sequence.UpdatePipeline(24.) + assert sequence.GetDataInformation().GetNumberOfCells() == counts[1] + sequence.TimeUnit = "d" + + # Without the plugin: ParaView's own glTF reader shows the geometry and the display copies (float32 point data) + import tempfile + + from openalea import encoding3d + from openalea.encoding3d.display import add_display + + with tempfile.TemporaryDirectory() as directory: + copied = os.path.join(directory, "roots_display.glb") + encoding3d.write(copied, add_display(encoding3d.read(os.path.join(conformance, "root_system.glb")), + ["water_potential"])) + plain = simple.glTFReader(FileName=copied) # ParaView's own reader, not the plugin + plain.UpdatePipeline() + names = set(plain.PointData.keys()) + print("glTF reader", type(plain).__name__, "point data", sorted(names)) + assert any("WATER_POTENTIAL" in name for name in names), sorted(names) + + # Views: a views.json next to the frame gives one block per layer, each drawn as its layer when shown; "Save + # Views" writes the display back; a picked segment grows to its axis, isolated, then put back + import shutil + + from openalea.encoding3d import views + + with tempfile.TemporaryDirectory() as directory: + shutil.copy(os.path.join(conformance, "root_system.glb"), directory) + layers = [{"name": "roots", + "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "tube": {"radius": "radius"}}, + {"name": "mesh", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "color_by": None, "opacity": 0.3, "visible": False}] + path = os.path.join(directory, views.VIEWS_FILE) + views.write_views(path, {"encoding3d_views": 1, "active": "default", + "configurations": {"default": {"layers": layers}, + "mesh only": {"layers": layers[1:]}}}) + viewed = encoding3dreader(FileName=os.path.join(directory, "root_system.glb")) # noqa: F821 + assert configuration_names(viewed) == ["default", "mesh only"], configuration_names(viewed) + viewed.UpdatePipeline() + assert block_names(viewed) == ["roots", "mesh"], block_names(viewed) + view = simple.GetActiveViewOrCreate("RenderView") + display = simple.Show(viewed, view) # as the GUI's Apply: drawn as its layers + check_drawn(display, views.configuration(views.read_views(path))[1]) + display.BlockOpacities = [views.block_selector("roots"), "0.5"] + viewed.InvokeCommand("SaveViews") # the reader's button + assert views.read_views(path)["configurations"]["default"]["layers"][0]["opacity"] == 0.5 + picture(viewed, "root_system_views", None, side=True) + + lateral = pick(viewed, "roots", "Axis.id", 101.) + selection = encoding3dSelectatscale(Input=viewed) # noqa: F821 (ParaView's selection, as the GUI takes it) + assert selection.Selection is not None + assert scale_names(selection) == ["Plant", "Axis", "SubOrgan"], scale_names(selection) + assert selection.Scale == "Plant" # the drop-down's first, the coarsest + selection.Scale = "Axis" + selection.UpdatePipeline() + grown = servermanager.Fetch(selection) + assert grown.GetNumberOfBlocks() == 2 # the layers, the hidden one too + assert set(grown.GetBlock(0).GetCellData().GetArray("Axis.id").GetRange()) == {101.}, lateral + assert 0 < grown.GetBlock(1).GetNumberOfCells() < servermanager.Fetch(viewed).GetBlock(1).GetNumberOfCells() + selection_display = simple.Show(selection, view) # as the GUI's Apply: drawn as the layers + check_drawn(selection_display, views.configuration(views.read_views(path))[1], selection=True) + legend = simple.GetScalarBar(simple.GetDisplayProperties(viewed, view).LookupTable, view) + legend.Visibility = 1 # a legend shown, as the GUI's Apply does + selection.InvokeCommand("Isolate") + assert not visible(viewed, view) and visible(selection, view) + selection.InvokeCommand("PutBack") + assert visible(viewed, view) and not visible(selection, view) + assert legend.Visibility # put back with what Isolate hid + selection.InvokeCommand("Isolate") + selection.InvokeCommand("ShowInPanel") # puts the isolation back first + assert visible(viewed, view) and legend.Visibility + selection.InvokeCommand("PutBack") + simple.Delete(selection) # the selection dropped + viewed.Configuration = "mesh only" # the drop-down: drawn again + simple.Render(view) + assert block_names(viewed) == ["mesh"] + check_drawn(display, views.configuration(views.read_views(path), "mesh only")[1]) + + +# ── The design check, by hand ──────────────────────────────────────────────── + +def screenshot(view, name: str) -> None: + os.makedirs(pictures, exist_ok=True) + path = os.path.join(pictures, f"{name}.png") + simple.SaveScreenshot(path, view, ImageResolution=[900, 700]) + print("screenshot", path) + + +def layout_screenshot(layout, name: str) -> None: + os.makedirs(pictures, exist_ok=True) + path = os.path.join(pictures, f"{name}.png") + simple.SaveScreenshot(path, layout, ImageResolution=[1600, 700]) + print("screenshot", path) + + +def side_view(view) -> None: + """Seen from the side and a little above: z up.""" + view.CameraPosition, view.CameraFocalPoint, view.CameraViewUp = [0.3, -1., 0.35], [0., 0., 0.], [0., 0., 1.] + view.ResetCamera() + + +def display_mesh(layer: dict) -> bool: + """Whether *layer* shows a component's display mesh (e.g. sections: a mesh other than the shoot's triangles), + whatever its name (" ").""" + source = layer["source"] + return source["kind"] == "mesh" and source.get("mesh") != "triangles" + + +def check_design(index_path: str) -> None: + """The plugin's layers and selection by scale on a sequence with its views.json, screenshots in *pictures*.""" + import json + import shutil + + import numpy as np + from vtkmodules.util.numpy_support import vtk_to_numpy + + from openalea.encoding3d import views + + views_file = os.path.join(os.path.dirname(os.path.abspath(index_path)), views.VIEWS_FILE) + shutil.copy(views_file, views_file + ".before_check") + try: + document = views.read_views(views_file) + name, layers = views.configuration(document) + shoot = next(layer["name"] for layer in layers if layer["name"].endswith("shoot")) + roots = next(layer["name"] for layer in layers if layer["name"].endswith("roots")) + sections = next(layer["name"] for layer in layers if display_mesh(layer)) + grids = [layer["name"] for layer in layers if layer["source"]["kind"] == "grid"] + maps = {shoot: "Cool to Warm", roots: "Inferno (matplotlib)"} # a configuration of distinct maps + distinct = [dict(layer, colormap=maps.get(layer["name"], layer["colormap"])) for layer in layers] + roots_only = [dict(layer, visible=True) for layer in layers if layer["name"] in (roots, sections)] + for layer in roots_only: # and one of the roots by radius, a range, a log scale + if layer.get("tube"): + layer.update(color_by={"entity_set": "SubOrgan", "variable": "radius"}, range=[0.0002, 0.002], + log=True, colormap="Inferno (matplotlib)") + document = views.save_configuration(document, "distinct colour maps", distinct, active=False) + views.write_views(views_file, views.save_configuration(document, "roots and sections", roots_only, + active=False)) + reader = encoding3dreader(FileName=os.path.abspath(index_path)) # noqa: F821 + names = configuration_names(reader) + print("configurations", names) + assert names == [name, "distinct colour maps", "roots and sections"], names + domain = reader.GetProperty("Configuration").SMProperty.GetDomain("list") + assert [domain.GetString(number) for number in range(domain.GetNumberOfStrings())] == names + reader.UpdatePipelineInformation() + last = list(reader.TimestepValues)[-1] if reader.TimestepValues else None + view = simple.GetActiveViewOrCreate("RenderView") + view.ViewSize = [900, 700] + view.ViewTime = last if last is not None else 0. + reader.UpdatePipeline(last) + assert blocks(reader) == len(layers) + + display = simple.Show(reader, view) # as the GUI's Apply: no macro, drawn as the layers + check_drawn(display, layers) + print("drawn", json.dumps(drawn(display))) + encoding3d_displays.show_scalar_bars(display.SMProxy, view.SMProxy) # (the GUI's Apply shows them) + side_view(view) + screenshot(view, "1_default_configuration") + + reader.Configuration = "distinct colour maps" # the drop-down, Apply: drawn again + simple.Render(view) + check_drawn(display, distinct) + encoding3d_displays.show_scalar_bars(display.SMProxy, view.SMProxy) + colourings = views.block_displays(distinct, set(reader.CellData.keys()))["colourings"] + firsts = {array: tuple(list(simple.GetColorTransferFunction(array).RGBPoints)[1:4]) for _, array in colourings} + print("first colour of each layer's map", firsts) + assert len(set(firsts.values())) == len(firsts), firsts + screenshot(view, "2_distinct_colour_maps") + + reader.InvokeCommand("SaveViews") # saved unchanged: the same configuration + saved = views.configuration(views.read_views(views_file), "distinct colour maps")[1] + assert saved == [views.filled(layer) for layer in distinct], (saved, distinct) + display.BlockOpacities = [views.block_selector(grids[0]), "0.15", views.block_selector(shoot), "0.6"] + reader.SaveAs = "faded" # changed, saved as another configuration + reader.InvokeCommand("SaveViews") + faded = views.configuration(views.read_views(views_file), "faded")[1] + assert [layer["opacity"] for layer in faded if layer["name"] in (shoot, grids[0])] == [0.6, 0.15], faded + assert views.read_views(views_file)["active"] == "faded" and reader.Configuration == "faded" + reader.SaveAs = "" + simple.Render(view) + screenshot(view, "3_faded_saved") + + reader.Configuration = name # back to the default + simple.Render(view) + check_drawn(display, layers) + data = servermanager.Fetch(reader) + tubes = data.GetBlock(block_names(reader).index(roots)) + axes = vtk_to_numpy(tubes.GetCellData().GetArray("Axis.id")) + lateral = float(axes.max()) # the last axis: a lateral root + pick(reader, roots, "Axis.id", lateral) + selection = encoding3dSelectatscale(Input=reader) # noqa: F821 (the GUI's Filters > encoding3d, Apply) + scales = scale_names(selection) + print("scales", scales) + assert scales[0] == "Plant" and "Axis" in scales and "Compartment" in scales, scales + selection.Scale = "Axis" + selection_display = simple.Show(selection, view) # Apply shows it, drawn as the layers + check_drawn(selection_display, layers, selection=True) + selection.InvokeCommand("Isolate") + assert not visible(reader, view) and visible(selection, view) + screenshot(view, "4_isolate_axis") + assert set(servermanager.Fetch(selection).GetBlock(block_names(selection).index(roots)).GetCellData() + .GetArray("Axis.id").GetRange()) == {lateral} + axis_cells = cells(selection, roots) + selection.Scale = "Plant" # the drop-down changed, Apply: re-grown + simple.Render(view) + assert cells(selection, roots) > axis_cells, (cells(selection, roots), axis_cells) + screenshot(view, "5_isolate_scale_switched_to_plant") + if reader.TimestepValues: # stepping in time: the same plant followed + view.ViewTime = list(reader.TimestepValues)[0] + simple.Render(view) + screenshot(view, "6_isolate_plant_first_time_step") + view.ViewTime = last + simple.Render(view) + selection.InvokeCommand("PutBack") + assert visible(reader, view) and not visible(selection, view) + simple.Delete(selection) # the selection dropped + screenshot(view, "7_put_back") + + segment = float(vtk_to_numpy(tubes.GetCellData().GetArray("SubOrgan.id"))[np.flatnonzero(axes == lateral)[0]]) + pick(reader, roots, "SubOrgan.id", segment) + selection = encoding3dSelectatscale(Input=reader, Scale="SubOrgan") # noqa: F821 + simple.Show(selection, view) + selection.InvokeCommand("Isolate") + assert cells(selection, sections) > 0 # the hidden layer's part: the segment's section + assert views.block_selector(sections) in drawn(simple.GetDisplayProperties(selection, view))["BlockSelectors"] + view.ResetCamera() + screenshot(view, "8_isolate_segment_with_its_section") + selection.InvokeCommand("PutBack") + simple.Delete(selection) + side_view(view) + + shoot_data = data.GetBlock(block_names(reader).index(shoot)) + plant = float(vtk_to_numpy(shoot_data.GetCellData().GetArray("Plant.id"))[0]) + pick(reader, shoot, "Plant.id", plant) + plant_selection = encoding3dSelectatscale(Input=reader) # noqa: F821 (Scale: Plant, the first) + plant_selection.InvokeCommand("ShowInPanel") + layout = simple.GetLayout(view) + layout.SetSize(1600, 700) + layout_screenshot(layout, "9_panel_plant") + assert visible(reader, view) # the view untouched + panel = [proxy for proxy in simple.GetViews() if proxy.SMProxy is not view.SMProxy][0] + simple.SetActiveView(panel) # an axis of the plant, picked in the panel + plant_axes = vtk_to_numpy(servermanager.Fetch(plant_selection).GetBlock( + block_names(plant_selection).index(roots)).GetCellData().GetArray("Axis.id")) + pick(plant_selection, roots, "Axis.id", float(plant_axes.max())) + axis_selection = encoding3dSelectatscale(Input=plant_selection, Scale="Axis") # noqa: F821 + simple.Show(axis_selection, panel) + axis_selection.InvokeCommand("Isolate") # in the active view: the panel + assert not visible(plant_selection, panel) and visible(axis_selection, panel) + layout_screenshot(layout, "10_panel_an_axis_of_the_plant") + axis_selection.InvokeCommand("PutBack") # back to the plant in the panel + assert visible(plant_selection, panel) + simple.Delete(axis_selection) + plant_selection.InvokeCommand("PutBack") # the panel closed + assert len(simple.GetViews()) == 1, simple.GetViews() + simple.Delete(plant_selection) + simple.SetActiveView(view) + view.ViewSize = [900, 700] + screenshot(view, "11_panel_closed") + + reader.Configuration = "roots and sections" # the drop-down + simple.Render(view) + check_drawn(display, roots_only) + colours = simple.GetColorTransferFunction("SubOrgan.radius") + points = list(colours.RGBPoints) + assert colours.UseLogScale and abs(points[0] - 0.0002) < 1e-12 and abs(points[-4] - 0.002) < 1e-12, points + side_view(view) + screenshot(view, "12_configuration_roots_and_sections") + reader.InvokeCommand("SaveViews") # its range and log scale read back + saved = views.configuration(views.read_views(views_file), "roots and sections")[1] + assert saved == [views.filled(layer) for layer in roots_only], (saved, roots_only) + finally: + shutil.move(views_file + ".before_check", views_file) + print("encoding3d design check: done, look at the screenshots in", pictures) -roots = fspmTFreader(FileName=os.path.join(conformance, "soil_and_roots.glb")) # noqa: F821 -roots.UpdatePipeline() -found = arrays(roots) -for name in ("SubOrgan.water_potential", "Axis.branching_order", "Voxel.soil_water_potential"): - assert name in found, (name, sorted(found)) -roots_only = fspmTFreader(FileName=os.path.join(conformance, "root_system.glb")) # noqa: F821 -picture(roots_only, "root_system", "SubOrgan.water_potential", side=True) - -shoot = fspmTFreader(FileName=os.path.join(conformance, "wheat_shoot.glb")) # noqa: F821 -shoot.UpdatePipeline() -assert "SubOrgan.PARa" in arrays(shoot) -picture(shoot, "wheat_shoot", "SubOrgan.PARa") - -sequence = fspmTFreader(FileName=os.path.join(conformance, "growing_sequence", "scene.fspm.json")) # noqa: F821 -sequence.UpdatePipelineInformation() -assert list(sequence.TimestepValues) == [0., 1., 2.], list(sequence.TimestepValues) -cells = [] -for time in sequence.TimestepValues: - sequence.UpdatePipeline(time) - cells.append(sequence.GetDataInformation().GetNumberOfCells()) -assert cells[0] < cells[1] < cells[2], cells # the root system grows -print("cells per time step", cells) - -# Without the plugin: ParaView's own glTF reader shows the geometry and the display copies (float32 point data) -sys.path.insert(0, os.path.join(os.path.dirname(HERE), "bindings", "python", "src")) -import tempfile # noqa: E402 - -from openalea import fspm_tf # noqa: E402 -from openalea.fspm_tf.display import add_display # noqa: E402 - -with tempfile.TemporaryDirectory() as directory: - copied = os.path.join(directory, "roots_display.glb") - fspm_tf.write(copied, add_display(fspm_tf.read(os.path.join(conformance, "root_system.glb")), ["water_potential"])) - plain = simple.glTFReader(FileName=copied) # ParaView's own reader, not the plugin - plain.UpdatePipeline() - names = set() - point_data = plain.GetDataInformation().GetPointDataInformation() - for number in range(point_data.GetNumberOfArrays()): - names.add(point_data.GetArrayInformation(number).GetName()) - print("glTF reader", type(plain).__name__, "point data", sorted(names)) - assert any("WATER_POTENTIAL" in name for name in names), sorted(names) -print("fspm-TF reader: ok") +if DESIGN: + check_design(sys.argv[2]) +else: + check_conformance() +print("encoding3d reader: ok") diff --git a/paraview/encoding3d_displays.py b/paraview/encoding3d_displays.py new file mode 100644 index 0000000..48fad0d --- /dev/null +++ b/paraview/encoding3d_displays.py @@ -0,0 +1,593 @@ +""" +How encoding3d's ParaView plugin (plugin_encoding3d.py, which loads this file) draws the views' layers and runs its +buttons, in ParaView's client: + + install() watch the session: each representation ParaView makes of an encoding3d reader with views, + or of an "encoding3d Select at scale" drawn from one (Apply, Show, a new view), is drawn + as the layers of the reader's configuration, each layer a block with its own colour + variable, colour map, range, log scale or solid colour, opacity and visibility; drawn + again when the reader's Configuration or its views file changes + save_views(reader) the reader's "Save Views": the layers' display, as the representation in the active view + holds it (after edits in the MultiBlock Inspector), written back into the views file + isolate(selection) the "Select at scale" buttons: the selection alone in the active view, the other items + show_in_panel(...) hidden; or in a new render view beside it, its camera fit to it; put_back goes back one + put_back() step (what a step hid shown again, its panel closed). A selection made on a shown + selection (an axis of the plant shown) is shown over it, and going back returns to the plant + +What each layer gives is openalea.encoding3d.views.block_displays, read back by displays_from_blocks and +layer_from_display, tested without ParaView; this file only hands them to ParaView's server manager. The buttons run +where the reader runs: in ParaView's built-in server (a remote server has no client to draw in). +""" +import contextlib +import json +import os +import time + +READER = "encoding3dReader" # the plugin's proxy names +SELECT_AT_SCALE = "encoding3dSelectAtScale" +DRAWN = "encoding3d_layers" # a representation's annotation: the views it is drawn from +STATE = "encoding3d_selection" # a view's annotation: [{"selection", "hidden", "panel", ...}] +CELLS = 1 # vtkDataObject.CELL, FIELD_ASSOCIATION_CELLS +NEVER, GROW_ON_APPLY = -1, 0 # a colour map's AutomaticRescaleRangeMode (vtkSMTransferFunctionManager) + +_session = {"managers": {}, "watched": set(), "busy": False, "connections": False} # kept across plugin reloads + + +def _views(): + from openalea.encoding3d import views + return views + + +def _server_manager(): + from paraview.modules import vtkRemotingServerManager + return vtkRemotingServerManager + + +def _colour_maps(): + from paraview.modules import vtkRemotingViews + return vtkRemotingViews + + +def _proxy_manager(): + """The active session's proxy manager (None: no session, e.g. a server without a client).""" + manager = _server_manager().vtkSMProxyManager + if not manager.IsInitialized(): + return None + return manager.GetProxyManager().GetActiveSessionProxyManager() + + +def _proxies(group: str) -> list: + proxy_manager = _proxy_manager() + if proxy_manager is None: + return [] + iterator = _server_manager().vtkSMProxyIterator() + iterator.SetSessionProxyManager(proxy_manager) + iterator.SetModeToOneGroup() + found, seen = [], set() + iterator.Begin(group) + while not iterator.IsAtEnd(): + proxy = iterator.GetProxy() + if proxy is not None and proxy.GetGlobalID() not in seen: + seen.add(proxy.GetGlobalID()) + found.append(proxy) + iterator.Next() + return found + + +def _helper(proxy, name: str): + return _server_manager().vtkSMPropertyHelper(proxy, name) + + +def _text(proxy, name: str) -> str: + helper = _helper(proxy, name) + return (helper.GetAsString(0) or "") if helper.GetNumberOfElements() else "" + + +def _strings(proxy, name: str) -> list: + helper = _helper(proxy, name) + return [helper.GetAsString(number) for number in range(helper.GetNumberOfElements())] + + +def _set_strings(proxy, name: str, values: list) -> None: + helper = _helper(proxy, name) + helper.SetNumberOfElements(len(values)) + for number, value in enumerate(values): + helper.Set(number, str(value)) + + +def _represented(view) -> list: + helper = _helper(view, "Representations") + return [helper.GetAsProxy(number) for number in range(helper.GetNumberOfElements())] + + +# ── Watching the session ───────────────────────────────────────────────────── + +def install() -> None: + """Watch the active session (once per session): its representations drawn as layers (see the module's doc).""" + if not _session["connections"]: # a session made later (Edit > Reset Session, Connect) + from paraview.modules.vtkRemotingCore import vtkProcessModule + process = vtkProcessModule.GetProcessModule() + if process is not None: + process.AddObserver("ConnectionCreatedEvent", lambda *arguments: install()) + _session["connections"] = True + proxy_manager = _proxy_manager() + if proxy_manager is None: + return + key = proxy_manager.GetAddressAsString("vtkObject") + if key in _session["managers"]: + return + tag = proxy_manager.AddObserver("RegisterEvent", _registered) + _session["managers"][key] = (proxy_manager, tag) # kept: its address not reused by a later session's + _registered() + + +def _registered(*arguments) -> None: + """A proxy registered: watch the plugin's new items, draw the new representations of them.""" + if _session["busy"]: + return + _session["busy"] = True + try: + for proxy in _proxies("sources"): + if proxy.GetXMLName() in (READER, SELECT_AT_SCALE) and proxy.GetGlobalID() not in _session["watched"]: + proxy.AddObserver("UpdateDataEvent", _updated) + _session["watched"].add(proxy.GetGlobalID()) + for representation in _proxies("representations"): + if not representation.HasAnnotation(DRAWN) and _drawable(representation): + draw(representation) + except Exception as error: + _warn(f"layers not drawn ({error})") + finally: + _session["busy"] = False + + +def _updated(source, event=None) -> None: + """The plugin's item updated (a time step, another Configuration, the views file changed): its representations + drawn again when their views changed.""" + if _session["busy"]: + return + _session["busy"] = True + try: + reader = reader_of(source) + if reader is not None: + signature = _signature(reader) + for representation in representations_of(source): + if _annotation(representation) != signature: + draw(representation) + except Exception as error: + _warn(f"layers not drawn ({error})") + finally: + _session["busy"] = False + + +@contextlib.contextmanager +def _quiet(): + """VTK's warnings off (ParaView would show them in its Output Messages).""" + from vtkmodules.vtkCommonCore import vtkObject + shown = vtkObject.GetGlobalWarningDisplay() + vtkObject.GlobalWarningDisplayOff() + try: + yield + finally: + vtkObject.SetGlobalWarningDisplay(shown) + + +def _warn(message: str) -> None: + from vtkmodules.vtkCommonCore import vtkOutputWindow + vtkOutputWindow.GetInstance().DisplayWarningText(f"encoding3d: {message}\n") + + +def _annotation(proxy) -> str: + return proxy.GetAnnotation(DRAWN) if proxy.HasAnnotation(DRAWN) else None + + +def _input(proxy): + """The proxy feeding *proxy*'s Input (None: none).""" + if proxy is None or proxy.GetProperty("Input") is None: + return None + helper = _helper(proxy, "Input") + return helper.GetAsProxy(0) if helper.GetNumberOfElements() else None + + +def _drawable(representation) -> bool: + source = _input(representation) + return (representation.GetProperty("BlockColorArrayNames") is not None and source is not None + and source.GetXMLName() in (READER, SELECT_AT_SCALE)) + + +def reader_of(proxy): + """The encoding3d reader *proxy* is (or is drawn from, through each item's Input); None: none.""" + while proxy is not None: + if proxy.GetXMLName() == READER: + return proxy + proxy = _input(proxy) + return None + + +def representations_of(source) -> list: + """The representations (render views') of *source*.""" + found = [] + for number in range(source.GetNumberOfConsumers()): + consumer = source.GetConsumerProxy(number) + if (consumer is not None and consumer.GetXMLGroup() == "representations" + and consumer.GetProperty("BlockColorArrayNames") is not None and _input(consumer) is source + and consumer not in found): + found.append(consumer) + return found + + +def _reader_views(reader) -> tuple: + """(views file path or None, configuration name, layers) of *reader*.""" + views = _views() + path = views.views_path(_text(reader, "FileName"), _text(reader, "Views") or None) + if path is None: + return None, None, [] + name, layers = views.configuration(views.read_views(path), _text(reader, "Configuration") or None) + return path, name, layers + + +def _signature(reader) -> str: + path, name, _ = _reader_views(reader) + return json.dumps([path, name, os.path.getmtime(path) if path else None]) + + +def _cell_arrays(source, port: int = 0) -> set: + information = source.GetDataInformation(port).GetAttributeInformation(CELLS) + return {information.GetArrayInformation(number).GetName() for number in range(information.GetNumberOfArrays())} + + +# ── The layers drawn ───────────────────────────────────────────────────────── + +def _colour_map(array: str): + """The colour map (lookup table proxy) ParaView shares among the items coloured by *array*.""" + manager = _colour_maps().vtkSMTransferFunctionManager() + return manager.GetColorTransferFunction(array, _proxy_manager()) + + +def _apply_preset(colour_map, colormap: str) -> None: + presets = _colour_maps().vtkSMTransferFunctionPresets.GetInstance() + for name in _views().preset_names(colormap): + if presets.HasPreset(name): + colour_map.ApplyPreset(name, True) + return + _warn(f"no colour map preset '{colormap}' in this ParaView, its default kept") + + +def draw(representation, rescale: bool = None) -> bool: + """*representation* (of an encoding3d reader with views, or of a selection drawn from one) drawn as the layers of + the reader's configuration, each layer its block; whether it was. A colour map following the data is fitted to it + (not for a selection, unless *rescale*: the scene's range kept).""" + source = _input(representation) + reader = reader_of(source) + if reader is None: + return False + path, _, layers = _reader_views(reader) + representation.SetAnnotation(DRAWN, _signature(reader)) + if path is None: # no views: ParaView's own display + return False + if source.GetXMLName() == SELECT_AT_SCALE: # a selection: every layer's part, the hidden ones too (a + layers = [dict(layer, visible=True) for layer in layers] # segment's section) + views = _views() + connection = _helper(representation, "Input") + port = connection.GetOutputPort(0) if connection.GetNumberOfElements() else 0 + present = _cell_arrays(source, port) + displays = views.block_displays(layers, present) + for key in ("colourings", "solid_colours", "opacities"): # by selector, as ParaView's block editors keep them + displays[key] = sorted(displays[key]) # (BlockLookupTables follows BlockColorArrayNames' order) + colour_maps = {array: _colour_map(array) for array in displays["colour_maps"]} + _set_strings(representation, "BlockSelectors", displays["selectors"]) + if displays["main_array"]: # the representation coloured: then its blocks can be + _helper(representation, "ColorArrayName").SetInputArrayToProcess(CELLS, displays["main_array"]) + _helper(representation, "LookupTable").Set(colour_maps[displays["main_array"]]) + _set_strings(representation, "BlockColorArrayNames", # (set as properties, not by the colour map helper: + [value for selector, array in displays["colourings"] # it fails while the view updates) + for value in (selector, CELLS, array)]) + tables = _helper(representation, "BlockLookupTables") + tables.RemoveAllValues() + for _, array in displays["colourings"]: + tables.Add(colour_maps[array]) + _set_strings(representation, "BlockUseSeparateColorMaps", + [value for selector, _ in displays["colourings"] for value in (selector, 0)]) + _set_strings(representation, "BlockColors", + [value for selector, colour in displays["solid_colours"] for value in [selector] + list(colour)]) + _set_strings(representation, "BlockOpacities", + [value for selector, opacity in displays["opacities"] for value in (selector, opacity)]) + if representation.GetProperty("Representation") is not None: + _helper(representation, "Representation").Set(displays["representation"]) + representation.UpdateVTKObjects() + if rescale is None: + rescale = source.GetXMLName() == READER + information = source.GetDataInformation(port) + for array, colour in displays["colour_maps"].items(): + colour_map = colour_maps[array] + _apply_preset(colour_map, colour["colormap"]) + if colour["range"] is None: + _helper(colour_map, "AutomaticRescaleRangeMode").Set(GROW_ON_APPLY) + found = information.GetArrayInformation(array, CELLS) + if rescale and found is not None: # (a layer empty at this time step: not yet) + low, high = found.GetComponentRange(-1 if found.GetNumberOfComponents() > 1 else 0) + colour_map.RescaleTransferFunction(low, high if high > low else low + 1e-12 * max(1., abs(low))) + else: + _helper(colour_map, "AutomaticRescaleRangeMode").Set(NEVER) + colour_map.RescaleTransferFunction(*colour["range"]) + _helper(colour_map, "UseLogScale").Set(int(colour["log"])) + colour_map.UpdateVTKObjects() + for view in _views_of(representation): + show_scalar_bars(representation, view, displays, present) + return True + + +def _views_of(representation) -> list: + return [view for view in _proxies("views") + if view.GetProperty("Representations") is not None and representation in _represented(view)] + + +def show_scalar_bars(representation, view, displays: dict = None, present: set = None) -> None: + """The colour legends of the visible coloured layers of *representation* shown in *view* (those whose array its + data holds, *present*).""" + if present is None: + present = _cell_arrays(_input(representation)) + if displays is None: + displays = _views().block_displays(_reader_views(reader_of(_input(representation)))[2], present) + shown = [selector for selector, array in displays["colourings"] + if selector in displays["selectors"] and array in present] + with _quiet(): # (a layer empty at this time step: no legend yet) + for selector in shown: + _colour_maps().vtkSMColorMapEditorHelper.SetBlocksScalarBarVisibility(representation, view, [selector], + True) + + +# ── Saving the views ───────────────────────────────────────────────────────── + +def _proxy_of(algorithm): + """The pipeline item (source proxy) whose server object is *algorithm* (the built-in server's).""" + if hasattr(algorithm, "GetXMLName"): # a proxy already + return algorithm + address = algorithm.GetAddressAsString("vtkObject") + for proxy in _proxies("sources"): + found = proxy.GetClientSideObject() + if found is not None and found.GetAddressAsString("vtkObject") == address: + return proxy + raise RuntimeError("its item is not in this session (a remote server: the buttons need ParaView's built-in one)") + + +def _active_view(): + proxy_manager = _proxy_manager() + selection = proxy_manager.GetSelectionModel("ActiveView") if proxy_manager is not None else None + return selection.GetCurrentProxy() if selection is not None else None + + +def _representation_in(source, view): + """*source*'s representation in *view* (else its first one; None: none).""" + found = representations_of(source) + in_view = [representation for representation in found if view is not None and representation in _represented(view)] + return (in_view or found or [None])[0] + + +def representation_state(representation) -> dict: + """The properties displays_from_blocks reads, from *representation*.""" + state = {name: _strings(representation, name) + for name in ("BlockSelectors", "BlockColorArrayNames", "BlockColors", "BlockOpacities")} + state["ColorArrayName"] = _helper(representation, "ColorArrayName").GetInputArrayNameToProcess() or None + diffuse = _helper(representation, "DiffuseColor") + state["DiffuseColor"] = [diffuse.GetAsDouble(number) for number in range(3)] + state["Opacity"] = _helper(representation, "Opacity").GetAsDouble(0) + state["Representation"] = _text(representation, "Representation") or None + named = state["BlockColorArrayNames"] + arrays = [named[start + 2] for start in range(0, len(named) - 2, 3) if named[start + 2]] + tables = _helper(representation, "BlockLookupTables") + own = {array: tables.GetAsProxy(number) for number, array in enumerate(arrays)} \ + if tables.GetNumberOfElements() == len(arrays) else {} + state["colour_maps"] = {} + for array in arrays + ([state["ColorArrayName"]] if state["ColorArrayName"] else []): + colour_map = own.get(array) or _colour_map(array) + points = _helper(colour_map, "RGBPoints") + values = [points.GetAsDouble(number) for number in range(points.GetNumberOfElements())] + follows = _helper(colour_map, "AutomaticRescaleRangeMode").GetAsInt(0) != NEVER + state["colour_maps"][array] = (None if follows or len(values) < 8 else [values[0], values[-4]], + bool(_helper(colour_map, "UseLogScale").GetAsInt(0))) + return state + + +def save_views(algorithm, configuration: str = None) -> str: + """The reader's "Save Views": its layers' display in the active view (else its first representation) written + into its views file, over its configuration or into *configuration*, made the active one; the file's path.""" + views = _views() + reader = _proxy_of(algorithm) + path, name, _ = _reader_views(reader) + if path is None: + raise RuntimeError(f"no views file for {_text(reader, 'FileName')} (a views.json next to it, or Views)") + representation = _representation_in(reader, _active_view()) + if representation is None: + raise RuntimeError("the reader is shown in no view: nothing to save") + document = views.read_views(path) + layers = document["configurations"][name]["layers"] + displays = views.displays_from_blocks(layers, representation_state(representation)) + saved = [views.layer_from_display(layer, display) for layer, display in zip(layers, displays)] + target = configuration or name + views.write_views(path, views.save_configuration(document, target, saved)) + if target != name: + _helper(reader, "Configuration").Set(target) + reader.UpdateVTKObjects() + reader.UpdatePropertyInformation() # the new configuration in the drop-down + signature = _signature(reader) # drawn as saved: not drawn again + for source in [reader] + _selections_of(reader): + for drawn in representations_of(source): + drawn.SetAnnotation(DRAWN, signature) + print(f"encoding3d views: configuration '{target}' saved in {path}") + return path + + +def _selections_of(reader) -> list: + return [proxy for proxy in _proxies("sources") + if proxy.GetXMLName() == SELECT_AT_SCALE and reader_of(proxy) is reader] + + +# ── Showing a selection: isolated, in a panel, put back ────────────────────── + +def _simple(): + """paraview.simple, on the active session (the GUI's, when a button runs it).""" + from paraview import servermanager, simple + if servermanager.ActiveConnection is None: + servermanager.SetActiveConnection(servermanager.GetConnectionFromSession( + _server_manager().vtkSMProxyManager.GetProxyManager().GetActiveSession())) + return simple + + +def _python_proxy(proxy): + from paraview import servermanager + return servermanager._getPyProxy(proxy) + + +def _name_of(proxy) -> str: + return _proxy_manager().GetProxyName("sources", proxy) + + +def _named_views() -> dict: + """{registration name: view proxy} of the session's views.""" + proxy_manager = _proxy_manager() + return {proxy_manager.GetProxyName("views", view): view for view in _proxies("views")} + + +def _steps(view) -> list: + return json.loads(view.GetAnnotation(STATE)) if view.HasAnnotation(STATE) else [] + + +def _keep_steps(view, steps: list) -> None: + if steps: + view.SetAnnotation(STATE, json.dumps(steps)) + elif view.HasAnnotation(STATE): + view.RemoveAnnotation(STATE) + + +def shown_steps() -> list: + """[(view proxy, step)] of every selection shown, the last shown last.""" + found = [(view, step) for view in _proxies("views") for step in _steps(view)] + return sorted(found, key=lambda pair: pair[1]["order"]) + + +def _upstream_names(proxy) -> set: + names = set() + proxy = _input(proxy) + while proxy is not None: + names.add(_name_of(proxy)) + proxy = _input(proxy) + return names + + +def _visible_items(view) -> dict: + """{item name: visible} of the pipeline items drawn in *view*.""" + found = {} + for representation in _represented(view): + source = _input(representation) if representation is not None else None + name = _name_of(source) if source is not None else None + if name: + found[name] = found.get(name, False) or bool(_helper(representation, "Visibility").GetAsInt(0)) + return found + + +def _selection(algorithm): + proxy = _proxy_of(algorithm) + if proxy.GetXMLName() != SELECT_AT_SCALE: + raise RuntimeError("not an 'encoding3d Select at scale' item") + return proxy + + +def _show(selection, mode: str, view=None) -> None: + simple, views = _simple(), _views() + view = view or _active_view() + if view is None: + raise RuntimeError("no active view") + above = _upstream_names(selection) + while shown_steps() and shown_steps()[-1][1]["selection"] not in above: + put_back() # not drawn from the last selection shown: put it back + selection_item, view_item = _python_proxy(selection), _python_proxy(view) + picked_on = _input(selection) + if picked_on is not None: + simple.ClearSelection(_python_proxy(picked_on)) # the picked cells' highlight (the filter keeps a copy) + hidden, bars, panel = [], [], None + if mode == "isolate": + simple.Show(selection_item, view_item) + hidden = views.isolated(_visible_items(view), [_name_of(selection)]) + for name in hidden: + item = simple.FindSource(name) + display = simple.GetDisplayProperties(item, view_item) + if display.LookupTable is not None and simple.GetScalarBar(display.LookupTable, view_item).Visibility: + bars.append(name) + simple.Hide(item, view_item) + simple.HideUnusedScalarBars(view_item) + else: + panel = _panel(view_item) + display = simple.Show(selection_item, panel) + show_scalar_bars(display.SMProxy, panel.SMProxy) + panel.ResetCamera() + simple.Render(panel) + step = {"selection": _name_of(selection), "hidden": hidden, "scalar_bars": bars, + "panel": None if panel is None else _proxy_manager().GetProxyName("views", panel.SMProxy), + "order": time.time()} + _keep_steps(view, _steps(view) + [step]) + simple.SetActiveView(view_item) + simple.Render(view_item) + print(f"encoding3d selection shown ({mode}): {_name_of(selection)}, scale {_text(selection, 'Scale')}") + + +def isolate(algorithm, view=None) -> None: + """The selection ("Select at scale" item, or its server object) alone in *view* (the active one).""" + _show(_selection(algorithm), "isolate", view) + + +def show_in_panel(algorithm, view=None) -> None: + """The selection in a new render view beside *view* (the active one).""" + _show(_selection(algorithm), "panel", view) + + +def _panel(view): + """A new render view beside *view*, at its time, its camera to start with.""" + simple = _simple() + panel = simple.CreateView("RenderView") + layout = simple.GetLayout(view) + if layout is None: + layout = simple.CreateLayout("encoding3d") + layout.AssignView(0, view) + location = layout.GetViewLocation(view) + layout.SplitHorizontal(location, 0.5) + layout.AssignView(2 * location + 2, panel) + panel.ViewSize = list(view.ViewSize) + panel.CameraPosition, panel.CameraFocalPoint = list(view.CameraPosition), list(view.CameraFocalPoint) + panel.CameraViewUp = list(view.CameraViewUp) + panel.ViewTime = view.ViewTime + return panel + + +def put_back(algorithm=None) -> bool: + """One step back: the last selection shown put back (hidden, or its panel closed), what it hid shown again; + whether there was one.""" + simple = _simple() + steps = shown_steps() + if not steps: + return False + view, step = steps[-1] + view_item = _python_proxy(view) + for name in step["hidden"]: + item = simple.FindSource(name) + if item is not None: + display = simple.Show(item, view_item) + if name in step.get("scalar_bars", []) and display.LookupTable is not None: + simple.GetScalarBar(display.LookupTable, view_item).Visibility = 1 # its legend, as Isolate found it + selection = simple.FindSource(step["selection"]) + if step["panel"]: + panel = _named_views().get(step["panel"]) + if panel is not None: + panel_item = _python_proxy(panel) + layout = simple.GetLayout(panel_item) + if layout is not None: + location = layout.GetViewLocation(panel_item) + layout.RemoveView(panel_item) + if location > 0: + layout.Collapse(location) + simple.Delete(panel_item) + elif selection is not None: + simple.Hide(selection, view_item) + _keep_steps(view, [kept for kept in _steps(view) if kept["order"] != step["order"]]) + simple.Render(view_item) + print(f"encoding3d selection put back: {step['selection']}") + return True diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py deleted file mode 100644 index f133287..0000000 --- a/paraview/fspm_tf_reader.py +++ /dev/null @@ -1,335 +0,0 @@ -""" -fspm-TF reader for ParaView: opens a frame (.glb) or a sequence (scene.fspm.json, its frames as time steps), with -every variable of every scale as data arrays. - - ParaView ≥ 5.10: Tools > Manage Plugins > Load New..., choose this file (tick "Auto Load" to keep it); then - File > Open a .glb or a scene.fspm.json and pick "fspm-TF reader". - -The reader needs openalea.fspm_tf: found in ParaView's Python when installed there, else in the bindings/python/src -folder of the fspm-TF clone holding this file (nothing to install: the codec needs numpy only, which ParaView ships). - -What it shows, one block per data structure: - - mesh its triangles; cell data: for each entity set the mesh represents (by feature IDs), the entity's - id and every numeric variable of it, named "." (NaN where a triangle has no - entity at that scale); point data: the display copies - for an entity set with segment ends (the standard names proximal_end_x ... distal_end_z, or - segments columns x1 ... z2), one line per entity with its variables as cell data (a Tube filter on - "radius" draws roots) - grid a regular grid as image data (z pointing down when the grid says so), an adaptive grid as - hexahedra, the cells' variables as cell data - -Labels and other text columns become integer codes (".label_code", the names in the block's field data). -The graph's links (complex, parent, edge_type) and the columns of role "setting" (a platform's own bookkeeping, e.g. -metafspm's MPG links) are left out unless "Show Settings" is ticked. -Data structures are placed by their origin. - -frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests. -""" -import os -import sys - -import numpy as np -from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray -from vtkmodules.vtkCommonCore import vtkPoints, vtkStringArray -from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkImageData, vtkMultiBlockDataSet, - vtkPolyData, vtkUnstructuredGrid) -from vtkmodules.vtkCommonExecutionModel import vtkStreamingDemandDrivenPipeline - - -def _fspm_tf(): - try: - from openalea import fspm_tf - except ImportError: - source = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "bindings", "python", "src") - if source not in sys.path: - sys.path.insert(0, source) - from openalea import fspm_tf - return fspm_tf - - -SEGMENT_ENDS = (("proximal_end_x", "x1"), ("proximal_end_y", "y1"), ("proximal_end_z", "z1"), - ("distal_end_x", "x2"), ("distal_end_y", "y2"), ("distal_end_z", "z2")) - - -# ── Frames to VTK data sets ────────────────────────────────────────────────── - -def _array(name: str, values: np.ndarray): - values = np.ascontiguousarray(values) - array = numpy_to_vtk(values, deep=True) - array.SetName(name) - return array - - -def _numeric(column: np.ndarray) -> bool: - return column.dtype != object and column.dtype.kind in "fiub" - - -def _codes(column: np.ndarray) -> tuple: - names = sorted(set(column.tolist())) - code_of = {name: code for code, name in enumerate(names)} - return np.array([code_of[value] for value in column.tolist()], dtype=np.int64), names - - -LINKS = ("complex", "parent", "edge_type") - - -def _shown(entity_set, name: str, settings: bool) -> bool: - """Whether column *name* is shown: the variables, the id and the label; the graph's links and the columns of role - "setting" (a platform's own bookkeeping) only with *settings*.""" - if settings: - return True - declared = entity_set.variables.get(name) - return name not in LINKS and (declared is None or declared.role != "setting") - - -def _add_columns(data, entity_set, rows: np.ndarray, field_data, settings: bool = False) -> None: - """Cell data of *entity_set*'s columns, one value per cell from *rows* (-1: no entity, NaN).""" - valid = rows >= 0 - taken = np.where(valid, rows, 0) - for name, column in entity_set.columns.items(): - if not _shown(entity_set, name, settings): - continue - label = f"{entity_set.name}.{name}" - if column.dtype == object: - if name not in ("label", "relation") and len(set(column.tolist())) > 64: - continue # free text: not a colouring - codes, names = _codes(column) - values = np.where(valid, codes[taken], -1) - data.AddArray(_array(f"{label}_code", values)) - listed = vtkStringArray() - listed.SetName(f"{label}_names") - for item in names: - listed.InsertNextValue(str(item)) - field_data.AddArray(listed) - continue - if not _numeric(column): - continue - values = column[taken].astype(np.float64) # float64: NaN where no entity - values = np.where(valid.reshape((-1,) + (1,) * (values.ndim - 1)), values, np.nan) - data.AddArray(_array(label, values)) - - -def _transform(points: np.ndarray, origin: np.ndarray) -> np.ndarray: - if np.array_equal(origin, np.eye(4)): - return points - return points @ origin[:3, :3].T + origin[:3, 3] - - -def _mesh(mesh, data_structure, settings: bool = False): - polydata = vtkPolyData() - points = vtkPoints() - points.SetData(numpy_to_vtk(np.ascontiguousarray(_transform(mesh.positions, data_structure.origin)), deep=True)) - polydata.SetPoints(points) - corners = mesh.indices if mesh.indices is not None else np.arange(len(mesh.positions)) - triangles = corners.reshape(-1, 3) - offsets = np.arange(0, 3 * len(triangles) + 1, 3, dtype=np.int64) - cells = vtkCellArray() - cells.SetData(numpy_to_vtkIdTypeArray(offsets, deep=True), - numpy_to_vtkIdTypeArray(np.ascontiguousarray(triangles.reshape(-1), dtype=np.int64), deep=True)) - polydata.SetPolys(cells) - for set_name, rows in mesh.feature_rows.items(): - _add_columns(polydata.GetCellData(), data_structure.entity_set(set_name), rows[triangles[:, 0]], - polydata.GetFieldData(), settings) - for name, values in mesh.display.items(): - polydata.GetPointData().AddArray(_array(f"display.{name}", np.asarray(values, dtype=np.float32))) - return polydata - - -def _segment_columns(entity_set) -> list: - by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() - if variable.standard_name} - found = [] - for standard, plain in SEGMENT_ENDS: - name = by_standard.get(standard, plain) - if name not in entity_set.columns or not _numeric(entity_set.columns[name]): - return [] - found.append(name) - return found - - -def _segments(entity_set, columns: list, data_structure, settings: bool = False): - ends = np.stack([entity_set.columns[name].astype(np.float64) for name in columns], axis=1) - count = entity_set.count - points_array = _transform(np.concatenate([ends[:, :3], ends[:, 3:]]), data_structure.origin) - polydata = vtkPolyData() - points = vtkPoints() - points.SetData(numpy_to_vtk(np.ascontiguousarray(points_array), deep=True)) - polydata.SetPoints(points) - connectivity = np.stack([np.arange(count), np.arange(count) + count], axis=1).reshape(-1).astype(np.int64) - cells = vtkCellArray() - cells.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 2 * count + 1, 2, dtype=np.int64), deep=True), - numpy_to_vtkIdTypeArray(connectivity, deep=True)) - polydata.SetLines(cells) - _add_columns(polydata.GetCellData(), entity_set, np.arange(count), polydata.GetFieldData(), settings) - return polydata - - -def _regular_grid(data_structure, settings: bool = False): - grid = data_structure.grid - cells = data_structure.entity_set(grid.cells) - dimensions = len(grid.shape) - shape = tuple(grid.shape) + (1,) * (3 - dimensions) - spacing = tuple(grid.spacing) + (1.,) * (3 - dimensions) - origin = tuple(grid.origin) + (0.,) * (3 - dimensions) - image = vtkImageData() - image.SetDimensions(*(n + 1 for n in shape)) - image.SetSpacing(*spacing) - image.SetOrigin(*origin) - if not grid.z_up and dimensions == 3: - image.SetDirectionMatrix(1., 0., 0., 0., 1., 0., 0., 0., -1.) # z pointing down - order = np.arange(int(np.prod(shape))).reshape(shape).ravel(order="F") # VTK: the first axis varies fastest - _add_columns(image.GetCellData(), cells, order, image.GetFieldData(), settings) - return image - - -def _adaptive_grid(data_structure, settings: bool = False): - grid = data_structure.grid - cells = data_structure.entity_set(grid.cells) - level = cells.columns["level"].astype(np.int64) - index = cells.columns["index"].astype(np.float64) - size = np.asarray(grid.spacing, dtype=np.float64)[None, :] / (2. ** level)[:, None] - lower = np.asarray(grid.origin, dtype=np.float64)[None, :] + index * size - upper = lower + size - if not grid.z_up: - lower[:, 2], upper[:, 2] = -upper[:, 2], -lower[:, 2] - corners = [] - for z in (0, 1): - for y, x in ((0, 0), (0, 1), (1, 1), (1, 0)): # VTK hexahedron order - corners.append(np.stack([np.where(x, upper[:, 0], lower[:, 0]), np.where(y, upper[:, 1], lower[:, 1]), - np.where(z, upper[:, 2], lower[:, 2])], axis=1)) - points_array = np.stack(corners, axis=1).reshape(-1, 3) - unstructured = vtkUnstructuredGrid() - points = vtkPoints() - points.SetData(numpy_to_vtk(np.ascontiguousarray(points_array), deep=True)) - unstructured.SetPoints(points) - count = cells.count - cell_array = vtkCellArray() - cell_array.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 8 * count + 1, 8, dtype=np.int64), deep=True), - numpy_to_vtkIdTypeArray(np.arange(8 * count, dtype=np.int64), deep=True)) - unstructured.SetCells(VTK_HEXAHEDRON, cell_array) - _add_columns(unstructured.GetCellData(), cells, np.arange(count), unstructured.GetFieldData(), settings) - return unstructured - - -def _named(blocks: vtkMultiBlockDataSet, data_set, name: str) -> None: - number = blocks.GetNumberOfBlocks() - blocks.SetBlock(number, data_set) - blocks.GetMetaData(number).Set(vtkMultiBlockDataSet.NAME(), name) - - -def frame_to_vtk(frame, settings: bool = False) -> vtkMultiBlockDataSet: - """The frame as a multiblock data set: one block per data structure, holding its meshes, its entity sets with - segment ends as lines, and its grid. *settings*: also the graph's links (complex, parent, edge_type) and the - columns of role "setting".""" - output = vtkMultiBlockDataSet() - for data_structure in frame.data_structures: - blocks = vtkMultiBlockDataSet() - for mesh in data_structure.meshes: - _named(blocks, _mesh(mesh, data_structure, settings), f"mesh {mesh.name}") - for entity_set in data_structure.entity_sets: - columns = _segment_columns(entity_set) - if columns: - _named(blocks, _segments(entity_set, columns, data_structure, settings), f"{entity_set.name} segments") - if data_structure.grid is not None: - build = _regular_grid if data_structure.grid.kind == "regular" else _adaptive_grid - _named(blocks, build(data_structure, settings), "grid") - if blocks.GetNumberOfBlocks(): - _named(output, blocks, data_structure.id) - return output - - -class Source: - """A .glb frame or a scene.fspm.json sequence: its times, and its frame at a time.""" - - def __init__(self, path: str): - self.path = path - fspm_tf = _fspm_tf() - if path.endswith(".json"): - index = fspm_tf.read_index(path) - self.paths = [entry["path"] for entry in index["frames"]] - self.times = [float(entry["time"]) for entry in index["frames"]] - else: - self.paths = [path] - self.times = [float(fspm_tf.read(path).time)] - self._cache = {} - - def frame_at(self, time: float = None): - """The frame of the last time not after *time* (the first frame for None or an earlier time).""" - number = 0 - if time is not None: - earlier = [n for n, frame_time in enumerate(self.times) if frame_time <= time + 1e-12 * max(1., abs(time))] - number = earlier[-1] if earlier else 0 - if number not in self._cache: - self._cache = {number: _fspm_tf().read(self.paths[number])} - return self._cache[number] - - -# ── The ParaView reader ────────────────────────────────────────────────────── - -try: - from paraview.util.vtkAlgorithm import VTKPythonAlgorithmBase, smdomain, smhint, smproperty, smproxy -except ImportError: # VTK without ParaView: frame_to_vtk and Source only - VTKPythonAlgorithmBase = None - -if VTKPythonAlgorithmBase is not None: - @smproxy.reader(name="fspmTFReader", label="fspm-TF reader", extensions="glb json", - file_description="fspm-TF frames (.glb) and sequences (scene.fspm.json)") - class FspmTFReader(VTKPythonAlgorithmBase): - def __init__(self): - VTKPythonAlgorithmBase.__init__(self, nInputPorts=0, nOutputPorts=1, outputType="vtkMultiBlockDataSet") - self._file_name, self._source, self._settings = None, None, False - - @smproperty.stringvector(name="FileName") - @smdomain.filelist() - @smhint.filechooser(extensions="glb json", file_description="fspm-TF frames and sequences") - def SetFileName(self, name): - if name != self._file_name: - self._file_name, self._source = name, None - self.Modified() - - @smproperty.xml(""" - - Also the graph's links (complex, parent, edge_type) and the columns of - role "setting" (a platform's own bookkeeping). - """) - def SetShowSettings(self, value): - if bool(value) != self._settings: - self._settings = bool(value) - self.Modified() - - def _opened(self): - if self._source is None and self._file_name: - self._source = Source(self._file_name) - return self._source - - @smproperty.doublevector(name="TimestepValues", information_only="1", si_class="vtkSITimeStepsProperty") - def GetTimestepValues(self): - source = self._opened() - return source.times if source is not None and len(source.times) > 1 else None - - def RequestInformation(self, request, in_info, out_info): - information = out_info.GetInformationObject(0) - pipeline = vtkStreamingDemandDrivenPipeline - information.Remove(pipeline.TIME_STEPS()) - information.Remove(pipeline.TIME_RANGE()) - source = self._opened() - if source is not None and len(source.times) > 1: - for time in source.times: - information.Append(pipeline.TIME_STEPS(), time) - information.Append(pipeline.TIME_RANGE(), source.times[0]) - information.Append(pipeline.TIME_RANGE(), source.times[-1]) - return 1 - - def RequestData(self, request, in_info, out_info): - information = out_info.GetInformationObject(0) - pipeline = vtkStreamingDemandDrivenPipeline - time = information.Get(pipeline.UPDATE_TIME_STEP()) if information.Has(pipeline.UPDATE_TIME_STEP()) \ - else None - frame = self._opened().frame_at(time) - output = vtkMultiBlockDataSet.GetData(out_info, 0) - output.ShallowCopy(frame_to_vtk(frame, settings=self._settings)) - if time is not None: - output.GetInformation().Set(output.DATA_TIME_STEP(), frame.time) - return 1 diff --git a/paraview/plugin_encoding3d.py b/paraview/plugin_encoding3d.py new file mode 100644 index 0000000..b455830 --- /dev/null +++ b/paraview/plugin_encoding3d.py @@ -0,0 +1,883 @@ +""" +encoding3d's ParaView plugin: the "encoding3d reader" opens a frame (.glb) or a sequence (scene.fspm.json, its frames as +time steps), with every variable of every scale as data arrays, and draws its views (views.json) layer by layer; the +filter "encoding3d Select at scale" grows picked cells to their plant, axis, organ ... (paraview/README.md, the how-to). + + ParaView >= 5.11: Tools > Manage Plugins > Load New..., choose this file (tick "Auto Load" to keep it); then + File > Open a .glb or a scene.fspm.json and pick "encoding3d reader". + +The reader needs openalea.encoding3d: found in ParaView's Python when installed there, else in the bindings/python/src +folder of the encoding3d clone holding this file (nothing to install: the codec needs numpy only, which ParaView ships). + +What it shows, one block per data structure: + + mesh its triangles; cell data: for each entity set the mesh represents (by feature IDs), the entity's + id and every numeric variable of it, named "." (NaN where a triangle has no + entity at that scale); point data: the display copies + for an entity set with segment ends (the standard names proximal_end_x ... distal_end_z, or + segments columns x1 ... z2), one line per entity with its variables as cell data (a Tube filter on + "radius" draws roots) + grid a regular grid as image data (z pointing down when the grid says so), an adaptive grid as + hexahedra, the cells' variables as cell data + +Labels and other text columns become integer codes (".label_code", the names in the block's field data). +The graph's links (complex, parent, edge_type) and the columns of role "setting" (a platform's own bookkeeping, e.g. +metafspm's MPG links) are left out unless "Show Settings" is ticked. +Data structures are placed by their origin. + +With views (specification §8.1: a views.json next to the file, or the one "Views" names), the output is instead one +block per layer of the chosen "Configuration" (the file's active one by default), named by the layer: only its +selected entities (a mesh's triangles whose rows at the select entity set match, the segments whose rows match, the +grid's cells, the others hidden), its colour variable as the active scalars, a segments layer with a "tube" as tubes +(radius also as point data "tube_radius"); hidden layers are there too, and a layer whose names a frame lacks is an +empty block. The plugin then draws each layer as its block, with its own colour variable, colour map, range, log +scale or solid colour, opacity and visibility (encoding3d_displays.py, next to this file: it watches the +representations ParaView makes, and draws them again when "Configuration" changes); the MultiBlock Inspector edits a +layer, the reader's "Save Views" button writes the display back into the views file ("Save As": a new configuration). + +Every cell also has the ids of the entities containing its own (".id" at each coarser scale, through the +complex links: a root segment's Organ, Axis and Plant), and every block its data structure (field data +encoding3d_data_structure) and its graph's scales (encoding3d_scales, coarser first). The plugin's filter "encoding3d Select +at scale" (Filters > encoding3d), applied with cells picked on the reader or any item drawn from it, grows them to their +plant, axis, organ ... ("Scale", a drop-down of the scales) with everything it holds at finer scales, in every block +of the reader (every layer, the hidden ones included), the same blocks with their arrays, drawn as the layers; the +picked entities are followed through the time steps. Its buttons Isolate (the other items hidden), Show In Panel (a +render view of its own) and Put Back (one step back) show it. + +frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests; picked_cells and +grow_selection, the filter's. +""" +import os +import sys + +import numpy as np +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray, vtk_to_numpy +from vtkmodules.vtkCommonCore import vtkOutputWindow, vtkPoints, vtkStringArray +from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkDataObject, vtkDataObjectTree, + vtkDataSetAttributes, vtkImageData, vtkMultiBlockDataSet, vtkPolyData, + vtkSelection, vtkUnstructuredGrid) +from vtkmodules.vtkCommonExecutionModel import vtkAlgorithm, vtkStreamingDemandDrivenPipeline +from vtkmodules.vtkFiltersCore import vtkThreshold +from vtkmodules.vtkFiltersExtraction import vtkExtractSelection +from vtkmodules.vtkFiltersGeometry import vtkGeometryFilter + + +def _encoding3d(): + try: + from openalea import encoding3d + except ImportError: + source = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "bindings", "python", "src") + if source not in sys.path: + sys.path.insert(0, source) + from openalea import encoding3d + return encoding3d + + +def _views(): + _encoding3d() + from openalea.encoding3d import views + return views + + +SEGMENT_ENDS = (("proximal_end_x", "x1"), ("proximal_end_y", "y1"), ("proximal_end_z", "z1"), + ("distal_end_x", "x2"), ("distal_end_y", "y2"), ("distal_end_z", "z2")) + + +# ── Frames to VTK data sets ────────────────────────────────────────────────── + +def _array(name: str, values: np.ndarray): + values = np.ascontiguousarray(values) + array = numpy_to_vtk(values, deep=True) + array.SetName(name) + return array + + +def _numeric(column: np.ndarray) -> bool: + return column.dtype != object and column.dtype.kind in "fiub" + + +def _codes(column: np.ndarray) -> tuple: + """Each text value's code among the column's sorted values; a missing value (None): -1.""" + names = sorted({value for value in column.tolist() if value is not None}) + code_of = {name: code for code, name in enumerate(names)} + return np.array([code_of.get(value, -1) for value in column.tolist()], dtype=np.int64), names + + +LINKS = ("complex", "parent", "edge_type") + + +def _shown(entity_set, name: str, settings: bool) -> bool: + """Whether column *name* is shown: the variables, the id and the label; the graph's links and the columns of role + "setting" (a platform's own bookkeeping) only with *settings*.""" + if settings: + return True + declared = entity_set.variables.get(name) + return name not in LINKS and (declared is None or declared.role != "setting") + + +def _add_columns(data, entity_set, rows: np.ndarray, field_data, settings: bool = False, forced=()) -> None: + """Cell data of *entity_set*'s columns, one value per cell from *rows* (-1: no entity, NaN); the columns + *forced* (a layer's colour variable) whatever their role.""" + valid = rows >= 0 + taken = np.where(valid, rows, 0) + for name, column in entity_set.columns.items(): + if not _shown(entity_set, name, settings) and name not in forced: + continue + label = f"{entity_set.name}.{name}" + if column.dtype == object: + if name not in ("label", "relation") and name not in forced and len(set(column.tolist())) > 64: + continue # free text: not a colouring + codes, names = _codes(column) + values = np.where(valid, codes[taken], -1) + data.AddArray(_array(f"{label}_code", values)) + listed = vtkStringArray() + listed.SetName(f"{label}_names") + for item in names: + listed.InsertNextValue(str(item)) + field_data.AddArray(listed) + continue + if not _numeric(column): + continue + values = column[taken].astype(np.float64) # float64: NaN where no entity + if np.ma.isMaskedArray(values): + values = values.filled(np.nan) # and where a value is missing + values = np.where(valid.reshape((-1,) + (1,) * (values.ndim - 1)), values, np.nan) + data.AddArray(_array(label, values)) + + +def _transform(points: np.ndarray, origin: np.ndarray) -> np.ndarray: + if np.array_equal(origin, np.eye(4)): + return points + return points @ origin[:3, :3].T + origin[:3, 3] + + +def _mesh(mesh, data_structure, settings: bool = False, kept: np.ndarray = None, forced: dict = None): + """The mesh's triangles (those *kept*, a mask, when given); *forced*: {entity set: columns}, see _add_columns.""" + polydata = vtkPolyData() + points = vtkPoints() + points.SetData(numpy_to_vtk(np.ascontiguousarray(_transform(mesh.positions, data_structure.origin)), deep=True)) + polydata.SetPoints(points) + corners = mesh.indices if mesh.indices is not None else np.arange(len(mesh.positions)) + triangles = corners.reshape(-1, 3) + if kept is not None: + triangles = triangles[kept] + offsets = np.arange(0, 3 * len(triangles) + 1, 3, dtype=np.int64) + cells = vtkCellArray() + cells.SetData(numpy_to_vtkIdTypeArray(offsets, deep=True), + numpy_to_vtkIdTypeArray(np.ascontiguousarray(triangles.reshape(-1), dtype=np.int64), deep=True)) + polydata.SetPolys(cells) + for set_name, rows in mesh.feature_rows.items(): + _add_columns(polydata.GetCellData(), data_structure.entity_set(set_name), rows[triangles[:, 0]], + polydata.GetFieldData(), settings, (forced or {}).get(set_name, ())) + ranks = {name: data_structure.entity_set(name).rank for name in mesh.feature_rows} + ranked = [name for name in ranks if ranks[name] is not None] + if ranked: # the finest scale's containing entities, where the mesh lacks them + finest = max(ranked, key=ranks.get) + _add_ancestors(polydata, data_structure, finest, mesh.feature_rows[finest][triangles[:, 0]]) + _identify(polydata, data_structure) + for name, values in mesh.display.items(): + polydata.GetPointData().AddArray(_array(f"display.{name}", np.asarray(values, dtype=np.float32))) + return polydata + + +def _add_ancestors(data_set, data_structure, set_name: str, rows: np.ndarray) -> None: + """Cell data ".id" of the entities containing each cell's entity (at *rows* of *set_name*), for the + coarser scales the cells have no array of yet: what selection by scale grows a picked cell to.""" + cell_data = data_set.GetCellData() + for name, ids in _views().ancestor_ids(data_structure, set_name, rows).items(): + label = _views().id_array(name) + if not cell_data.HasArray(label): + cell_data.AddArray(_array(label, np.where(ids >= 0, ids, np.nan).astype(np.float64))) + + +def _identify(data_set, data_structure) -> None: + """Field data: the block's data structure, and its graph's scales (coarser first).""" + views = _views() + for name, values in ((views.DATA_STRUCTURE_FIELD, [data_structure.id]), + (views.SCALES_FIELD, views.ranked_sets(data_structure))): + listed = vtkStringArray() + listed.SetName(name) + for value in values: + listed.InsertNextValue(str(value)) + data_set.GetFieldData().AddArray(listed) + + +def _segment_columns(entity_set) -> list: + by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() + if variable.standard_name} + found = [] + for standard, plain in SEGMENT_ENDS: + name = by_standard.get(standard, plain) + if name not in entity_set.columns or not _numeric(entity_set.columns[name]): + return [] + found.append(name) + return found + + +def _segments(entity_set, columns: list, data_structure, settings: bool = False, rows: np.ndarray = None, + forced=(), tube_radius: np.ndarray = None): + """One line per entity (per row of *rows* when given); *tube_radius*: a radius per entity, as point data.""" + rows = np.arange(entity_set.count) if rows is None else rows + ends = np.stack([entity_set.columns[name].astype(np.float64)[rows] for name in columns], axis=1) + count = len(rows) + points_array = _transform(np.concatenate([ends[:, :3], ends[:, 3:]]), data_structure.origin) + polydata = vtkPolyData() + points = vtkPoints() + points.SetData(numpy_to_vtk(np.ascontiguousarray(points_array), deep=True)) + polydata.SetPoints(points) + connectivity = np.stack([np.arange(count), np.arange(count) + count], axis=1).reshape(-1).astype(np.int64) + cells = vtkCellArray() + cells.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 2 * count + 1, 2, dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(connectivity, deep=True)) + polydata.SetLines(cells) + _add_columns(polydata.GetCellData(), entity_set, rows, polydata.GetFieldData(), settings, forced) + _add_ancestors(polydata, data_structure, entity_set.name, rows) + _identify(polydata, data_structure) + if tube_radius is not None: + radius = np.asarray(tube_radius, dtype=np.float64)[rows] + polydata.GetPointData().AddArray(_array(_views().TUBE_RADIUS, np.concatenate([radius, radius]))) + return polydata + + +def _regular_grid(data_structure, settings: bool = False, kept: np.ndarray = None, forced=()): + """The grid as image data; the cells not *kept* (a mask over the cells' rows) hidden.""" + grid = data_structure.grid + cells = data_structure.entity_set(grid.cells) + dimensions = len(grid.shape) + shape = tuple(grid.shape) + (1,) * (3 - dimensions) + spacing = tuple(grid.spacing) + (1.,) * (3 - dimensions) + origin = tuple(grid.origin) + (0.,) * (3 - dimensions) + image = vtkImageData() + image.SetDimensions(*(n + 1 for n in shape)) + image.SetSpacing(*spacing) + image.SetOrigin(*origin) + if not grid.z_up and dimensions == 3: + image.SetDirectionMatrix(1., 0., 0., 0., 1., 0., 0., 0., -1.) # z pointing down + order = np.arange(int(np.prod(shape))).reshape(shape).ravel(order="F") # VTK: the first axis varies fastest + _add_columns(image.GetCellData(), cells, order, image.GetFieldData(), settings, forced) + if kept is not None: + ghosts = np.where(kept[order], 0, vtkDataSetAttributes.HIDDENCELL).astype(np.uint8) + image.GetCellData().AddArray(_array(vtkDataSetAttributes.GhostArrayName(), ghosts)) + _identify(image, data_structure) + return image + + +def _adaptive_grid(data_structure, settings: bool = False, kept: np.ndarray = None, forced=()): + """The grid's leaf cells as hexahedra (those *kept*, a mask, when given).""" + grid = data_structure.grid + cells = data_structure.entity_set(grid.cells) + rows = np.arange(cells.count) if kept is None else np.flatnonzero(kept) + level = cells.columns["level"].astype(np.int64)[rows] + index = cells.columns["index"].astype(np.float64)[rows] + size = np.asarray(grid.spacing, dtype=np.float64)[None, :] / (2. ** level)[:, None] + lower = np.asarray(grid.origin, dtype=np.float64)[None, :] + index * size + upper = lower + size + if not grid.z_up: + lower[:, 2], upper[:, 2] = -upper[:, 2], -lower[:, 2] + corners = [] + for z in (0, 1): + for y, x in ((0, 0), (0, 1), (1, 1), (1, 0)): # VTK hexahedron order + corners.append(np.stack([np.where(x, upper[:, 0], lower[:, 0]), np.where(y, upper[:, 1], lower[:, 1]), + np.where(z, upper[:, 2], lower[:, 2])], axis=1)) + points_array = np.stack(corners, axis=1).reshape(-1, 3) + unstructured = vtkUnstructuredGrid() + points = vtkPoints() + points.SetData(numpy_to_vtk(np.ascontiguousarray(points_array), deep=True)) + unstructured.SetPoints(points) + count = len(rows) + cell_array = vtkCellArray() + cell_array.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 8 * count + 1, 8, dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(np.arange(8 * count, dtype=np.int64), deep=True)) + unstructured.SetCells(VTK_HEXAHEDRON, cell_array) + _add_columns(unstructured.GetCellData(), cells, rows, unstructured.GetFieldData(), settings, forced) + _identify(unstructured, data_structure) + return unstructured + + +def _named(blocks: vtkMultiBlockDataSet, data_set, name: str) -> None: + number = blocks.GetNumberOfBlocks() + blocks.SetBlock(number, data_set) + blocks.GetMetaData(number).Set(vtkMultiBlockDataSet.NAME(), name) + + +TUBE_SIDES = 8 + + +def _tubes(lines): + """The *lines* (a segments layer's: the first ends, then the second ends) as tubes of radius "tube_radius" (point + data), one closed triangle strip per line holding its cell data: the layer is drawn as tubes with no Tube filter, + and picked as such, an entity a cell.""" + count = lines.GetNumberOfCells() + points = vtk_to_numpy(lines.GetPoints().GetData()).astype(np.float64) + first, second = points[:count], points[count:] + radius = vtk_to_numpy(lines.GetPointData().GetArray(_views().TUBE_RADIUS)).astype(np.float64)[:count] + direction = second - first + length = np.linalg.norm(direction, axis=1) + axis = np.where(length[:, None] > 0, direction / np.where(length > 0, length, 1.)[:, None], [0., 0., 1.]) + helper = np.where((np.abs(axis[:, 0]) > 0.9)[:, None], [0., 1., 0.], [1., 0., 0.]) + across = np.cross(axis, helper) + across /= np.linalg.norm(across, axis=1)[:, None] + other = np.cross(axis, across) + angles = 2. * np.pi * np.arange(TUBE_SIDES) / TUBE_SIDES + normals = np.cos(angles)[None, :, None] * across[:, None, :] + np.sin(angles)[None, :, None] * other[:, None, :] + rings = np.stack([first[:, None, :] + radius[:, None, None] * normals, + second[:, None, :] + radius[:, None, None] * normals], axis=2) # segment, side, end, xyz + tubes = vtkPolyData() + tube_points = vtkPoints() + tube_points.SetData(numpy_to_vtk(np.ascontiguousarray(rings.reshape(-1, 3)), deep=True)) + tubes.SetPoints(tube_points) + per_tube = 2 * TUBE_SIDES # around the tube, back to its first side + corners = np.concatenate([np.arange(per_tube), [0, 1]]) + connectivity = (np.arange(count)[:, None] * per_tube + corners[None, :]).reshape(-1).astype(np.int64) + strips = vtkCellArray() + strips.SetData(numpy_to_vtkIdTypeArray(np.arange(0, len(corners) * count + 1, len(corners), dtype=np.int64), + deep=True), numpy_to_vtkIdTypeArray(connectivity, deep=True)) + tubes.SetStrips(strips) + tubes.GetCellData().ShallowCopy(lines.GetCellData()) + tubes.GetFieldData().ShallowCopy(lines.GetFieldData()) + tubes.GetPointData().AddArray(_array(_views().TUBE_RADIUS, np.repeat(radius, per_tube))) + point_normals = _array("Normals", np.repeat(normals, 2, axis=1).reshape(-1, 3).astype(np.float32)) + tubes.GetPointData().SetNormals(point_normals) + return tubes + + +def _layer(frame, layer: dict, settings: bool = False): + """The data set of a layer (views.configuration's, its defaults filled): its selected entities only, its colour + variable as the active scalars; empty when the frame lacks a name it gives (e.g. a mesh drawn from a later frame + on), so that every frame of a sequence has the same blocks.""" + views = _views() + report = views.check_views({"encoding3d_views": views.FORMAT, "configurations": {"layer": {"layers": [layer]}}}, frame) + if not report.ok: + return vtkPolyData() + source = layer["source"] + data_structure = frame.data_structure(source["data_structure"]) + select = layer.get("select") + chosen = (data_structure.entity_set(select["entity_set"]), select["where"]) if select else None + colour = layer.get("color_by") + forced = {colour["entity_set"]: (colour["variable"],)} if colour else {} + if source["kind"] == "mesh": + mesh = next(mesh for mesh in data_structure.meshes if mesh.name == source["mesh"]) + kept = None + if chosen is not None: + corners = mesh.indices if mesh.indices is not None else np.arange(len(mesh.positions)) + rows = mesh.feature_rows[chosen[0].name][corners.reshape(-1, 3)[:, 0]] + matching = views.selected(*chosen) + kept = (rows >= 0) & matching[np.where(rows >= 0, rows, 0)] + data_set = _mesh(mesh, data_structure, settings, kept, forced) + elif source["kind"] == "segments": + entity_set = data_structure.entity_set(source["entity_set"]) + rows = np.flatnonzero(views.selected(*chosen)) if chosen is not None else None + tube = layer.get("tube") + radius = None + if tube: # a missing radius: none (no NaN points) + radius = np.ma.filled(np.ma.asarray(entity_set.columns[tube["radius"]], dtype=np.float64), np.nan) + radius = np.nan_to_num(radius * float(tube.get("scale", 1.)), nan=0.) + data_set = _segments(entity_set, _segment_columns(entity_set), data_structure, settings, rows, + forced.get(entity_set.name, ()), radius) + if tube: + data_set = _tubes(data_set) + else: + kept = views.selected(*chosen) if chosen is not None else None + build = _regular_grid if data_structure.grid.kind == "regular" else _adaptive_grid + data_set = build(data_structure, settings, kept, forced.get(data_structure.grid.cells, ())) + array = views.colour_array(layer, {data_set.GetCellData().GetArrayName(number) + for number in range(data_set.GetCellData().GetNumberOfArrays())}) + if array is not None: + data_set.GetCellData().SetActiveScalars(array) + return data_set + + +def frame_to_vtk(frame, settings: bool = False, views=None, configuration: str = None) -> vtkMultiBlockDataSet: + """The frame as a multiblock data set: one block per data structure, holding its meshes, its entity sets with + segment ends as lines, and its grid. *settings*: also the graph's links (complex, parent, edge_type) and the + columns of role "setting". + + *views* (a views document, or the path of a views.json): instead, one block per layer of the *configuration* + (None: the active one), named by the layer, hidden layers included (see _layer).""" + if views is not None: + views_module = _views() + document = views_module.read_views(views) if isinstance(views, (str, os.PathLike)) else views + report = views_module.check_views(document) + if not report.ok: + raise ValueError(str(report)) + output = vtkMultiBlockDataSet() + for layer in views_module.configuration(document, configuration)[1]: + _named(output, _layer(frame, layer, settings), layer["name"]) + return output + output = vtkMultiBlockDataSet() + for data_structure in frame.data_structures: + blocks = vtkMultiBlockDataSet() + for mesh in data_structure.meshes: + _named(blocks, _mesh(mesh, data_structure, settings), f"mesh {mesh.name}") + for entity_set in data_structure.entity_sets: + columns = _segment_columns(entity_set) + if columns: + _named(blocks, _segments(entity_set, columns, data_structure, settings), f"{entity_set.name} segments") + if data_structure.grid is not None: + build = _regular_grid if data_structure.grid.kind == "regular" else _adaptive_grid + _named(blocks, build(data_structure, settings), "grid") + if blocks.GetNumberOfBlocks(): + _named(output, blocks, data_structure.id) + return output + + +TIME_UNITS = {"s": 1., "min": 60., "h": 3600., "d": 86400.} # in seconds: the reader's Time Unit choices + + +class Source: + """A .glb frame or a scene.fspm.json sequence: its times, and its frame at a time.""" + + def __init__(self, path: str): + self.path = path + encoding3d = _encoding3d() + if path.endswith(".json"): + index = encoding3d.read_index(path) + self.paths = [entry["path"] for entry in index["frames"]] + self.times = [float(entry["time"]) for entry in index["frames"]] + self.time_unit = index.get("timeUnit", "d") + else: + self.paths = [path] + with encoding3d.read(path, lazy=True) as frame: # its time, from the file's JSON alone + self.times = [float(frame.time)] + self.time_unit = frame.time_unit + self._cache = {} + + def factor(self, unit: str) -> float: + """The factor from the files' time unit to *unit* (one of TIME_UNITS; empty, or a unit of the files not in + TIME_UNITS: 1, the files' times).""" + if not unit or unit not in TIME_UNITS or self.time_unit not in TIME_UNITS: + return 1. + return TIME_UNITS[self.time_unit] / TIME_UNITS[unit] + + def frame_at(self, time: float = None): + """The frame of the last time not after *time* (the first frame for None or an earlier time).""" + number = 0 + if time is not None: + earlier = [n for n, frame_time in enumerate(self.times) if frame_time <= time + 1e-12 * max(1., abs(time))] + number = earlier[-1] if earlier else 0 + if number not in self._cache: # read lazily: only what the display reads is decoded + for frame in self._cache.values(): + frame.close() + self._cache = {number: _encoding3d().read(self.paths[number], lazy=True)} + return self._cache[number] + + +# ── Selection by scale ─────────────────────────────────────────────────────── + +SELECT_AT_SCALE = "encoding3dSelectAtScale" # the filter's proxy name +SELECTED = "encoding3d_selected" # the cell array _kept thresholds + + +def leaves(data) -> list: + """The data sets of *data*, a data set or a composite one, in its tree's order.""" + if data is None: + return [] + if not isinstance(data, vtkDataObjectTree): + return [data] + iterator = data.NewTreeIterator() + iterator.VisitOnlyLeavesOn() + iterator.InitTraversal() + found = [] + while not iterator.IsDoneWithTraversal(): + found.append(iterator.GetCurrentDataObject()) + iterator.GoToNextItem() + return found + + +def _texts(data_set, name: str) -> list: + array = data_set.GetFieldData().GetAbstractArray(name) + return [array.GetValue(number) for number in range(array.GetNumberOfValues())] if array is not None else [] + + +def _id_arrays(data_set) -> dict: + """{".id": values} of a data set's cells.""" + cell_data = data_set.GetCellData() + names = [cell_data.GetArrayName(number) for number in range(cell_data.GetNumberOfArrays())] + return {name: vtk_to_numpy(cell_data.GetArray(name)) for name in names if name and name.endswith(".id")} + + +def encoding3d_data(data) -> bool: + """Whether *data* holds blocks of the encoding3d reader (their data structure in field data).""" + return any(_texts(leaf, _views().DATA_STRUCTURE_FIELD) for leaf in leaves(data)) + + +def scales_of(data) -> list: + """The scales of *data*'s blocks (their field data encoding3d_scales), coarser first.""" + return _views().scale_names([_texts(leaf, _views().SCALES_FIELD) for leaf in leaves(data)]) + + +def picked_cells(data, selection) -> list: + """[(data structure, {".id": values})] of the cells of *data* (the reader's output or an item drawn from + it: an ExtractBlock, a Tube ...) in *selection* (a vtkSelection: ids, a frustum, a query ...), one entry per block + holding some.""" + if data is None or selection is None or not selection.GetNumberOfNodes(): + return [] + copied, chosen = data.NewInstance(), vtkSelection() + copied.ShallowCopy(data) + chosen.ShallowCopy(selection) + extraction = vtkExtractSelection() + extraction.PreserveTopologyOn() # the same blocks, each cell marked inside or not + extraction.SetInputData(0, copied) + extraction.SetInputData(1, chosen) + extraction.Update() + marked_data, found = extraction.GetOutputDataObject(0), [] + if isinstance(data, vtkDataObjectTree): # each block and its marks (a block without any: none) + iterator = data.NewTreeIterator() + iterator.InitTraversal() + pairs = [] + while not iterator.IsDoneWithTraversal(): + pairs.append((iterator.GetCurrentDataObject(), marked_data.GetDataSet(iterator))) + iterator.GoToNextItem() + else: + pairs = [(data, marked_data)] + for leaf, marked in pairs: + inside = marked.GetCellData().GetArray("vtkInsidedness") if marked is not None else None + data_structure = _texts(leaf, _views().DATA_STRUCTURE_FIELD) + if inside is None or not data_structure: + continue + cells = vtk_to_numpy(inside).astype(bool) + if cells.any(): + found.append((data_structure[0], {name: values[cells] for name, values in _id_arrays(leaf).items()})) + return found + + +def _kept(data_set, chosen: dict, scale: str): + """The cells of *data_set* inside the entities *chosen* at *scale* (a grid layer's hidden cells included), its + field data and active scalars kept; polygonal data stays polygonal (a Tube can follow).""" + views = _views() + data_structure = (_texts(data_set, views.DATA_STRUCTURE_FIELD) or [""])[0] + mask = views.selection_mask(data_structure, _id_arrays(data_set), chosen, scale) + if mask.size != data_set.GetNumberOfCells() or not mask.any(): # a block without id arrays, or none in it + empty = data_set.NewInstance() + empty.GetFieldData().ShallowCopy(data_set.GetFieldData()) + return empty + marked = data_set.NewInstance() + marked.ShallowCopy(data_set) + marked.GetCellData().RemoveArray(vtkDataSetAttributes.GhostArrayName()) + marked.GetCellData().AddArray(_array(SELECTED, mask.astype(np.uint8))) + threshold = vtkThreshold() + threshold.SetInputData(marked) + threshold.SetInputArrayToProcess(0, 0, 0, vtkDataObject.FIELD_ASSOCIATION_CELLS, SELECTED) + threshold.SetThresholdFunction(vtkThreshold.THRESHOLD_UPPER) + threshold.SetUpperThreshold(0.5) + threshold.SetLowerThreshold(0.5) + last = threshold + if isinstance(data_set, vtkPolyData): + last = vtkGeometryFilter() + last.SetInputConnection(threshold.GetOutputPort()) + last.Update() + output = last.GetOutput() + output.GetCellData().RemoveArray(SELECTED) + scalars = data_set.GetCellData().GetScalars() + if scalars is not None and scalars.GetName(): + output.GetCellData().SetActiveScalars(scalars.GetName()) + output.GetFieldData().ShallowCopy(data_set.GetFieldData()) + return output + + +def grow_selection(frame_data, picked: list, scale: str) -> vtkMultiBlockDataSet: + """The cells of *frame_data* (the reader's output: its blocks, a layer's hidden ones included) inside the entities + at *scale* containing the *picked* cells (picked_cells'), with everything they contain at finer scales: the same + blocks, named alike, each holding its part (empty when none). A ValueError when no picked cell has an entity at + *scale*.""" + chosen = _views().picked_entities(picked, scale) + output = vtkMultiBlockDataSet() + if isinstance(frame_data, vtkDataObjectTree): + output.CopyStructure(frame_data) + iterator = frame_data.NewTreeIterator() + iterator.InitTraversal() + while not iterator.IsDoneWithTraversal(): + output.SetDataSet(iterator, _kept(iterator.GetCurrentDataObject(), chosen, scale)) + iterator.GoToNextItem() + elif frame_data is not None: + output.SetBlock(0, _kept(frame_data, chosen, scale)) + return output + + +def frame_upstream(algorithm, port: int = 0): + """The output of the encoding3d reader upstream of *algorithm*'s input *port*, through the items drawn from it (the + farthest output holding encoding3d blocks): where selection by scale grows, in every layer, the hidden ones included; + None when there is none.""" + found = None + while algorithm.GetNumberOfInputPorts() > port and algorithm.GetNumberOfInputConnections(port): + connection = algorithm.GetInputConnection(port, 0) + producer = connection.GetProducer() + data = producer.GetOutputDataObject(connection.GetIndex()) + if encoding3d_data(data): + found = data + algorithm, port = producer, 0 + return found + + +# ── The ParaView reader ────────────────────────────────────────────────────── + +try: + from paraview.util.vtkAlgorithm import VTKPythonAlgorithmBase, smdomain, smhint, smproperty, smproxy +except ImportError: # VTK without ParaView: frame_to_vtk and Source only + VTKPythonAlgorithmBase = None + +def _displays(): + """encoding3d_displays (next to this file): how the reader's representations draw the layers, the buttons' work.""" + import encoding3d_displays + return encoding3d_displays + + +def _run(action: str, algorithm, *arguments) -> None: + """A button's *action* of encoding3d_displays on the item of *algorithm*; what goes wrong, in ParaView's output.""" + try: + getattr(_displays(), action)(algorithm, *arguments) + except Exception as error: # a button: ParaView's Output Messages, not a crash + vtkOutputWindow.GetInstance().DisplayWarningText(f"encoding3d: {error}\n") + + +if VTKPythonAlgorithmBase is not None: + try: + _encoding3d() # on ParaView's path now + except ImportError: + pass + if os.path.dirname(os.path.realpath(__file__)) not in sys.path: # and encoding3d_displays (a link: its folder) + sys.path.append(os.path.dirname(os.path.realpath(__file__))) + try: + _displays().install() # the layers drawn on each representation ParaView makes + except Exception as error: # (none in a server without a client: nothing to draw) + vtkOutputWindow.GetInstance().DisplayWarningText(f"encoding3d: layers not drawn ({error})\n") + + @smproxy.reader(name="encoding3dReader", label="encoding3d reader", extensions="glb json", + file_description="encoding3d frames (.glb) and sequences (scene.fspm.json)") + class Encoding3dReader(VTKPythonAlgorithmBase): + def __init__(self): + VTKPythonAlgorithmBase.__init__(self, nInputPorts=0, nOutputPorts=1, outputType="vtkMultiBlockDataSet") + self._file_name, self._source, self._settings = None, None, False + self._views, self._configuration, self._save_as = "", "", "" + self._time_unit = "d" + + @smproperty.stringvector(name="FileName") + @smdomain.filelist() + @smhint.filechooser(extensions="glb json", file_description="encoding3d frames and sequences") + def SetFileName(self, name): + if name != self._file_name: + self._file_name, self._source = name, None + self.Modified() + + @smproperty.xml(""" + + Also the graph's links (complex, parent, edge_type) and the columns of + role "setting" (a platform's own bookkeeping). + """) + def SetShowSettings(self, value): + if bool(value) != self._settings: + self._settings = bool(value) + self.Modified() + + @smproperty.stringvector(name="TimeUnit", label="Time Unit", number_of_elements="1", default_values="d") + @smdomain.xml(""" + + """) + def SetTimeUnit(self, unit): + """The unit of the times ParaView shows (the animation toolbar, the time inspector): the frames' times + (in their file's unit) converted, the files unchanged.""" + if (unit or "") != self._time_unit: + self._time_unit = unit or "" + self.Modified() + + def _factor(self): + source = self._opened() + return source.factor(self._time_unit) if source is not None else 1. + + @smproperty.stringvector(name="Views", default_values="") + @smdomain.filelist() + @smhint.filechooser(extensions="json", file_description="encoding3d views (views.json)") + def SetViews(self, name): + """The views file (empty: the views.json next to the file, when there is one; none: today's blocks).""" + if (name or "") != self._views: + self._views = name or "" + self.Modified() + + def _views_path(self): + return _views().views_path(self._file_name, self._views) if self._file_name else None + + @smproperty.stringvector(name="ConfigurationNames", information_only="1") + def GetConfigurationNames(self): + path = self._views_path() + return _views().configuration_names(_views().read_views(path)) if path else [] + + @smproperty.stringvector(name="Configuration", number_of_elements="1", default_values="") + @smdomain.xml(""" + + + + """) + def SetConfiguration(self, name): + """The configuration shown (empty: the views' active one).""" + if (name or "") != self._configuration: + self._configuration = name or "" + self.Modified() + + @smproperty.stringvector(name="SaveAs", label="Save As", number_of_elements="1", default_values="") + def SetSaveAs(self, name): + """The configuration "Save Views" writes (empty: the one shown).""" + self._save_as = name or "" # not Modified(): the output does not change + + @smproperty.xml(""" + Write each layer's display (visibility, colour variable or solid colour, + range, log scale, opacity, representation) into the views file: over the configuration + shown, or into the one "Save As" names, made the active one. + """) + def SaveViews(self): + _run("save_views", self, self._save_as or None) + + def _opened(self): + if self._source is None and self._file_name: + self._source = Source(self._file_name) + return self._source + + @smproperty.doublevector(name="TimestepValues", information_only="1", si_class="vtkSITimeStepsProperty") + def GetTimestepValues(self): + source = self._opened() + factor = self._factor() + return [time * factor for time in source.times] if source is not None and len(source.times) > 1 else None + + def RequestInformation(self, request, in_info, out_info): + try: + _displays().install() # a session made after the plugin was loaded + except Exception: + pass + information = out_info.GetInformationObject(0) + pipeline = vtkStreamingDemandDrivenPipeline + information.Remove(pipeline.TIME_STEPS()) + information.Remove(pipeline.TIME_RANGE()) + source = self._opened() + if source is not None and len(source.times) > 1: + factor = self._factor() + for time in source.times: + information.Append(pipeline.TIME_STEPS(), time * factor) + information.Append(pipeline.TIME_RANGE(), source.times[0] * factor) + information.Append(pipeline.TIME_RANGE(), source.times[-1] * factor) + return 1 + + def RequestData(self, request, in_info, out_info): + information = out_info.GetInformationObject(0) + pipeline = vtkStreamingDemandDrivenPipeline + time = information.Get(pipeline.UPDATE_TIME_STEP()) if information.Has(pipeline.UPDATE_TIME_STEP()) \ + else None + factor = self._factor() + frame = self._opened().frame_at(None if time is None else time / factor) + output = vtkMultiBlockDataSet.GetData(out_info, 0) + path = self._views_path() + output.ShallowCopy(frame_to_vtk(frame, settings=self._settings, views=path, + configuration=self._configuration or None)) + if time is not None: + output.GetInformation().Set(output.DATA_TIME_STEP(), frame.time * factor) + return 1 + + @smproxy.filter(name=SELECT_AT_SCALE, label="encoding3d Select at scale") + @smhint.xml("""""") + @smhint.xml("""""") + @smhint.xml("""""") + @smproperty.xml(""" + + + + + The cells picked in the view: ParaView's selection when the filter is made + ("Copy Active Selection" to take another). + + """) + @smproperty.xml(""" + + + + + The item the cells were picked on: the encoding3d reader, a layer, a selection + ... + """) + class Encoding3dSelectAtScale(VTKPythonAlgorithmBase): + """The cells picked on its input grown to their entity at Scale (a plant, an axis, an organ, a segment ...) + with everything it contains at finer scales, in every block of the encoding3d reader upstream (every layer, the + hidden ones included: a segment's section). The picked entities are kept across time steps (the plant + followed as it grows) until the selection changes; Scale re-grows them.""" + + def __init__(self): + VTKPythonAlgorithmBase.__init__(self, nInputPorts=2, nOutputPorts=1, outputType="vtkMultiBlockDataSet") + self._scale, self._scales = "", [] + self._picked, self._picked_from = [], None + + def FillInputPortInformation(self, port, info): + if port == 1: + info.Set(vtkAlgorithm.INPUT_REQUIRED_DATA_TYPE(), "vtkSelection") + info.Set(vtkAlgorithm.INPUT_IS_OPTIONAL(), 1) + else: + info.Set(vtkAlgorithm.INPUT_REQUIRED_DATA_TYPE(), "vtkDataObject") + return 1 + + @smproperty.stringvector(name="ScaleNames", information_only="1") + def GetScaleNames(self): + return self._scales + + @smproperty.stringvector(name="Scale", number_of_elements="1", default_values="") + @smdomain.xml(""" + + + + """) + def SetScale(self, name): + """The scale the picked cells grow to (empty: the coarsest, e.g. Plant).""" + if (name or "") != self._scale: + self._scale = name or "" + self.Modified() + + @smproperty.xml(""" + Show the selection alone in the active view, the other items hidden + (Put Back shows them again). + """) + def Isolate(self): + _run("isolate", self) + + @smproperty.xml(""" + Show the selection in a new render view beside the active one, its + camera fit to it; the view itself untouched (Put Back closes it). + """) + def ShowInPanel(self): + _run("show_in_panel", self) + + @smproperty.xml(""" + One step back: the last selection isolated or shown in a panel put back, + the items it hid shown again, its panel closed. + """) + def PutBack(self): + _run("put_back", self) + + def RequestInformation(self, request, in_info, out_info): + frame_data = frame_upstream(self) + if frame_data is None: + frame_data = in_info[0].GetInformationObject(0).Get(vtkDataObject.DATA_OBJECT()) + self._scales = scales_of(frame_data) + return 1 + + def RequestData(self, request, in_info, out_info): + output = vtkMultiBlockDataSet.GetData(out_info, 0) + source = in_info[0].GetInformationObject(0).Get(vtkDataObject.DATA_OBJECT()) + selection = vtkSelection.GetData(in_info[1], 0) if in_info[1].GetNumberOfInformationObjects() else None + frame_data = frame_upstream(self) or source + self._scales = scales_of(frame_data) + picked_from = (selection.GetMTime() if selection is not None else None, + self.GetInputConnection(0, 0).GetProducer().GetAddressAsString("vtkObject")) + if picked_from != self._picked_from: # a new selection: its entities picked at this time, + self._picked = picked_cells(source, selection) # then followed through the time steps + self._picked_from = picked_from + scale = self._scale or (self._scales[0] if self._scales else "") + try: + output.ShallowCopy(grow_selection(frame_data, self._picked, scale)) + except ValueError as error: + vtkOutputWindow.GetInstance().DisplayWarningText(f"encoding3d Select at scale: {error}\n") + empty = vtkMultiBlockDataSet() + if isinstance(frame_data, vtkDataObjectTree): + empty.CopyStructure(frame_data) + output.ShallowCopy(empty) + if source is not None and source.GetInformation().Has(vtkDataObject.DATA_TIME_STEP()): + output.GetInformation().Set(vtkDataObject.DATA_TIME_STEP(), + source.GetInformation().Get(vtkDataObject.DATA_TIME_STEP())) + return 1 diff --git a/pyproject.toml b/pyproject.toml index 634f4df..22d62f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,22 +8,22 @@ build-backend = "setuptools.build_meta" [tool.setuptools] include-package-data = false # force explicit declaration of data (disable automatic inclusion) # the code in bindings/python/src; the specification (spec/: its text, schemas and vocabulary) shipped as -# openalea.fspm_tf.spec. This file stays at the root, next to spec/ (a project cannot ship files outside its folder) -packages = ["openalea.fspm_tf", "openalea.fspm_tf.spec"] -package-dir = {"" = "bindings/python/src", "openalea.fspm_tf.spec" = "spec"} +# openalea.encoding3d.spec. This file stays at the root, next to spec/ (a project cannot ship files outside its folder) +packages = ["openalea.encoding3d", "openalea.encoding3d.coupling", "openalea.encoding3d.spec"] +package-dir = {"" = "bindings/python/src", "openalea.encoding3d.spec" = "spec"} [tool.setuptools.package-data] -"openalea.fspm_tf.spec" = ["*.md", "schema/*.json", "vocabulary/*.csv", "vocabulary/*.json"] +"openalea.encoding3d.spec" = ["*.md", "schema/*.json", "vocabulary/*.csv", "vocabulary/*.json"] # enable dynamic version based on git tags [tool.setuptools_scm] [project] -name = "openalea.fspm_tf" +name = "openalea.encoding3d" authors = [ { name = "Tristan Gérault", email = "tristan.gerault@inrae.fr" }, ] -description = "fspm-TF, an exchange format for whole FSPM scenes, with its Python reference codec, validator and vocabulary" +description = "encoding3d, an exchange format for whole FSPM scenes, with its Python reference codec, validator and vocabulary" readme = "README.md" license = "CECILL-C" license-files = ["LICEN[CS]E*"] @@ -52,30 +52,38 @@ dependencies = [ ] [project.optional-dependencies] +coupling = [ + "pyzmq", # the coupling's in-memory transport (openalea.encoding3d.coupling); its folder transport needs nothing +] test = [ "pytest", + "pyzmq", ] dev = [ "pytest >=6", "pytest-cov >=3", ] doc = [ - "sphinx", "pydata-sphinx-theme", "myst-parser", "sphinx-favicon", + "ipykernel", "sphinx-copybutton", + "ipython_genutils", + "nbsphinx", + "nbmake", # the notebooks are tested: pytest --nbmake doc/notebooks + "pandas", # the notebooks' tables ] [project.scripts] -fspm-tf = "openalea.fspm_tf.cli:main" +encoding3d = "openalea.encoding3d.cli:main" [project.urls] -Repository = "https://github.com/openalea/fspm-TF" -Homepage = "https://github.com/openalea/fspm-TF" +Repository = "https://github.com/openalea/encoding3d" +Homepage = "https://github.com/openalea/encoding3d" [tool.coverage.run] -source = ["openalea.fspm_tf"] +source = ["openalea.encoding3d"] branch = true [tool.coverage.report] diff --git a/spec/schema/coupling.schema.json b/spec/schema/coupling.schema.json new file mode 100644 index 0000000..6068a5c --- /dev/null +++ b/spec/schema/coupling.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openalea/encoding3d/spec/schema/coupling.schema.json", + "title": "encoding3d coupling messages", + "description": "The JSON header of a coupling session's message (specification §11). Turns and updates also carry a payload: a partial frame's GLB bytes.", + "oneOf": [ + {"$ref": "#/$defs/hello"}, {"$ref": "#/$defs/welcome"}, {"$ref": "#/$defs/turn"}, {"$ref": "#/$defs/update"}, + {"$ref": "#/$defs/error"}, {"$ref": "#/$defs/stop"} + ], + "$defs": { + "identifier": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, + "name": {"type": "string", "minLength": 1}, + "payload": {"description": "In the folder transport: the payload's file name.", "type": "string"}, + "entitySet": { + "type": "object", "required": ["dataStructure", "name", "rights"], + "properties": { + "dataStructure": {"$ref": "#/$defs/identifier"}, "name": {"$ref": "#/$defs/name"}, + "rank": {"type": ["integer", "null"]}, "rights": {"enum": ["owns", "refines", "follows"]} + }, + "additionalProperties": false + }, + "variable": { + "type": "object", "required": ["dataStructure", "entitySet", "name", "direction", "unit"], + "properties": { + "dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"$ref": "#/$defs/name"}, + "name": {"$ref": "#/$defs/name"}, "direction": {"enum": ["provides", "requires"]}, "unit": {"type": "string"}, + "standardName": {"type": "string"}, "kind": {"type": "string"}, "optional": {"type": "boolean"} + }, + "additionalProperties": false + }, + "declaration": { + "type": "object", "required": ["participant", "entitySets", "variables"], + "properties": { + "participant": {"$ref": "#/$defs/identifier"}, "platform": {"type": "string"}, + "entitySets": {"type": "array", "items": {"$ref": "#/$defs/entitySet"}}, + "variables": {"type": "array", "items": {"$ref": "#/$defs/variable"}}, + "mappings": {"type": "array", "items": {"$ref": "#/$defs/mapping"}} + }, + "additionalProperties": false + }, + "end": { + "type": "object", "required": ["dataStructure", "entitySet"], + "properties": {"dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"$ref": "#/$defs/name"}}, + "additionalProperties": false + }, + "mapping": { + "type": "object", "required": ["name", "source", "target"], + "properties": {"name": {"$ref": "#/$defs/identifier"}, "source": {"$ref": "#/$defs/end"}, + "target": {"$ref": "#/$defs/end"}}, + "additionalProperties": false + }, + "link": { + "type": "object", "required": ["receiver", "variable", "provider", "source"], + "properties": { + "receiver": {"$ref": "#/$defs/identifier"}, "variable": {"$ref": "#/$defs/name"}, + "provider": {"$ref": "#/$defs/identifier"}, "source": {"$ref": "#/$defs/name"}, + "entitySet": {"$ref": "#/$defs/name"}, "sourceEntitySet": {"$ref": "#/$defs/name"}, + "factor": {"type": "number"}, "aggregation": {"enum": ["sum", "mean", "max", "min", "copy"]}, + "interpolation": {"enum": ["hold", "linear"]}, "mapping": {"$ref": "#/$defs/identifier"} + }, + "additionalProperties": false + }, + "hello": { + "type": "object", "required": ["type", "protocol", "declaration"], + "properties": { + "type": {"const": "hello"}, "protocol": {"type": "string"}, "declaration": {"$ref": "#/$defs/declaration"}, + "links": {"type": "array", "items": {"$ref": "#/$defs/link"}}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "welcome": { + "type": "object", "required": ["type", "idOffset", "idCount", "session"], + "properties": { + "type": {"const": "welcome"}, "idOffset": {"type": "integer", "minimum": 0}, + "idCount": {"type": "integer", "minimum": 1}, + "session": { + "type": "object", "required": ["start", "interval", "timeUnit", "steps", "stages", "participants"], + "properties": { + "start": {"type": "number"}, "interval": {"type": "number", "exclusiveMinimum": 0}, + "timeUnit": {"type": "string"}, "steps": {"type": "integer", "minimum": 0}, + "stages": {"type": "array", "items": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}}, + "participants": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}, + "keepMessages": {"description": "Over a folder: every message kept (else each deleted by its reader once read).", "type": "boolean"} + } + }, + "couplings": {"type": "array", "items": {"type": "object"}}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "turn": { + "type": "object", "required": ["type", "step", "phase", "time", "until"], + "properties": { + "type": {"const": "turn"}, "step": {"type": "integer", "minimum": 0}, "phase": {"enum": ["init", "step"]}, + "time": {"type": "number"}, "until": {"type": "number"}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "update": { + "type": "object", "required": ["type", "step"], + "properties": {"type": {"const": "update"}, "step": {"type": "integer", "minimum": 0}, + "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + }, + "error": { + "type": "object", "required": ["type", "message"], + "properties": {"type": {"const": "error"}, "message": {"type": "string"}, "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + }, + "stop": { + "type": "object", "required": ["type"], + "properties": {"type": {"const": "stop"}, "reason": {"type": "string"}, "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + } + } +} diff --git a/spec/schema/fspm_scene.schema.json b/spec/schema/fspm_scene.schema.json index 018f39d..3b0c966 100644 --- a/spec/schema/fspm_scene.schema.json +++ b/spec/schema/fspm_scene.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.1/fspm_scene.schema.json", + "$id": "https://github.com/openalea/encoding3d/schema/0.3/fspm_scene.schema.json", "title": "FSPM_scene", - "description": "The FSPM_scene root extension of a glTF 2.0 asset (fspm-TF 0.1): one frame of a scene.", + "description": "The FSPM_scene root extension of a glTF 2.0 asset (encoding3d 0.3): one frame of a scene.", "type": "object", "required": ["version", "frame", "dataStructures"], "properties": { @@ -14,6 +14,19 @@ "step": {"type": "integer", "minimum": 0}}, "additionalProperties": false}, "writer": {"type": "object"}, + "update": { + "description": "Set in a partial frame, a coupling session's message (specification §11): its sender, and the entities it removed.", + "type": "object", "required": ["participant"], + "properties": { + "participant": {"$ref": "#/$defs/identifier"}, + "removed": {"type": "array", "items": { + "type": "object", "required": ["dataStructure", "entitySet", "propertyTable"], + "properties": {"dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"type": "string", "minLength": 1}, + "propertyTable": {"$ref": "#/$defs/index"}}, + "additionalProperties": false}} + }, + "additionalProperties": false + }, "dataStructures": {"type": "array", "items": {"$ref": "#/$defs/dataStructure"}}, "mappings": {"type": "array", "items": {"$ref": "#/$defs/mapping"}}, "variables": { @@ -39,6 +52,8 @@ "entitySets": {"type": "array", "items": { "type": "object", "required": ["name", "propertyTable"], "properties": {"name": {"$ref": "#/$defs/identifier"}, "rank": {"type": ["integer", "null"]}, + "scale": {"enum": ["Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", + "Cell", "Compartment", "Connection", "mixed"]}, "propertyTable": {"$ref": "#/$defs/index"}}, "additionalProperties": false}}, "relations": {"type": "array", "items": { @@ -47,6 +62,7 @@ "targetSet": {"$ref": "#/$defs/identifier"}, "propertyTable": {"$ref": "#/$defs/index"}}, "additionalProperties": false}}, "scalars": {"$ref": "#/$defs/index"}, + "labels": {"type": "object", "additionalProperties": {"type": "string"}}, "grid": {"$ref": "#/$defs/grid"}, "extras": {"type": "object"}, "meshes": {"type": "array", "items": { @@ -94,7 +110,8 @@ "role": {"enum": ["state", "input", "parameter", "derived", "setting"]}, "minimum": {"type": "number"}, "maximum": {"type": "number"}, "references": {"type": "array", "items": {"type": "string"}}, - "doi": {"type": "array", "items": {"type": "string"}}}, + "doi": {"type": "array", "items": {"type": "string"}}, + "format": {"enum": ["date", "date-time", "time"]}}, "additionalProperties": false} } } diff --git a/spec/schema/index.schema.json b/spec/schema/index.schema.json index abac4bb..64db146 100644 --- a/spec/schema/index.schema.json +++ b/spec/schema/index.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.1/index.schema.json", - "title": "fspm-TF sequence index", - "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (fspm-TF 0.1).", + "$id": "https://github.com/openalea/encoding3d/schema/0.3/index.schema.json", + "title": "encoding3d sequence index", + "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (encoding3d 0.3).", "type": "object", "required": ["fspmIndex", "timeUnit", "frames"], "properties": { diff --git a/spec/schema/views.schema.json b/spec/schema/views.schema.json new file mode 100644 index 0000000..ace8977 --- /dev/null +++ b/spec/schema/views.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openalea/encoding3d/schema/0.3/views.schema.json", + "title": "encoding3d frame views", + "description": "views.json, an optional companion of a sequence (next to its scene.fspm.json) or of a frame: named configurations of layers, each layer a part of a data structure (a mesh, segments or a grid), the entities it selects and how it is drawn (encoding3d 0.3, §8.1).", + "type": "object", + "required": ["encoding3d_views", "configurations"], + "properties": { + "encoding3d_views": {"const": 1}, + "active": {"type": "string", "minLength": 1}, + "configurations": {"type": "object", "minProperties": 1, + "additionalProperties": {"$ref": "#/$defs/configuration"}}, + "extras": {} + }, + "additionalProperties": false, + "$defs": { + "configuration": { + "type": "object", "required": ["layers"], + "properties": {"layers": {"type": "array", "items": {"$ref": "#/$defs/layer"}}, "extras": {}}, + "additionalProperties": false}, + "layer": { + "type": "object", "required": ["name", "source"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "source": {"$ref": "#/$defs/source"}, + "select": {"type": "object", "required": ["entity_set", "where"], + "properties": {"entity_set": {"type": "string", "minLength": 1}, + "where": {"type": "object", "additionalProperties": {"$ref": "#/$defs/condition"}}}, + "additionalProperties": false}, + "color_by": {"oneOf": [{"type": "null"}, + {"type": "object", "required": ["entity_set", "variable"], + "properties": {"entity_set": {"type": "string", "minLength": 1}, + "variable": {"type": "string", "minLength": 1}}, + "additionalProperties": false}]}, + "solid_color": {"type": "array", "items": {"type": "number", "minimum": 0, "maximum": 1}, + "minItems": 3, "maxItems": 3}, + "colormap": {"type": "string", "minLength": 1}, + "range": {"oneOf": [{"const": "auto"}, + {"type": "array", "items": {"type": "number"}, "minItems": 2, "maxItems": 2}]}, + "log": {"type": "boolean"}, + "opacity": {"type": "number", "minimum": 0, "maximum": 1}, + "visible": {"type": "boolean"}, + "representation": {"enum": ["Surface", "Surface With Edges", "Wireframe", "Points", "Outline"]}, + "tube": {"type": "object", "required": ["radius"], + "properties": {"radius": {"type": "string", "minLength": 1}, + "scale": {"type": "number", "exclusiveMinimum": 0}}, + "additionalProperties": false}, + "extras": {} + }, + "additionalProperties": false}, + "source": { + "type": "object", "required": ["data_structure", "kind"], + "properties": {"data_structure": {"type": "string", "minLength": 1}, + "kind": {"enum": ["mesh", "segments", "grid"]}, + "mesh": {"type": "string", "minLength": 1}, + "entity_set": {"type": "string", "minLength": 1}}, + "additionalProperties": false, + "allOf": [ + {"if": {"properties": {"kind": {"const": "mesh"}}}, "then": {"required": ["mesh"]}}, + {"if": {"properties": {"kind": {"const": "segments"}}}, "then": {"required": ["entity_set"]}}]}, + "condition": { + "description": "A value, a list of values (labels and other text by name), or a comparison such as \">=0\" or \"!=2\".", + "oneOf": [{"type": ["number", "boolean"]}, + {"type": "string", "minLength": 1}, + {"type": "array", "items": {"type": ["string", "number", "boolean"]}}]} + } +} diff --git a/spec/specification.md b/spec/specification.md index a91ec32..93eba02 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -1,8 +1,8 @@ -# fspm-TF 0.1 specification +# encoding3d 0.3 specification -Status: draft (2026-10-29). The key words MUST, SHOULD and MAY are to be read as in RFC 2119. +Status: draft (2026-11-03). 0.3 adds coupling sessions and partial frames to 0.2 (see "Changes from 0.2" at the end); a 0.1 or 0.2 file is a valid 0.3 file. The key words MUST, SHOULD and MAY are to be read as in RFC 2119 (https://www.rfc-editor.org/rfc/rfc2119). -fspm-TF is an exchange format for whole functional-structural plant model scenes: multiscale plant graphs, their +encoding3d is an exchange format for whole functional-structural plant model scenes: multiscale plant graphs, their variables and parameters, their geometry, and the environment's grids, at successive time steps. A **frame** (one time step) is one glTF 2.0 binary file (GLB). A **sequence** of frames is a folder of GLB files with an index. @@ -12,10 +12,21 @@ The language-neutral files of this version are part of the specification: |---|---| | `spec/schema/fspm_scene.schema.json` | JSON Schema of the `FSPM_scene` extension (§3) | | `spec/schema/index.schema.json` | JSON Schema of a sequence's index (§8) | +| `spec/schema/views.schema.json` | JSON Schema of a sequence's optional views, `views.json` (§8.1) | | `spec/vocabulary/standard_names.csv` | the standard names, their canonical units and kinds, and each platform's names for them (§6) | | `spec/vocabulary/labels.csv` | entity labels, their scales and Plant Ontology terms (§6) | | `spec/vocabulary/enums.json` | the values of `edge_type`, `kind`, `role`, data structure and grid kinds; the base scales | -| `conformance/` | reference files and their expected values (§9) | +| `conformance/` | reference files and their expected values (§9): the specification as test cases | +| `spec/schema/coupling.schema.json` | JSON Schema of a coupling session's messages (§11) | + +When these sources disagree, `specification.md` prevails, then the schemas, then the conformance files; the +disagreement is a defect of the lower one, to be fixed. + +The format builds on glTF 2.0 (https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html) and two of its +extensions: `EXT_structural_metadata` for tables +(https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata) and +`EXT_mesh_features` for feature IDs +(https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features). ## 1. The data model @@ -32,11 +43,21 @@ The language-neutral files of this version are part of the specification: Its `origin` is a 4×4 matrix placing its coordinates in the scene's frame. - **The scene's frame** is right-handed, in metres, with **z up**: z = 0 at the soil surface, negative below it. - Every position (meshes, the segment ends of the vocabulary's `proximal_end_*` and `distal_end_*`, grid origins and - spacings) is in metres. A platform with other axes or units (e.g. y up, or centimetres) converts its coordinates, or - puts the conversion in the data structure's origin. + Meshes, grid origins and grid spacings are in metres. Position columns (e.g. the vocabulary's `proximal_end_*` and + `distal_end_*`) keep the platform's length unit, declared like any variable's (§5), so that values stay at the + platform's precision; readers convert them with their declared unit. A platform with other axes (e.g. y up) puts + the rotation in the data structure's origin. - An **entity set** is a table: one row per entity, one column per variable. A graph has one entity set per scale, with that scale's `rank` (coarser scales have smaller ranks). A grid's cells form one entity set, with no rank. + - An entity set MAY declare its `scale`: one of the vocabulary's base scales (`enums.json`: Plant, Axis, + GrowthUnit, Phytomer, Organ, SubOrgan, Layer, Cell, Compartment, Connection), or `mixed` when its rows belong to + several base scales (e.g. CPlantBox's organs: whole root axes and leaves). + - The `rank` SHOULD be the position of the set's base scale in that list, from 1 (Plant 1, Axis 2, Organ 5, + SubOrgan 6), when it has one. Readers MUST NOT assume consecutive ranks. + - Platforms have different scales. A reader SHOULD find the tables it needs by what they hold (standard names, and + the `complex` chain between sets), not by their names. For example, the segments are the set holding segment + ends, and their organs are the set their `complex` points to. Writers SHOULD declare the standard names of + positions, lengths and radii. - A **relation** is an edge table between two entity sets of the same data structure. Examples: a cell network, or connections between compartments. - The **scalars** are a one-row table of values for the whole data structure. @@ -51,9 +72,11 @@ A frame holding only data structures with parameters, and no geometry, is a **pa A frame MUST be a GLB file (glTF 2.0 binary): -1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.1. +1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.3. 2. Every buffer view holding table values MUST start at a multiple of 8 bytes. The JSON chunk MUST be padded with - spaces and the binary chunk with zeros to 8 bytes, as `EXT_structural_metadata` requires. + spaces and the binary chunk with zeros to 8 bytes, as `EXT_structural_metadata` requires. The buffer's + `byteLength` SHOULD be the binary chunk's padded length (glTF-Validator warns of more than 3 bytes of padding + beyond the buffer). 3. `extensionsUsed` MUST list `FSPM_scene` and `EXT_structural_metadata`, plus `EXT_mesh_features` when there are meshes. `FSPM_scene` SHOULD NOT be listed in `extensionsRequired`, so that every glTF viewer opens the file. 4. The glTF scene's root node, `fspm_scene`, turns the scene's frame (z up) into glTF's (y up), so that glTF viewers @@ -70,14 +93,16 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: ```text { - "version": "0.1", - "vocabulary": "0.1", + "version": "0.3", + "vocabulary": "0.2", "frame": {"time": 1.0, "timeUnit": "d", "step": 24}, "writer": {"platform": "...", "model": "..."}, "dataStructures": [{ "id": "plants", "kind": "graph", "platform": "...", "model": "...", "origin": [16 numbers, column-major], "node": 0, - "entitySets": [{"name": "Plant", "rank": 1, "propertyTable": 0}, {"name": "SubOrgan", "rank": 6, "propertyTable": 1}], + "entitySets": [{"name": "Plant", "rank": 1, "scale": "Plant", "propertyTable": 0}, + {"name": "SubOrgan", "rank": 6, "scale": "SubOrgan", "propertyTable": 1}], + "labels": {"Internode": "StemInternode"}, "relations": [{"name": "Connection", "sourceSet": "Cell", "targetSet": "Cell", "propertyTable": 2}], "scalars": 3, "grid": {"kind": "regular", "shape": [..], "origin": [..], "spacing": [..], "axes": [..], "periodic": [..], @@ -88,19 +113,26 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: "mappings": [{"name": "roots_in_soil", "source": {"dataStructure": "plants", "entitySet": "SubOrgan"}, "target": {"dataStructure": "soil", "entitySet": "Voxel"}, "method": "overlap", "propertyTable": 4}], "variables": {"": {"": {"unit": "m", "standardName": "length", "kind": "extensive", - "role": "state", "minimum": 0, "references": [], "doi": []}}} + "role": "state", "minimum": 0, "references": [], "doi": [], + "format": "date"}}} } ``` - **Identifiers.** Data structure ids, entity set, relation and mapping names, and column names MUST be identifiers - (`^[a-zA-Z_][a-zA-Z0-9_]*$`). Within a frame, data structure ids are unique, and so are the names of a data - structure's entity sets. + (`^[a-zA-Z_][a-zA-Z0-9_]*$`). Within a frame, data structure ids are unique. Within a data structure, entity set + names are unique, and so are relation names. +- **Labels.** A data structure MAY carry `labels`, a map from the labels it uses (e.g. a model's class names) to the + vocabulary's labels (§6). Readers looking for an entity of the vocabulary (e.g. the leaves) use it; the entities + keep the platform's labels. - **Extras.** A data structure MAY carry `extras`, a JSON object for the writing platform, e.g. what it needs to - rebuild its own data structure exactly. Its keys SHOULD be the platform's name. Other readers MUST NOT need it, - and SHOULD keep it. + rebuild its own data structure exactly. Its keys SHOULD be the platform's name as in the vocabulary's platform + columns, in lower case (`metafspm`, `cplantbox`, `groimp`, `virtualplantlab`). Other readers MUST NOT need it, and + SHOULD keep it. - **Property tables.** Every table of §1 is one `EXT_structural_metadata` property table of its own class, and no property table serves two tables. Writers SHOULD name the classes `__
` (mappings: `mapping__`). Readers MUST find tables through the `propertyTable` indices, not through class names. +- **Update.** A frame with an `update` object is a partial frame, a coupling session's message (§11.1); a frame + written as a time step has none. ## 4. Tables @@ -116,11 +148,21 @@ A column is one property of the table's class: | text | `STRING` | | labels, edge types and relation labels | `ENUM` (§4.2) | -- Variable-length arrays and the other glTF metadata types are not part of 0.1. +- Variable-length arrays and the other glTF metadata types are not part of 0.3. - **NaN.** NaN and infinities are forbidden in property tables. A float column with missing values (NaN) declares `noData`: the most negative finite value of its type, −1.7976931348623157e308 for FLOAT64 or −3.4028234663852886e38 for FLOAT32, stored where the value is NaN. Readers MUST read `noData` back as NaN. A column cannot hold both NaN and that value. +- **Missing values in other columns.** A variable column whose entities do not all have a value (e.g. a field that + only some of a platform's classes have) declares `noData`, and readers MUST read the rows holding it as missing: + - an integer column: a value of its type that the column does not hold; writers SHOULD use the type's minimum + (signed) or maximum (unsigned) when it is free; + - a text (`STRING`) column: a string the column does not hold; writers SHOULD use the empty string when it is free; + - booleans: `BOOLEAN` has no `noData`, so a boolean column with missing values is written as `UINT8` (0 false, + 1 true) with `noData` 255, and its class property declares `"extras": {"FSPM_boolean": true}`. Readers read it as + booleans with missing values. + + The reserved columns' `noData` (−1 in `complex` and `parent`) is not a missing value: it is the id of no entity. - **Infinities.** A float column holding +∞ or −∞ declares, in its class property's `extras`, `FSPM_infinity`: `{"positive": P, "negative": N}`, with P the largest finite value of its type (1.7976931348623157e308 for FLOAT64, 3.4028234663852886e38 for FLOAT32) and N the next value after the most negative one @@ -140,7 +182,8 @@ These columns have fixed meanings, in the tables where they appear: | `label` | ENUM | the entity's label (§6) | | `complex` | INT64, `noData` −1 | the id of the entity containing it, in a coarser entity set of the graph (MTG decomposition `/`); usually the nearest coarser one, but e.g. a cell wall may belong to its segment directly | | `parent` | INT64, `noData` −1 | the id of its predecessor in the same entity set | -| `edge_type` | ENUM `FSPM_edge_type` | how it follows its parent: `none` (no parent), `successor` (MTG `<`), `branch` (MTG `+`) | +| `edge_type` | ENUM `FSPM_edge_type` | how it follows its parent: `none` (no parent), `successor` (MTG `<`), `branch` (MTG `+`), `unknown` (it has a parent, but the writer does not know how it follows it) | +| `sibling_order` | INT64, optional | the entity's position, from 0, among the entities that share its parent in the writer's tree (its `parent`, else its bearer, else its `complex`), all entity sets together: for platforms that keep the order of children | | `level`, `index` | INT64; INT64 vector, one per dimension | an adaptive grid's leaf cell: its refinement level, and its integer index at that level | **Relations** @@ -164,14 +207,20 @@ Rules on these columns: complex. - `parent` and `edge_type` go together. `edge_type` is `none` exactly where `parent` is −1. The parents MUST form no cycle. +- `parent` MAY be in another complex than the entity's own: the first entity of a branch (e.g. a lateral root's + first segment, whose parent is a segment of the bearing root) usually is. - Every id referred to MUST exist in the entity set referred to. - `parent` links entities of the same scale. When an axis's first entity grows on an entity of another scale (in an MTG, a branch axis's first internode, linked to its bearer only through the axes), its `parent` is −1. Its bearer MAY be written as a relation named `bearer`, from the entity to the one it is borne on. + Its `relation` labels say how: `successor`, `branch` or `decomposition` (MTG `<`, `+`, or `/` to an entity that is + not its complex). - Other links between entities are relations, one per kind of link. A pair of entities linked in several ways (e.g. a - successor edge that is also a platform's own edge) appears in each of these relations. + successor edge that is also a platform's own edge) appears in each of these relations. A kind of link between + several pairs of entity sets is one relation per pair, named ``, or `__` when + there are several (e.g. `bearer_Axis_Internode`); readers find them by the prefix. - Each label column or relation column is an `ENUM` of its own, listing the values used. The enum - `FSPM_edge_type` is fixed: `none` = 0, `successor` = 1, `branch` = 2. + `FSPM_edge_type` is fixed: `none` = 0, `successor` = 1, `branch` = 2, `unknown` = 3. - A grid's cells MAY omit `id`; their ids are then their row numbers. A regular grid's rows are its cells in C order of `shape`, the last axis varying fastest, and its ids, if written, MUST be these row numbers. @@ -188,6 +237,7 @@ Every other column is a variable. It SHOULD be declared in `FSPM_scene.variables | `role` | required: `state`, `input`, `parameter`, `derived` or `setting` (enums.json) | | `minimum`, `maximum` | the variable's plausible range, as documentation (not a constraint on values) | | `references`, `doi` | its sources | +| `format` | for a text column holding dates or times: `date`, `date-time` or `time`, as ISO 8601 (`2022-08-24`, `2022-08-24T13:00:00`, `13:00:00`) | Parameters are variables with role `parameter`. A parameter goes in the table it applies to: @@ -210,7 +260,7 @@ Units are UCUM codes, with: ## 6. Vocabulary -The vocabulary (version `0.1`) lists standard names. For each, it gives: +The vocabulary (version `0.2`) lists standard names. For each, it gives: - a canonical unit; - a kind; @@ -253,7 +303,7 @@ two platforms. A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json`, following `index.schema.json`: ```text -{"fspmIndex": "0.1", "timeUnit": "d", +{"fspmIndex": "0.3", "timeUnit": "d", "frames": [{"time": 0.0, "step": 0, "file": "frame_000000.glb"}, ...]} ``` @@ -263,7 +313,42 @@ A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json` given to another entity in a later frame of the same data structure and entity set. Labels and parents MAY change, e.g. an apex moved by growth. A platform whose own ids may come back (a counter restarted, a deleted maximum's id given again) keeps, across a sequence, a map from its entities to the ids it writes. -- **Packing.** Packing a sequence into one GLB is not part of 0.1. +- **Ids are not positions.** Ids may be sparse (e.g. offsets keeping them stable as a plant grows). Readers SHOULD + NOT size arrays by the largest id. +- **Packing.** Packing a sequence into one GLB is not part of 0.3. + +### 8.1 Views + +A sequence (or a frame) MAY come with a companion `views.json`, next to its `scene.fspm.json` (or its GLB), following +`views.schema.json`. It says how to look at the frames, not what they hold: readers MAY ignore it, and a sequence is +valid without it. It is platform-independent; any viewer may apply it. + +```text +{"encoding3d_views": 1, "active": "default", + "configurations": {"default": {"layers": [ + {"name": "roots", "source": {"data_structure": "plants", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "colormap": "Viridis (matplotlib)", + "range": "auto", "log": false, "opacity": 1, "visible": true, "representation": "Surface", + "tube": {"radius": "radius", "scale": 1}}]}}} +``` + +- **Configurations.** Named lists of layers; `active` names the one shown first (else the first). +- **A layer** draws one part of a data structure: `source.kind` `mesh` (with the mesh's `name`), `segments` (an entity + set with segment ends, `entity_set`) or `grid` (the grid's cells). Its name is unique in its configuration. +- **Selection.** `select` keeps the entities whose row in `select.entity_set` holds every condition of `where`: a + value, a list of values (labels and other text by name), or a comparison (`">=0"`, `"!=2"`); a missing value never + holds. For a mesh, the entity set is any the mesh represents (a triangle is kept when its feature row in that set + is); for segments and grids, the source's own set. +- **Drawing.** `color_by` names a variable of an entity set the layer's entities are in (as `select`), or is null + (then `solid_color`, RGB in [0, 1]); `colormap` is a colour map's name (ParaView's preset names), `range` `"auto"` + or `[min, max]`, `log` a log scale; `opacity`, `visible` (when opened), `representation`; `tube` (segments only) + draws them as tubes of radius the `radius` column times `scale`. +- **Defaults.** `colormap` `"Viridis (matplotlib)"`, `range` `"auto"`, `log` false, `opacity` 1, `visible` true, + `representation` `"Surface"`, `solid_color` white, a tube's `scale` 1. +- Every name a layer gives (data structure, mesh, entity sets, columns) SHOULD be in the frames it goes with; + the Python reference implementation checks them (`openalea.encoding3d.views.check_views(views, frame)`). In a frame + that lacks one (e.g. a mesh drawn from a later frame on), a viewer draws the layer empty. ## 9. Conformance @@ -278,15 +363,204 @@ sequence: | `soil_and_roots` | a root system, a regular soil grid with z pointing down, and their mapping | | `adaptive_soil` | an adaptive grid's leaf cells (`level`, `index`) | | `parameter_set` | a `table` data structure with parameters per plant and in the scalars, references and DOIs; an infinite value | +| `missing_values` | integer, text and boolean columns with missing values; a date column; `unknown` edge types and a `sibling_order` column; a `mixed` entity set; a labels map; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system and their index | -An implementation conforms to 0.1 when two things hold: +An implementation conforms to 0.3 when two things hold: + +- **Reading:** what it reads from every conformance GLB equals the expected JSON. Floats are compared exactly; NaN and + missing values are null in the JSON, +∞ and −∞ the strings `"Infinity"` and `"-Infinity"`; positions are the + FLOAT32 values. +- **Writing:** what it writes passes `encoding3d validate` with no error, and Khronos' glTF-Validator with no error. -- **Reading:** what it reads from every conformance GLB equals the expected JSON. Floats are compared exactly; NaN is - null in the JSON, +∞ and −∞ the strings `"Infinity"` and `"-Infinity"`; positions are the FLOAT32 values. -- **Writing:** what it writes passes `fspm-tf validate` with no error, and Khronos' glTF-Validator with no error. +A codec's participant (§11) conforms when it runs every **coupling session** of `conformance/coupling/` as expected, +over both transports. A session's folder holds the coordinator's settings (`session.json`), a peer (a sequence the +coordinator replays), the subject's declaration and the updates it sends (`subject/`), and the expected values +(`expected/turn_.json`: what the subject reads from each turn; `expected/record/frame_.json`: the +session's record). The participant under test is a program run as `COMMAND SESSION OUT`: it joins with the +declaration, writes what it reads from each turn to `OUT/turn_.json` (in the expected values' layout), and sends +`subject/update_.glb` as its codec reads and writes it again. `encoding3d couple-conformance conformance/coupling +--participant COMMAND` runs and compares them all. + +| Session | Covers | +|---|---| +| `two_scales` | a subject refining a replayed peer's organs into segments: new entities, removals cascading to the subject's segments, a unit conversion and a factor, a copy to finer entities | +| `two_data_structures` | a subject owning another data structure, linked to the peer's by the mapping it provides: a sum through a mapping, mappings in updates | ## 10. Keeping what one does not read -A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.1 this covers +A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.3 this covers the glTF root's other extensions and `extras`, when they hold no buffer views. + +## 11. Coupling + +Platforms couple their models through encoding3d in a **session**: each **participant** is a platform's model running +in its own process, or a recorded sequence (§11.6), and a **coordinator** runs the session. The coordinator holds the +session's shared structure (the entities of every data structure and the values each participant provides), applies +the links between them and sends each participant its inputs. A participant only declares itself, then answers each +of its turns with its outputs. Nothing about a given coupling is fixed in advance: declarations and links are passed +when the session starts. + +### 11.1 Partial frames + +Turns and updates are **partial frames**: frames whose `FSPM_scene` has an `update` object. + +- `update.participant` is the sender's name, or `"coordinator"`. +- `update.removed` lists, per data structure and entity set, a property table of one `id` column: the entities + removed (by the sender, or since the receiver's last turn). +- Its entity sets hold some rows and some columns only. A row is identified by its `id`. +- `complex` and `parent` MAY refer to entities the frame does not hold, and MAY be absent. Relations and mappings MAY + likewise refer to absent entities. +- Every other rule of a frame holds. The validator checks partial frames with the reference checks lifted. + +### 11.2 Messages + +A message is a JSON header and, for a turn or an update, a payload: a partial frame's GLB bytes. + +| Message | From | Header | Payload | +|---|---|---|---| +| hello | participant | `type`, `protocol` (`"0.3"`), `declaration`, `links` (the links it proposes, may be empty) | – | +| welcome | coordinator | `type`, `idOffset`, `idCount`, `session` (`start`, `interval`, `timeUnit`, `steps`, `stages`, `participants`, `keepMessages`), `couplings` (its inputs' providers) | – | +| turn | coordinator | `type`, `step`, `phase` (`"init"` at step 0, else `"step"`), `time`, `until` | its inputs | +| update | participant | `type`, `step` | its outputs | +| error | participant | `type`, `message` | – | +| stop | coordinator | `type`, `reason` | – | + +**Declaration.** +- `participant`: its name, an identifier. +- `platform`: optional. +- `entitySets`: each entity set it works on, as `dataStructure`, `name`, `rank`, and `rights` on its topology: + - `owns`: it adds and removes the entities. An entity set has one owner at most. + - `refines`: it adds entities, and removes the ones it created. + - `follows`: it reads. +- `variables`: each one with `dataStructure`, `entitySet`, `name`, `direction` (`provides`: an output; `requires`: an + input), `unit` (UCUM), and optionally `standardName`, `kind`, `optional`. +- `mappings` (optional): the mappings it provides, each with its `name`, `source` and `target` (`dataStructure`, + `entitySet`). Its updates carry them as the frame's mappings (§1) of those names; each one sent replaces the + previous. + +**Link.** A link names the input it feeds and the output that feeds it: +- the input: `receiver`, `variable`, `entitySet` (when the name is ambiguous); +- the output: `provider`, `source`, `sourceEntitySet`; +- its conversion: `factor` (applied after the unit conversion), `aggregation` and `interpolation` (`hold` or + `linear`, §11.6); +- `mapping`: between two data structures, the name of a mapping a participant provides, read in either direction. + +**Aggregation.** A link may change the scale: +- `sum`, `mean`, `max` or `min` from a finer entity set, through the `complex` chain. The default is `sum` for an + extensive source, `mean` otherwise. +- `copy` from a coarser one. An extensive source needs `copy` stated explicitly, because copying it to finer entities + counts it several times. + +**Between two data structures**, a link goes through its mapping (weights `w` between source entities `s` and target +entities `t`): +- `sum` (the default for an extensive source) shares each source's value among its targets in proportion to their + weights: `x_t = Σ_s x_s · w_st / Σ_t' w_st'`. The total amount is kept. +- `mean` (the default otherwise) gives each target the weighted mean of its sources: `x_t = Σ_s w_st · x_s / Σ_s w_st`. +- A target no source reaches has no value. + +Formulas are not part of a link: a receiver computes them from its linked inputs. + +### 11.3 Resolution + +Before the first step, the coordinator gives each required input exactly one provider, in this order: +1. a link given to the coordinator; +2. a link proposed by a participant; +3. the output with the same standard name, preferring one on the same entity set; +4. the output with the same name. + +It refuses the session, and stops every participant with the reason, when any of these holds: +- an input that is not optional has no provider, or has several at the same priority; +- the units are not convertible; +- an entity set has two owners; +- a stage holds two participants that change one entity set's topology; +- a link joins two data structures without a mapping, or with a mapping whose ends are not the link's. + +### 11.4 Steps and stages + +- The session runs `steps` exchange steps of `interval` (in `timeUnit`), from `start`. +- Participants are grouped in **stages**: given to the coordinator, or by default the owners, then the refiners, then + the others. +- **Initial state (step 0).** Stage by stage, each participant is sent a turn with the entities so far, and answers + with its initial entities and outputs. +- **Each step k ≥ 1.** Stage by stage: + 1. each participant of the stage is sent a turn from `time` to `until`, holding its inputs and the entities added or + removed since its last turn; + 2. once all of them answered, their updates are merged, in the stage's order. + + Participants in one stage therefore see the same state, and the next stage sees theirs. The coupling is explicit: + one pass per step. +- After the last step, every participant is sent `stop`. An error message, or an update the coordinator refuses, also + stops the session, and every participant gets the reason. + +### 11.5 One structure, several participants + +- **Ids.** A participant's new entities take ids in `[idOffset, idOffset + idCount)`, so ids never collide. + Participants keep their own ids, and map them, if they need to. +- **Rights.** A participant may write: + - only the outputs it declared; + - the topology columns only of entities it created, or of an entity set it owns. +- **Removal.** Removing an entity removes, down the finer entity sets, the entities whose complex it is. Its successors + in its own entity set become roots. +- **New entities.** A participant receives new entities with their `parent` and `complex`, and initializes its own + variables on them. When an entity splits, the participant shares its own extensive quantities and dilutes its own + concentrations by its rules; the entity's lineage is its `parent`. +- **Labels.** An entity created without a label takes its entity set's name. +- **Scales a participant does not know.** A participant declares only the entity sets it works on. One that grows a + coarse architecture (say, organs) needs to know nothing of the finer entities another participant refines under + them (segments, cells): it is never sent them, and when it removes an organ, the coordinator removes the organ's + segments and tells their refiner. + +### 11.6 Transports + +Both transports carry the same messages. + +- **ZeroMQ** (endpoint `tcp://…` or `ipc://…`). + - The coordinator binds a ROUTER socket. + - Each participant connects a DEALER socket whose identity is its name. + - A message is one or two frames: the header, then the payload. +- **A folder** (endpoint `files:`). + - Participant `p` has two queues, `/p/to_coordinator/` and `/p/to_participant/`. + - Message n of a queue is `n.json`, n as 8 digits from `00000000`. + - Its payload is `n.glb`, named in the header's `payload` key. + - The payload is written first, and the header is renamed into place last, so a header present means a complete + message. + - Its reader deletes a message's files once read, unless the session keeps them (`keepMessages` true in the + welcome's `session`; absent: kept). A participant applies it from the welcome on, deleting the welcome too; the + coordinator, from the start. Kept, the folder is the session's log; a session needs a new folder. The usual log of + a session is its record, a sequence of full frames (§8) the coordinator writes when asked. +- **Launching.** A participant launched by the coordinator finds the endpoint in the environment variable + `ENCODING3D_ENDPOINT` and its name in `ENCODING3D_PARTICIPANT`. +- **Replay.** A recorded sequence (§8) may be a participant, so a platform's log feeds another platform's inputs without + the first one running. + - It owns its entity sets, and provides their variables. + - Its ids are renumbered within its range. + - Between two recorded times, its values are those of the last frame at or before the session's time (`hold`), or + interpolated linearly for links stating `"interpolation": "linear"`. + +## Changes from 0.2 + +All additions; a 0.2 file is a valid 0.3 file. + +- §8.1: views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): configurations of + layers, each a selection of a mesh's, segments' or grid's entities and how they are drawn. + +- §11: coupling sessions: partial frames (`FSPM_scene.update`), the messages (`spec/schema/coupling.schema.json`), + declarations and rights, links (units, factors, changes of scale, mappings between data structures), stages, the + shared structure, two transports, replay; conformance sessions (§9). + +## Changes from 0.1 + +All additions; a 0.1 file is a valid 0.2 file. + +- Sources: `conformance/` is part of the agreement; the order in which the sources prevail. +- §1: an entity set's `scale`, ranks as base scale positions, reading by content; position columns in their declared + length unit. +- §2: the buffer's `byteLength` holds the padding. +- §3: unique relation names, a data structure's `labels`, the extras key. +- §4.1: missing values in integer, text and boolean columns. +- §4.2: `edge_type` `unknown`, the `sibling_order` column, a parent in another complex, bearer relations' labels and names. +- §5: a variable's `format`. +- §6: the vocabulary 0.2 (`Seed`, `mtg_index`). +- §8: sparse ids. +- §9: the `missing_values` scene. diff --git a/spec/vocabulary/enums.json b/spec/vocabulary/enums.json index 1135935..d9b0491 100644 --- a/spec/vocabulary/enums.json +++ b/spec/vocabulary/enums.json @@ -1,8 +1,9 @@ { - "version": "0.1", + "version": "0.2", "edge_type": {"none": "the entity's scale has no predecessor for it (a root of its scale)", "successor": "it follows its parent on the same axis (MTG '<')", - "branch": "it branches from its parent (MTG '+')"}, + "branch": "it branches from its parent (MTG '+')", + "unknown": "it has a parent, but the writer does not know how it follows it"}, "kind": {"extensive": "proportional to the amount of matter: summed when entities are merged, split by weights", "intensive": "independent of the amount: averaged (weighted by volume or mass) when entities are merged", "massic_concentration": "an amount per mass: averaged weighted by mass", @@ -20,5 +21,11 @@ "grid_kind": {"regular": "shape cells of size spacing from origin", "adaptive": "a regular base grid refined up to maxLevel; the leaf cells are listed"}, "scales": ["Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", - "Connection"] + "Connection"], + "scale": {"mixed": "an entity set whose rows belong to several base scales (e.g. CPlantBox's organs: root axes and leaves)"}, + "format": {"date": "ISO 8601 date: 2022-08-24", "date-time": "ISO 8601 date and time: 2022-08-24T13:00:00", + "time": "ISO 8601 time: 13:00:00"}, + "bearer_relation": {"successor": "MTG '<' to an entity of another scale", + "branch": "MTG '+' to an entity of another scale", + "decomposition": "MTG '/' to an entity that is not its complex"} } diff --git a/spec/vocabulary/labels.csv b/spec/vocabulary/labels.csv index 0db358b..cefdf81 100644 --- a/spec/vocabulary/labels.csv +++ b/spec/vocabulary/labels.csv @@ -9,6 +9,7 @@ ShootGrowthUnit,GrowthUnit,,,no Plant Ontology term RootGrowthUnit,GrowthUnit,,,no Plant Ontology term ShootPhytomer,Phytomer,,,no Plant Ontology term RootPhytomer,Phytomer,,,no Plant Ontology term +Seed,Organ,PO:0009010,seed,"the seed (or a grass's caryopsis), bearing the seminal roots and the shoot" Meristem,Organ,PO:0020147,root apical meristem,or a shoot apical meristem: the term depends on the axis StemInternode,Organ,PO:0020142,stem internode, RootInternode,Organ,PO:0009005,root,a root portion between two branching points: no own term diff --git a/spec/vocabulary/standard_names.csv b/spec/vocabulary/standard_names.csv index 5eaae3b..b1379ae 100644 --- a/spec/vocabulary/standard_names.csv +++ b/spec/vocabulary/standard_names.csv @@ -40,3 +40,4 @@ cell_area,m2,extensive,area,cell,cross section,,Cell,area of the cell in the cro wall_thickness,m,intensive,thickness,cell wall,,,Connection Cell,thickness of the cell wall,wall_thickness,,,,GRANAP/MECHA: wall_thickness (um) net_co2_assimilation_rate,mol/(m2.s),intensive,assimilation rate,carbon dioxide,leaf,net,SubOrgan Organ,net CO2 assimilated per unit of leaf area (photosynthesis minus respiration),,,,,"not in metafspm yet (VPL's Ecophys.jl, GroIMP's photosynthesis models)" nitrate_concentration,mol/m3,volumic_concentration,amount concentration,nitrate,entity or soil,,SubOrgan Voxel,amount of nitrate per volume,,,,,not in metafspm yet (CPlantBox/DuMux solute transport) +mtg_index,1,descriptor,index,entity,,in its MTG code,any,the index of an entity in its MTG code: 1 in Internode1,,,,mtg_index, diff --git a/tutorials/.gitignore b/tutorials/.gitignore new file mode 100644 index 0000000..d90b79f --- /dev/null +++ b/tutorials/.gitignore @@ -0,0 +1,7 @@ +# What the tutorials write when they run +outputs/ +__pycache__/ +build/ +target/ +.ipynb_checkpoints/ +Manifest.toml diff --git a/tutorials/01_encoding3d_alone/README.md b/tutorials/01_encoding3d_alone/README.md new file mode 100644 index 0000000..e637a69 --- /dev/null +++ b/tutorials/01_encoding3d_alone/README.md @@ -0,0 +1,20 @@ +# Tutorial 1: encoding3d on its own + +**What you learn:** what an encoding3d file holds, and how to write, check, read and view one, with no plant +modelling platform at all. + +**What you need:** Python with `openalea.encoding3d` (from the repository's root: `pip install -e .`), numpy, pandas +and Jupyter; ParaView 5.11 or later to look at the result (optional). + +**How to run it:** open `encoding3d_alone.ipynb` in Jupyter and run the cells one after the other, or run it all at +once with `./run.sh`. It writes its files in `outputs/`. + +In the notebook you + +1. describe a small root system at three scales (plant, axes, segments), its parts with ids that never change, and + its variables with their units and standard names, then write it as a frame and check it with the validator; +2. let it grow for five days and write a frame per day, a sequence, with its index `scene.fspm.json`; +3. read the sequence back, follow one segment from day to day, and read a frame lazily (only the columns you use); +4. say how viewers should draw it (`views.json`) and open it in ParaView. + +Next: tutorial 2 shows the same kind of files written and read by the platforms themselves. diff --git a/tutorials/01_encoding3d_alone/encoding3d_alone.ipynb b/tutorials/01_encoding3d_alone/encoding3d_alone.ipynb new file mode 100644 index 0000000..5148675 --- /dev/null +++ b/tutorials/01_encoding3d_alone/encoding3d_alone.ipynb @@ -0,0 +1,409 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Tutorial 1: encoding3d on its own\n", + "\n", + "encoding3d stores a plant model's results as files any platform can read: one file per time step, called a\n", + "**frame**. A frame holds the plant's parts at several **scales** (the whole plant, its axes, the small segments\n", + "of each axis), how they connect, and the **variables** a model computed on them, each with its unit.\n", + "\n", + "In this tutorial you write such files yourself, with no plant modelling platform at all: a small root system\n", + "that grows for five days, made with a few lines of numpy. You will\n", + "\n", + "1. describe the plant at three scales and write one frame;\n", + "2. write a frame per day, a **sequence**, and check it;\n", + "3. read it back, all at once or only the columns you need;\n", + "4. say how ParaView should colour it, and open it there.\n", + "\n", + "Everything is written in `outputs/` next to this notebook." + ] + }, + { + "cell_type": "code", + "id": "cell-1", + "metadata": {}, + "source": [ + "import os\n", + "import subprocess\n", + "import sys\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import encoding3d\n", + "from openalea.encoding3d import views\n", + "from openalea.encoding3d.validate import validate_bytes, validate_sequence\n", + "\n", + "OUTPUTS = \"outputs\"\n", + "os.makedirs(OUTPUTS, exist_ok=True)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## 1. A plant at three scales\n", + "\n", + "Our root system has one **plant**, a few **axes** (the main root and its laterals), and each axis is a chain of\n", + "**segments**, 1 cm long, the last one being the growing tip. A toy growth rule (not a real model): every day the main\n", + "root grows three segments, each lateral one, and a new lateral starts every second segment of the main root.\n", + "\n", + "Each part gets an **id**, a number that never changes and is never reused: that is how a segment is followed from\n", + "one day to the next." + ] + }, + { + "cell_type": "code", + "id": "cell-3", + "metadata": {}, + "source": [ + "SEGMENT = 0.01 # m: every segment is 1 cm long\n", + "\n", + "\n", + "class ToyRoots:\n", + " \"\"\"A root system growing by a fixed rule, its parts with stable ids.\"\"\"\n", + "\n", + " def __init__(self):\n", + " self.next_id = 1\n", + " self.plant = self.new_id()\n", + " self.axes = {} # axis id: (order, parent segment id or None, its segment ids)\n", + " self.segments = {} # segment id: dict(axis, parent, start, end, age)\n", + " self.add_axis(order=1, parent=None, start=np.zeros(3), direction=np.array([0., 0., -1.]))\n", + "\n", + " def new_id(self):\n", + " self.next_id += 1\n", + " return self.next_id - 1\n", + "\n", + " def add_axis(self, order, parent, start, direction):\n", + " axis = self.new_id()\n", + " self.axes[axis] = (order, parent, [], direction)\n", + " return axis\n", + "\n", + " def grow_axis(self, axis, count):\n", + " order, parent, chain, direction = self.axes[axis]\n", + " for _ in range(count):\n", + " previous = chain[-1] if chain else parent\n", + " start = self.segments[previous][\"end\"] if previous is not None else np.zeros(3)\n", + " segment = self.new_id()\n", + " self.segments[segment] = dict(axis=axis, parent=previous, start=start, end=start + SEGMENT * direction,\n", + " age=0.)\n", + " chain.append(segment)\n", + "\n", + " def day(self):\n", + " for segment in self.segments.values():\n", + " segment[\"age\"] += 1.\n", + " main = next(iter(self.axes))\n", + " self.grow_axis(main, 3)\n", + " for axis, (order, parent, chain, direction) in list(self.axes.items()):\n", + " if order == 2:\n", + " self.grow_axis(axis, 1)\n", + " main_chain = self.axes[main][2]\n", + " bearers = {parent for order, parent, _, _ in self.axes.values() if order == 2}\n", + " for position, segment in enumerate(main_chain[:-1]):\n", + " if position % 2 == 1 and segment not in bearers:\n", + " angle = 2.4 * len(bearers)\n", + " side = np.array([np.cos(angle), np.sin(angle), -0.3])\n", + " lateral = self.add_axis(order=2, parent=segment, start=self.segments[segment][\"end\"],\n", + " direction=side / np.linalg.norm(side))\n", + " self.grow_axis(lateral, 1)\n", + " bearers.add(segment)\n", + "\n", + "\n", + "roots = ToyRoots()\n", + "roots.day()\n", + "len(roots.axes), len(roots.segments)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "Now the plant as encoding3d tables. Each scale is an **entity set**, a table with one row per part:\n", + "\n", + "- `id`: the part's id;\n", + "- `label`: what it is (`RootAxis`, `RootSegment`: names from encoding3d's shared vocabulary);\n", + "- `complex`: the part one scale up that contains it (a segment's axis, an axis's plant);\n", + "- `parent` and `edge_type`: the part before it at the same scale (a segment's predecessor, or for the first segment\n", + " of a lateral, the segment it branches from: `branch`);\n", + "- then the variables, each declared with its **unit** and its **standard name**, so that any platform knows what\n", + " it is (`length` in m, `water_potential` in MPa, converted when needed).\n", + "\n", + "The segments' ends (`x1` … `z2`, declared as `proximal_end_x` … `distal_end_z`) let viewers draw them as tubes." + ] + }, + { + "cell_type": "code", + "id": "cell-5", + "metadata": {}, + "source": [ + "def frame_of(roots, day):\n", + " \"\"\"The root system on *day* as an encoding3d frame.\"\"\"\n", + " plant_set = encoding3d.EntitySet(\"Plant\", rank=1, columns=dict(\n", + " id=np.array([roots.plant]), label=np.array([\"Wheat\"], dtype=object)))\n", + "\n", + " axis_ids = list(roots.axes)\n", + " first_of = {axis: chain[0] for axis, (_, _, chain, _) in roots.axes.items()}\n", + " axis_set = encoding3d.EntitySet(\"Axis\", rank=2, columns=dict(\n", + " id=np.array(axis_ids), label=np.array([\"RootAxis\"] * len(axis_ids), dtype=object),\n", + " complex=np.full(len(axis_ids), roots.plant),\n", + " parent=np.array([roots.segments[roots.axes[a][1]][\"axis\"] if roots.axes[a][1] else encoding3d.NO_ID\n", + " for a in axis_ids]),\n", + " edge_type=np.array([\"branch\" if roots.axes[a][1] else \"none\" for a in axis_ids], dtype=object),\n", + " branching_order=np.array([roots.axes[a][0] for a in axis_ids])))\n", + " axis_set.variables[\"branching_order\"] = encoding3d.Variable(unit=\"1\", standard_name=\"branching_order\",\n", + " kind=\"descriptor\")\n", + "\n", + " ids = list(roots.segments)\n", + " segments = [roots.segments[i] for i in ids]\n", + " starts = np.array([s[\"start\"] for s in segments])\n", + " ends = np.array([s[\"end\"] for s in segments])\n", + " depth = -ends[:, 2]\n", + " tips = {chain[-1] for _, _, chain, _ in roots.axes.values()}\n", + " segment_set = encoding3d.EntitySet(\"SubOrgan\", rank=6, columns=dict(\n", + " id=np.array(ids), label=np.array([\"RootApex\" if i in tips else \"RootSegment\" for i in ids], dtype=object),\n", + " complex=np.array([s[\"axis\"] for s in segments]),\n", + " parent=np.array([s[\"parent\"] if s[\"parent\"] is not None else encoding3d.NO_ID for s in segments]),\n", + " edge_type=np.array([\"none\" if s[\"parent\"] is None else\n", + " \"successor\" if roots.segments[s[\"parent\"]][\"axis\"] == s[\"axis\"] else \"branch\"\n", + " for s in segments], dtype=object),\n", + " x1=starts[:, 0], y1=starts[:, 1], z1=starts[:, 2], x2=ends[:, 0], y2=ends[:, 1], z2=ends[:, 2],\n", + " length=np.full(len(ids), SEGMENT),\n", + " radius=0.0005 * (1 + 0.2 * np.array([s[\"age\"] for s in segments])), # m: thickening with age\n", + " age=np.array([s[\"age\"] for s in segments]),\n", + " water_potential=-0.1 - 2. * depth)) # MPa: a toy gradient, drier with depth\n", + " for name, standard_name in zip((\"x1\", \"y1\", \"z1\", \"x2\", \"y2\", \"z2\"),\n", + " (\"proximal_end_x\", \"proximal_end_y\", \"proximal_end_z\",\n", + " \"distal_end_x\", \"distal_end_y\", \"distal_end_z\")):\n", + " segment_set.variables[name] = encoding3d.Variable(unit=\"m\", standard_name=standard_name, kind=\"descriptor\")\n", + " segment_set.variables[\"length\"] = encoding3d.Variable(unit=\"m\", standard_name=\"length\", kind=\"extensive\")\n", + " segment_set.variables[\"radius\"] = encoding3d.Variable(unit=\"m\", standard_name=\"radius\", kind=\"intensive\")\n", + " segment_set.variables[\"age\"] = encoding3d.Variable(unit=\"d\", standard_name=\"age\", kind=\"intensive\")\n", + " segment_set.variables[\"water_potential\"] = encoding3d.Variable(\n", + " unit=\"MPa\", standard_name=\"water_potential\", kind=\"intensive\", description=\"a toy gradient, lower with depth\")\n", + "\n", + " plant = encoding3d.DataStructure(\"plant\", entity_sets=[plant_set, axis_set, segment_set], platform=\"tutorial\")\n", + " return encoding3d.Frame([plant], time=float(day), time_unit=\"d\", step=day,\n", + " writer={\"platform\": \"tutorial 1\"})\n", + "\n", + "\n", + "frame = frame_of(roots, 1)\n", + "pd.DataFrame(frame.data_structure(\"plant\").entity_set(\"SubOrgan\").columns).head()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "A frame becomes a file with `to_bytes` (or `write`). The **validator** checks it against the specification: ids\n", + "unique, every `complex` and `parent` found, units valid and convertible to their standard name's." + ] + }, + { + "cell_type": "code", + "id": "cell-7", + "metadata": {}, + "source": [ + "data = encoding3d.to_bytes(frame)\n", + "print(len(data), \"bytes\")\n", + "print(validate_bytes(data))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "A mistake is caught: a length declared in seconds cannot be converted to the standard name's metres." + ] + }, + { + "cell_type": "code", + "id": "cell-9", + "metadata": {}, + "source": [ + "wrong = frame_of(roots, 1)\n", + "wrong.data_structure(\"plant\").entity_set(\"SubOrgan\").variables[\"length\"].unit = \"s\"\n", + "print(validate_bytes(encoding3d.to_bytes(wrong)))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## 2. A sequence: one frame per day\n", + "\n", + "`write_sequence` writes each frame as a `.glb` file and an index, `scene.fspm.json`, listing their times." + ] + }, + { + "cell_type": "code", + "id": "cell-11", + "metadata": {}, + "source": [ + "roots = ToyRoots()\n", + "frames = []\n", + "for day in range(1, 6):\n", + " roots.day()\n", + " frames.append(frame_of(roots, day))\n", + "\n", + "index = encoding3d.write_sequence(os.path.join(OUTPUTS, \"frames\"), frames)\n", + "print(index)\n", + "print(validate_sequence(index))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "The command line does the same from a terminal: `encoding3d validate outputs/frames/scene.fspm.json`, or\n", + "`encoding3d info` to see what a frame holds." + ] + }, + { + "cell_type": "code", + "id": "cell-13", + "metadata": {}, + "source": [ + "info = subprocess.run([sys.executable, \"-m\", \"openalea.encoding3d.cli\", \"info\",\n", + " os.path.join(OUTPUTS, \"frames\", \"frame_000004.glb\")], capture_output=True, text=True)\n", + "print(info.stdout)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-14", + "metadata": {}, + "source": [ + "## 3. Reading it back\n", + "\n", + "`read_sequence` gives the frames one after another. Because ids are stable, you can follow a segment through the\n", + "days: here the main root's first segment ages and thickens." + ] + }, + { + "cell_type": "code", + "id": "cell-15", + "metadata": {}, + "source": [ + "first_segment = frames[0].data_structure(\"plant\").entity_set(\"SubOrgan\").columns[\"id\"][0]\n", + "rows = []\n", + "for again in encoding3d.read_sequence(index):\n", + " segments = again.data_structure(\"plant\").entity_set(\"SubOrgan\")\n", + " row = int(np.flatnonzero(segments.columns[\"id\"] == first_segment)[0])\n", + " rows.append(dict(day=again.time, segments=segments.count, first_segment_age=segments.columns[\"age\"][row],\n", + " first_segment_radius_mm=1000 * segments.columns[\"radius\"][row]))\n", + "pd.DataFrame(rows)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-16", + "metadata": {}, + "source": [ + "For large frames, read **lazily**: only the file's description is parsed at first, and each column is decoded the\n", + "first time you use it. The file stays open until the frame is closed (here, at the end of the `with` block)." + ] + }, + { + "cell_type": "code", + "id": "cell-17", + "metadata": {}, + "source": [ + "with encoding3d.read(os.path.join(OUTPUTS, \"frames\", \"frame_000004.glb\"), lazy=True) as last:\n", + " segments = last.data_structure(\"plant\").entity_set(\"SubOrgan\")\n", + " print(segments.count, \"segments; columns:\", list(segments.columns))\n", + " print(\"decoded so far:\", segments.columns.decoded())\n", + " print(\"lowest water potential:\", segments.columns[\"water_potential\"].min(), segments.variables[\"water_potential\"].unit)\n", + " print(\"decoded now:\", segments.columns.decoded())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-18", + "metadata": {}, + "source": [ + "## 4. How to look at it: views\n", + "\n", + "A `views.json` next to the frames tells viewers how to draw them: here one **layer**, the segments drawn as tubes\n", + "as thick as their radius, coloured by water potential. Viewers that do not know it simply ignore it." + ] + }, + { + "cell_type": "code", + "id": "cell-19", + "metadata": {}, + "source": [ + "roots_view = {\"encoding3d_views\": 1, \"active\": \"default\", \"configurations\": {\"default\": {\"layers\": [\n", + " {\"name\": \"roots\", \"source\": {\"data_structure\": \"plant\", \"kind\": \"segments\", \"entity_set\": \"SubOrgan\"},\n", + " \"color_by\": {\"entity_set\": \"SubOrgan\", \"variable\": \"water_potential\"}, \"colormap\": \"Viridis (matplotlib)\",\n", + " \"tube\": {\"radius\": \"radius\", \"scale\": 1}}]}}}\n", + "views.write_views(os.path.join(OUTPUTS, \"frames\", views.VIEWS_FILE), roots_view)\n", + "print(views.check_views(roots_view, frames[-1]))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-20", + "metadata": {}, + "source": [ + "**In ParaView** (5.11 or later):\n", + "\n", + "1. once: *Tools → Manage Plugins → Load New*, choose `paraview/plugin_encoding3d.py` of this repository, tick\n", + " *Auto Load*;\n", + "2. *File → Open* `tutorials/01_encoding3d_alone/outputs/frames/scene.fspm.json`, choose the *encoding3d reader*,\n", + " *Apply*;\n", + "3. press *Play*: the roots grow day by day, coloured as `views.json` says.\n", + "\n", + "`paraview/README.md` explains the rest (switching views, selecting a plant, an axis or a segment).\n", + "\n", + "## Try it yourself\n", + "\n", + "- Make the laterals grow two segments a day, and see the sequence change in ParaView.\n", + "- Add a variable of your own to the segments (say, a nitrogen concentration in `mol/m3`), declare it, validate.\n", + "- Colour the roots by `age` instead of water potential in `views.json`.\n", + "- Tutorial 2 shows the same files written by real platforms: metafspm, VirtualPlantLab, CPlantBox and GroIMP." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/01_encoding3d_alone/run.sh b/tutorials/01_encoding3d_alone/run.sh new file mode 100755 index 0000000..78a7008 --- /dev/null +++ b/tutorials/01_encoding3d_alone/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Runs tutorial 1 from start to end (as encoding3d's CI does): every cell of the notebook, its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +python -m pytest --nbmake encoding3d_alone.ipynb diff --git a/tutorials/02a_metafspm/README.md b/tutorials/02a_metafspm/README.md new file mode 100644 index 0000000..abdee67 --- /dev/null +++ b/tutorials/02a_metafspm/README.md @@ -0,0 +1,26 @@ +# Tutorial 2a: metafspm + +**What you learn:** how a metafspm scene is saved as encoding3d frames while it runs, how any platform reads them, +how to keep log files small, and how a frame becomes metafspm's data structures again. + +**What you need** (not encoding3d's requirements: metafspm's own environment). metafspm is installed from source, in +the environment its repository describes, with encoding3d beside it: + +```bash +git clone -b release2026 https://github.com/openalea/metafspm.git +git clone https://github.com/openalea/encoding3d.git +mamba env create -n metafspm -f metafspm/conda/environment.yml +mamba activate metafspm +pip install -e encoding3d -e metafspm +pip install jupyter nbmake +``` + +**How to run it:** open `metafspm.ipynb` in Jupyter, or run `./run.sh`. It writes its files in `outputs/`. + +In the notebook you + +1. write a small root growth component (a toy: a tip elongating at a constant rate, cut every centimetre); +2. grow two plants in a scene for two days, a frame recorded every six hours (`SceneRecorder`); +3. read those frames with encoding3d alone, as any platform would; +4. record smaller frames for logging, keeping two scales and two variables; +5. turn the last frame back into metafspm's data structures, exactly as they were (`import_frame`). diff --git a/tutorials/02a_metafspm/metafspm.ipynb b/tutorials/02a_metafspm/metafspm.ipynb new file mode 100644 index 0000000..b728f41 --- /dev/null +++ b/tutorials/02a_metafspm/metafspm.ipynb @@ -0,0 +1,304 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Tutorial 2a: metafspm writes and reads encoding3d\n", + "\n", + "[metafspm](https://github.com/openalea/metafspm) is OpenAlea's framework for plant models made of **components**: one\n", + "for root growth, one for water flow, one for light... A **scene** grows several plants together. This tutorial shows\n", + "how a metafspm scene is saved as encoding3d files, and read back:\n", + "\n", + "1. a small root growth component, written here for the tutorial (a toy, not a real model);\n", + "2. a scene of two plants, a frame recorded at each step;\n", + "3. those frames read with encoding3d alone, as any other platform would;\n", + "4. smaller frames, keeping only some scales or variables;\n", + "5. a frame turned back into metafspm's data structures, exactly as they were.\n", + "\n", + "**What you need:** metafspm and encoding3d installed in the same environment (this folder's `README.md`)." + ] + }, + { + "cell_type": "code", + "id": "cell-1", + "metadata": {}, + "source": [ + "import os\n", + "from dataclasses import dataclass\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import encoding3d\n", + "from openalea.encoding3d.validate import validate_sequence\n", + "from openalea.metafspm.coupling.component import StructuralComponent, parameter, state_variable\n", + "from openalea.metafspm.data_structure.configs import PropsConfig, ScalesConfig as scales\n", + "from openalea.metafspm.io.encoding3d import import_frame\n", + "from openalea.metafspm.scene.scene import Scene, SceneRecorder\n", + "from openalea.metafspm.solve.decorator import actual, segmentation\n", + "\n", + "OUTPUTS = \"outputs\"" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## 1. A root growth component\n", + "\n", + "A metafspm component declares its **variables** (with units, the scale they live at, their kind) and its\n", + "**processes**, the methods run at each time step. Ours grows one root axis: its tip elongates at a constant rate,\n", + "and when it is longer than 1 cm it is cut, a new tip continuing it. Each segment knows its two ends (`x1` … `z2`) and\n", + "its radius, so it can be drawn.\n", + "\n", + "`initiate_plant` builds each plant's first segment, under the scales metafspm uses (plant, axis, growth unit,\n", + "phytomer, organ, segment)." + ] + }, + { + "cell_type": "code", + "id": "cell-3", + "metadata": {}, + "source": [ + "SEGMENT = 0.01 # m\n", + "UNIT = dict(unit_comment=\"\", value_comment=\"\", references=\"\", DOI=[], min_value=None, max_value=None)\n", + "\n", + "\n", + "@dataclass\n", + "class ToyRootGrowth(StructuralComponent):\n", + " \"\"\"A root axis: the tip elongates at a constant rate, cut every 1 cm (a toy, for the tutorial).\"\"\"\n", + " length: float = state_variable(unit=\"m\", description=\"the segment's length\", default=0., scale=scales.SubOrgan,\n", + " kind=\"extensive\", **UNIT)\n", + " radius: float = state_variable(unit=\"m\", description=\"the segment's radius\", default=0.0005,\n", + " scale=scales.SubOrgan, kind=\"intensive\", **UNIT)\n", + " is_apex: float = state_variable(unit=\"adim\", description=\"1 for the tip\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " x1: float = state_variable(unit=\"m\", description=\"proximal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " y1: float = state_variable(unit=\"m\", description=\"proximal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " z1: float = state_variable(unit=\"m\", description=\"proximal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " x2: float = state_variable(unit=\"m\", description=\"distal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " y2: float = state_variable(unit=\"m\", description=\"distal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " z2: float = state_variable(unit=\"m\", description=\"distal end\", default=0., scale=scales.SubOrgan,\n", + " kind=\"descriptor\", **UNIT)\n", + " elongation_rate: float = parameter(unit=\"m.s-1\", description=\"the tip's elongation rate\", default=2e-7,\n", + " **UNIT)\n", + "\n", + " @classmethod\n", + " def initiate_plant(cls, g, plant, scenario):\n", + " s = g.scales\n", + " axis = g.add_component(plant, **PropsConfig(scale=s.Axis, edge_type=\"/\", label=g.labels.Axis.Root))\n", + " unit = g.add_component(axis, **PropsConfig(scale=s.GrowthUnit, edge_type=\"/\", label=g.labels.GrowthUnit.Root))\n", + " phytomer = g.add_component(unit, **PropsConfig(scale=s.Phytomer, edge_type=\"/\", label=g.labels.Phytomer.Root))\n", + " organ = g.add_component(phytomer, **PropsConfig(scale=s.Organ, edge_type=\"/\",\n", + " label=g.labels.Organ.RootInternode))\n", + " tip = g.add_component(organ, **PropsConfig(scale=s.SubOrgan, edge_type=\"/\", label=g.labels.SubOrgan.RootSegment))\n", + " x, y, z = (g.property(name)[plant] for name in (\"x\", \"y\", \"z\"))\n", + " for name, value in ((\"length\", 0.002), (\"radius\", 0.0005), (\"is_apex\", 1.), (\"x1\", x), (\"y1\", y), (\"z1\", z),\n", + " (\"x2\", x), (\"y2\", y), (\"z2\", z - 0.002)):\n", + " g.property(name)[tip] = value\n", + "\n", + " def _tips(self):\n", + " is_apex = self.mtg.property(\"is_apex\")\n", + " return [v for v, flag in is_apex.items() if flag]\n", + "\n", + " @actual\n", + " def _elongation(self):\n", + " \"\"\"The tips grow downwards.\"\"\"\n", + " g = self.mtg\n", + " length, z2 = g.property(\"length\"), g.property(\"z2\")\n", + " rate = dict(zip(self.data_structure.entity_ids(\"node\").tolist(),\n", + " self.parameter_values(\"elongation_rate\").tolist()))\n", + " for tip in self._tips():\n", + " growth = rate[tip] * self.dt\n", + " length[tip] += growth\n", + " z2[tip] -= growth\n", + "\n", + " @segmentation\n", + " def _segmentation(self):\n", + " \"\"\"A tip longer than 1 cm becomes a segment, continued by a new tip.\"\"\"\n", + " g = self.mtg\n", + " props = {name: g.property(name) for name in (\"length\", \"is_apex\", \"radius\", \"x1\", \"y1\", \"z1\", \"x2\", \"y2\", \"z2\")}\n", + " for tip in self._tips():\n", + " if props[\"length\"][tip] <= SEGMENT:\n", + " continue\n", + " rest = props[\"length\"][tip] - SEGMENT\n", + " props[\"length\"][tip], props[\"is_apex\"][tip] = SEGMENT, 0.\n", + " props[\"z2\"][tip] = props[\"z1\"][tip] - SEGMENT\n", + " x, y, z = props[\"x2\"][tip], props[\"y2\"][tip], props[\"z2\"][tip]\n", + " g.add_child(tip, **PropsConfig(scale=g.scales.SubOrgan, edge_type=\"<\", label=g.labels.SubOrgan.RootSegment,\n", + " length=rest, is_apex=1., radius=props[\"radius\"][tip],\n", + " x1=x, y1=y, z1=z, x2=x, y2=y, z2=z - rest))\n", + "\n", + "\n", + "class ToyRoots:\n", + " \"\"\"The plant model: one component, run once per step.\"\"\"\n", + " initiators = (ToyRootGrowth,)\n", + "\n", + " def __init__(self, data_structure, time_step):\n", + " self.growth = ToyRootGrowth(data_structure=data_structure)\n", + " self.components = [self.growth]\n", + "\n", + " def run(self):\n", + " self.growth()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## 2. A scene, recorded\n", + "\n", + "The **planting table** says which plants grow where. `SceneRecorder(frame_period=1)` writes a frame at the start and\n", + "after every step, with the index `scene.fspm.json`: an encoding3d sequence." + ] + }, + { + "cell_type": "code", + "id": "cell-5", + "metadata": {}, + "source": [ + "planting = pd.DataFrame([dict(plant=f\"plant_{i}\", model=ToyRoots, x=0.05 * i, y=0., z=0., rotation=0., scenario={})\n", + " for i in range(2)])\n", + "planting.attrs.update(xrange=0.2, yrange=0.1)\n", + "scene = Scene(planting=planting, time_step=6 * 3600., recorder=SceneRecorder(os.path.join(OUTPUTS, \"full\"),\n", + " summaries=False, frame_period=1))\n", + "for _ in range(8): # two days, a step every six hours\n", + " scene.run()\n", + "index = os.path.join(OUTPUTS, \"full\", \"frames\", \"scene.fspm.json\")\n", + "report = validate_sequence(index)\n", + "# warnings: metafspm also writes its graph's own properties (positions, types), which have no declared unit\n", + "print(\"valid:\", report.ok, \"|\", len(report.errors), \"errors,\", len(report.warnings), \"warnings\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "## 3. The frames, read by encoding3d alone\n", + "\n", + "Nothing in these files needs metafspm: any platform reads them. Each MPG scale is an entity set, every variable a\n", + "column with its unit, and the segments' triangles a mesh." + ] + }, + { + "cell_type": "code", + "id": "cell-7", + "metadata": {}, + "source": [ + "rows = []\n", + "for frame in encoding3d.read_sequence(index):\n", + " plants = frame.data_structures[0]\n", + " segments = plants.entity_set(\"SubOrgan\")\n", + " rows.append(dict(time_h=frame.time / 3600 if frame.time_unit == \"s\" else frame.time, step=frame.step,\n", + " segments=segments.count, total_length_cm=100 * segments.columns[\"length\"].sum()))\n", + "print(\"entity sets:\", [entity_set.name for entity_set in plants.entity_sets])\n", + "print(\"length's declaration:\", segments.variables[\"length\"])\n", + "pd.DataFrame(rows)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "## 4. Smaller frames for logging\n", + "\n", + "A long simulation writes many frames. `frame_options` keeps only what is needed: here the plants and the segments\n", + "(not the intermediate scales), and two variables. Such frames are valid encoding3d files, for outputs only." + ] + }, + { + "cell_type": "code", + "id": "cell-9", + "metadata": {}, + "source": [ + "scene = Scene(planting=planting, time_step=6 * 3600.,\n", + " recorder=SceneRecorder(os.path.join(OUTPUTS, \"log\"), summaries=False, frame_period=1,\n", + " frame_options=dict(scales=[\"Plant\", \"SubOrgan\"], variables=[\"length\", \"radius\"],\n", + " geometry=False)))\n", + "for _ in range(8):\n", + " scene.run()\n", + "log = os.path.join(OUTPUTS, \"log\", \"frames\")\n", + "full = os.path.join(OUTPUTS, \"full\", \"frames\")\n", + "size = lambda folder: sum(os.path.getsize(os.path.join(folder, name)) for name in os.listdir(folder))\n", + "small = encoding3d.read(os.path.join(log, sorted(os.listdir(log))[-2]))\n", + "print([entity_set.name for entity_set in small.data_structures[0].entity_sets],\n", + " small.data_structures[0].entity_set(\"SubOrgan\").variable_names())\n", + "print(f\"full: {size(full) / 1e3:.0f} kB, log: {size(log) / 1e3:.0f} kB\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## 5. Back into metafspm\n", + "\n", + "A full frame rebuilds metafspm's data structures exactly: the same graph, the same variables, bit for bit. A\n", + "simulation can continue from it." + ] + }, + { + "cell_type": "code", + "id": "cell-11", + "metadata": {}, + "source": [ + "last = sorted(name for name in os.listdir(full) if name.endswith(\".glb\"))[-1]\n", + "rebuilt = import_frame(os.path.join(full, last))\n", + "population = scene.populations[0].name\n", + "ds = rebuilt[population]\n", + "print(population, \"has\", ds.n_nodes(), \"segments; total length\",\n", + " round(100 * float(np.sum(ds.get(\"length\"))), 3), \"cm\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "**In ParaView:** with the plugin loaded (tutorial 1), open `outputs/full/frames/scene.fspm.json` with the encoding3d\n", + "reader and press *Play*: the two roots grow.\n", + "\n", + "## Try it yourself\n", + "\n", + "- Change `elongation_rate` for one plant only, through its `scenario` in the planting table (`{\"elongation_rate\": 4e-7}`).\n", + "- Record a frame every four steps (`frame_period=4`) and compare the files.\n", + "- Keep the plants' scale only in the log, and read their number of segments from the full frames instead." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/02a_metafspm/run.sh b/tutorials/02a_metafspm/run.sh new file mode 100755 index 0000000..ffc3607 --- /dev/null +++ b/tutorials/02a_metafspm/run.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Runs tutorial 2a from start to end, in an environment with metafspm (README.md); its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +python -m pytest --nbmake metafspm.ipynb diff --git a/tutorials/02b_virtualplantlab/Project.toml b/tutorials/02b_virtualplantlab/Project.toml new file mode 100644 index 0000000..7b1c30f --- /dev/null +++ b/tutorials/02b_virtualplantlab/Project.toml @@ -0,0 +1,16 @@ +[deps] +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" +PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" +PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[sources] +encoding3d = {path = "../../bindings/julia/encoding3d"} + +[compat] +MultiScaleTreeGraph = "0.16" +PlantGeomPrimitives = "1" +PlantGeomTurtle = "1" +PlantGraphs = "1" +julia = "1.12" diff --git a/tutorials/02b_virtualplantlab/README.md b/tutorials/02b_virtualplantlab/README.md new file mode 100644 index 0000000..f6e98df --- /dev/null +++ b/tutorials/02b_virtualplantlab/README.md @@ -0,0 +1,28 @@ +# Tutorial 2b: VirtualPlantLab + +**What you learn:** how a VirtualPlantLab plant (a PlantGraphs graph grown by rules, drawn by a turtle) is written as +encoding3d frames with its geometry, how a frame becomes a VirtualPlantLab graph again that keeps growing, and how an +MTG (MultiScaleTreeGraph.jl, used by VirtualPlantLab and PlantSimEngine) goes to and from encoding3d. + +**What you need** (not encoding3d's requirements: this folder's own Julia project, `Project.toml`): Julia 1.12 or +later. The project takes encoding3d.jl from this repository and VirtualPlantLab's packages (PlantGraphs, +PlantGeomTurtle, PlantGeomPrimitives) and MultiScaleTreeGraph.jl from Julia's registry; encoding3d.jl's extensions +for them load with them. These extensions are proposals for VirtualPlantLab and MTG.jl +(`bindings/julia/README.md`). + +**How to run it:** `./run.sh` (the first time, Julia installs and compiles the packages: a few minutes). It writes its +files in `outputs/`. The script, `virtualplantlab.jl`, is in four parts: + +1. **A plant.** Three node types (`Internode`, `Leaf`, `Meristem`), how the turtle draws them (a cylinder, an + ellipse), and a rule: each step, the meristem makes an internode, two opposite leaves and a new meristem. +2. **A frame per step.** `from_graph` writes the graph's nodes as one entity set, `Organ`, each node a row with its + type as label and its fields as columns (`length` declared in metres); `add_geometry!` adds the turtle's + triangles as a mesh, each vertex knowing its node. Five steps make a sequence. +3. **Back into VirtualPlantLab.** `to_graph` rebuilds each row as a node of its type; wrapped in a `Graph` with the + same rule, the plant grows on from where the file left it. +4. **An MTG.** The conformance root system (written by another platform, three scales) becomes an MTG with + `to_mtg` (the MTG's scale 0 is its scene, its root), and an encoding3d data structure again with `from_mtg`. + +Open `outputs/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1) to see the plant grow. + +Try it yourself: give the leaves a field `area`, declare it in `cm2`, and colour the leaves by it in a `views.json`. diff --git a/tutorials/02b_virtualplantlab/run.sh b/tutorials/02b_virtualplantlab/run.sh new file mode 100755 index 0000000..1e10a21 --- /dev/null +++ b/tutorials/02b_virtualplantlab/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Runs tutorial 2b (Julia 1.12 or later, with VirtualPlantLab's packages: README.md); its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +julia --project=. -e 'using Pkg; Pkg.instantiate()' +julia --project=. virtualplantlab.jl diff --git a/tutorials/02b_virtualplantlab/virtualplantlab.jl b/tutorials/02b_virtualplantlab/virtualplantlab.jl new file mode 100644 index 0000000..480a00c --- /dev/null +++ b/tutorials/02b_virtualplantlab/virtualplantlab.jl @@ -0,0 +1,68 @@ +# Tutorial 2b: VirtualPlantLab writes and reads encoding3d (README.md explains each part). +# +# julia --project=. -e 'using Pkg; Pkg.instantiate()' # once +# julia --project=. virtualplantlab.jl +using encoding3d +import MultiScaleTreeGraph as MTG # MTG.jl: its names qualified (PlantGraphs has some too) +using PlantGeomPrimitives +using PlantGeomTurtle +using PlantGraphs + +const OUTPUTS = joinpath(@__DIR__, "outputs") +mkpath(OUTPUTS) + +# ── 1. A VirtualPlantLab plant: its node types, how they are drawn, how it grows ───────────────────────── +struct Internode <: PlantGraphs.Node + length::Float64 + width::Float64 +end +struct Leaf <: PlantGraphs.Node + length::Float64 + width::Float64 +end +struct Meristem <: PlantGraphs.Node end + +# the turtle draws an internode as a cylinder (and moves to its end), a leaf as an ellipse +PlantGeomTurtle.feed!(turtle::Turtle, node::Internode, data) = + HollowCylinder!(turtle; length = node.length, width = node.width, height = node.width, move = true) +PlantGeomTurtle.feed!(turtle::Turtle, node::Leaf, data) = Ellipse!(turtle; length = node.length, width = node.width, move = false) + +# each step, the meristem makes an internode, two opposite leaves, and a meristem above them +grow = Rule(Meristem, rhs = meristem -> Internode(0.04, 0.004) + (RU(60.0) + Leaf(0.06, 0.02), RU(-60.0) + Leaf(0.06, 0.02)) + + RH(90.0) + Meristem()) +plant = Graph(axiom = Internode(0.04, 0.004) + Meristem(), rules = grow) + +# ── 2. A frame per step: the graph's nodes as an entity set, its geometry as a mesh ───────────────────── +length_declaration = Dict("length" => Variable(unit = "m", standard_name = "length", kind = "extensive"), + "width" => Variable(unit = "m", kind = "descriptor")) +frames = Frame[] +for step in 0:4 + step > 0 && rewrite!(plant) + data_structure = from_graph(plant; id = "plant", variables = length_declaration) # one entity set, "Organ" + add_geometry!(data_structure, plant) # the turtle's triangles + push!(frames, Frame(data_structures = [data_structure], time = Float64(step), time_unit = "d", step = step)) +end +index = write_sequence(joinpath(OUTPUTS, "frames"), frames) +organs = entity_set(frames[end].data_structures[1], "Organ") +println("written: ", index) +println("last frame: ", row_count(organs), " organs (", join(unique(organs.columns["label"]), ", "), "), ", + size(frames[end].data_structures[1].meshes[1].positions, 1), " mesh vertices") + +# ── 3. Back into a VirtualPlantLab graph: each row rebuilt as a node of its type ─────────────────────── +last_frame = read_frame(joinpath(OUTPUTS, "frames", "frame_000004.glb")) +graph, ids = to_graph(last_frame.data_structures[1]; + types = Dict("Internode" => Internode, "Leaf" => Leaf, "Meristem" => Meristem)) +rebuilt = Graph(axiom = graph, rules = grow) # a full VirtualPlantLab graph again, with its growth rule +leaves = apply(rebuilt, Query(Leaf)) +println("rebuilt: ", length(leaves), " leaves, total leaf length ", round(sum(leaf.length for leaf in leaves); digits = 3), " m") +rewrite!(rebuilt) # and it grows on from where the file left it +println("one more step: ", length(apply(rebuilt, Query(Leaf))), " leaves") + +# ── 4. MTG.jl: another platform's root system (the conformance file) as an MTG, and back ────────────── +conformance = joinpath(@__DIR__, "..", "..", "conformance", "root_system.glb") +roots = read_frame(conformance).data_structures[1] +mtg = to_mtg(roots) +println("as an MTG: ", length(MTG.traverse(mtg, node -> node)), " nodes, scales ", + sort(unique(Int[MTG.scale(node) for node in MTG.traverse(mtg, node -> node)]))) +again = from_mtg(mtg; id = "root_system", scales = Dict(1 => "Plant", 2 => "Axis", 6 => "SubOrgan")) +println("and back: ", [string(set.name, " ", row_count(set)) for set in again.entity_sets]) diff --git a/tutorials/02c_cplantbox/CMakeLists.txt b/tutorials/02c_cplantbox/CMakeLists.txt new file mode 100644 index 0000000..2ff882f --- /dev/null +++ b/tutorials/02c_cplantbox/CMakeLists.txt @@ -0,0 +1,21 @@ +# Tutorial 2c: CPlantBox writes and reads encoding3d. Built on CPlantBox's sources and library (README.md): +# +# cmake -S . -B build -DCPLANTBOX_SOURCE_DIR=/path/to/CPlantBox -DCPLANTBOX_LIBRARY=/path/to/libCPlantBox.a +# cmake --build build && ./build/cplantbox_tutorial /path/to/CPlantBox/modelparameter/structural outputs +cmake_minimum_required(VERSION 3.16) +project(encoding3d_tutorial_cplantbox LANGUAGES CXX) + +set(CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder") +set(CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox.a)") +if(NOT CPLANTBOX_SOURCE_DIR OR NOT CPLANTBOX_LIBRARY) + message(FATAL_ERROR "give CPlantBox: -DCPLANTBOX_SOURCE_DIR=... -DCPLANTBOX_LIBRARY=... (README.md)") +endif() + +# encoding3d's C++ codec and CPlantBox's adapter, from this repository +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "${CPLANTBOX_SOURCE_DIR}" CACHE PATH "" FORCE) +set(ENCODING3D_CPLANTBOX_LIBRARY "${CPLANTBOX_LIBRARY}" CACHE FILEPATH "" FORCE) +set(ENCODING3D_BUILD_TESTS OFF CACHE BOOL "" FORCE) +add_subdirectory(../../bindings/cpp encoding3d) + +add_executable(cplantbox_tutorial cplantbox_tutorial.cpp) +target_link_libraries(cplantbox_tutorial PRIVATE encoding3d::cplantbox) diff --git a/tutorials/02c_cplantbox/README.md b/tutorials/02c_cplantbox/README.md new file mode 100644 index 0000000..20f594a --- /dev/null +++ b/tutorials/02c_cplantbox/README.md @@ -0,0 +1,36 @@ +# Tutorial 2c: CPlantBox + +**What you learn:** how a CPlantBox plant (shoot and roots) is written as encoding3d frames while it grows, with its +geometry, a soil grid and which soil cell each root segment is in; and how a frame, CPlantBox's or another +platform's, becomes a CPlantBox root system again. + +**What you need** (not encoding3d's requirements): a C++17 compiler, CMake, and CPlantBox built as a library. The +adapter it uses (`bindings/cpp/cplantbox/`) is a proposal for CPlantBox, kept in this repository until CPlantBox +adopts it. Build CPlantBox once (the commit the adapter was last tested with): + +```bash +git clone https://github.com/Plant-Root-Soil-Interactions-Modelling/CPlantBox.git +git -C CPlantBox checkout 17e1fab617e72fa5e2773dfee723083bdf515053 +cmake -S CPlantBox -B CPlantBox-build -DCMAKE_BUILD_TYPE=Release -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF \ + -DCPB_BUILD_SHARED=OFF -DCPB_NATIVE_CPU_TUNING=OFF +cmake --build CPlantBox-build --target CPlantBox -j 4 +``` + +**How to run it:** `CPLANTBOX=/path/to/CPlantBox CPLANTBOX_LIBRARY=/path/to/CPlantBox-build/src/libCPlantBox.a ./run.sh`. +It builds the program (`cplantbox_tutorial.cpp`, with encoding3d's C++ codec from this repository), runs it, and +writes its files in `outputs/`. The program is in four parts: + +1. **A plant.** CPlantBox's `fspm2023` parameters (a shoot and its roots), grown one day at a time. +2. **A frame a day.** `to_data_structure` writes the plant, its organs and its segments with CPlantBox's variables; + `add_geometry` adds the shoot's blades and stems as a mesh; `soil_grid` and `segments_in_soil` add a soil grid + (2 cm cells) and the mapping from root segments to cells; `add_cell_values` puts a soil water potential in the + cells (a toy gradient, standing for the soil solver CPlantBox would be coupled with). Ten days make a sequence. +3. **Back into CPlantBox.** `to_root_system` rebuilds the last frame's roots as a CPlantBox root system. +4. **Another platform's frame.** The conformance root system (written by another platform, at three scales) becomes a + CPlantBox root system too: the adapter reads frames by what they hold, not by who wrote them. + +Open `outputs/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1): the shoot's mesh, the roots +as tubes, the soil cells. + +Try it yourself: write CPlantBox's own `age` of each segment in a `views.json` colouring, or grow the plant for 30 +days with Anagallis' root parameters (`plant/Anagallis_femina_Leitner_2010.xml`). diff --git a/tutorials/02c_cplantbox/cplantbox_tutorial.cpp b/tutorials/02c_cplantbox/cplantbox_tutorial.cpp new file mode 100644 index 0000000..0e2a507 --- /dev/null +++ b/tutorials/02c_cplantbox/cplantbox_tutorial.cpp @@ -0,0 +1,81 @@ +// Tutorial 2c: CPlantBox writes and reads encoding3d (README.md explains each part). +// +// cplantbox_tutorial MODELPARAMETER OUT MODELPARAMETER: CPlantBox's modelparameter/structural folder +#include +#include +#include +#include + +#include "encoding3d/cplantbox.hpp" +#include "encoding3d/encoding3d.hpp" + +#include "MappedOrganism.h" +#include "Plant.h" + +namespace filesystem = std::filesystem; +using namespace encoding3d; + +int main(int count, char **arguments) { + if (count != 3) { + std::cerr << "usage: cplantbox_tutorial MODELPARAMETER OUT\n"; + return 2; + } + const filesystem::path parameters = arguments[1], out = arguments[2]; + filesystem::create_directories(out); + + // ── 1. A CPlantBox plant (shoot and roots), grown day by day ─────────────────────────────────────── + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "fspm2023.xml").string(), "plant", true, false); + plant->initialize(false); + + // ── 2. A frame a day: the plant with its geometry, a soil grid, which cell each segment is in ───────── + std::vector frames; + for (int day = 1; day <= 10; ++day) { + plant->simulate(1., false); + auto data_structure = cplantbox::to_data_structure(*plant); // plant, organs, segments + cplantbox::add_geometry(data_structure, plant); // the shoot's blades and stems + plant->setRectangularGrid(CPlantBox::Vector3d(-10, -10, -30), CPlantBox::Vector3d(10, 10, 0), + CPlantBox::Vector3d(10, 10, 15), false); // cm: 2 cm cells, segments not cut + auto soil = cplantbox::soil_grid(*plant); + const auto mapping = cplantbox::segments_in_soil(*plant, data_structure, soil); + + // a soil water potential per cell, from a solver CPlantBox would be coupled with: here a toy gradient + std::vector potential; + for (int z = 0; z < 15; ++z) // CPlantBox's cell order: x fastest + for (int y = 0; y < 10; ++y) + for (int x = 0; x < 10; ++x) potential.push_back(-300. + 10. * z); + Variable declaration; + declaration.unit = "cm[H2O]"; + declaration.standard_name = "soil_water_potential"; + declaration.description = "a toy gradient, wetter with depth"; + cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", potential, declaration}}); + + Frame frame; + frame.time = plant->getSimTime(); + frame.time_unit = "d"; + frame.step = day; + frame.data_structures = {data_structure, soil}; + frame.mappings = {mapping}; + frames.push_back(frame); + std::cout << "day " << day << ": " << data_structure.entity_set("Organ").row_count() << " organs, " + << data_structure.entity_set("SubOrgan").row_count() << " segments, " + << mapping.row_count() << " in the soil\n"; + } + const auto index = write_sequence((out / "frames").string(), frames); + std::cout << "written: " << index << "\n"; + + // ── 3. A frame back as a CPlantBox root system ─────────────────────────────────────────────────────── + const auto last = read_frame((out / "frames" / "frame_000009.glb").string()); + auto roots = cplantbox::to_root_system(last.data_structure("plant")); + const auto lengths = roots->getParameter("length"); + std::cout << "read back as CPlantBox roots: " << roots->getOrgans(CPlantBox::Organism::ot_root).size() + << " roots, " << std::accumulate(lengths.begin(), lengths.end(), 0.) << " cm long in all\n"; + + // ── 4. Another platform's frame (the conformance root system) as a CPlantBox root system ──────────── + const filesystem::path conformance = filesystem::path(__FILE__).parent_path() / ".." / ".." / "conformance"; + const auto other = read_frame((conformance / "root_system.glb").string()); + auto rebuilt = cplantbox::to_root_system(other.data_structures[0]); + std::cout << "another platform's root system in CPlantBox: " << rebuilt->getOrgans().size() << " organs, " + << rebuilt->getNumberOfSegments() << " segments\n"; + return 0; +} diff --git a/tutorials/02c_cplantbox/run.sh b/tutorials/02c_cplantbox/run.sh new file mode 100755 index 0000000..61e6ebd --- /dev/null +++ b/tutorials/02c_cplantbox/run.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Runs tutorial 2c (README.md): CPLANTBOX (CPlantBox's source folder) and CPLANTBOX_LIBRARY (its libCPlantBox.a) set, +# its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +: "${CPLANTBOX:?set CPLANTBOX to the CPlantBox source folder}" "${CPLANTBOX_LIBRARY:?set CPLANTBOX_LIBRARY to libCPlantBox.a}" +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DCPLANTBOX_SOURCE_DIR="$CPLANTBOX" -DCPLANTBOX_LIBRARY="$CPLANTBOX_LIBRARY" +cmake --build build -j 4 +./build/cplantbox_tutorial "$CPLANTBOX/modelparameter/structural" outputs +if command -v encoding3d > /dev/null; then encoding3d validate outputs/frames/scene.fspm.json | head -1; fi diff --git a/tutorials/02d_groimp/README.md b/tutorials/02d_groimp/README.md new file mode 100644 index 0000000..1e3043c --- /dev/null +++ b/tutorials/02d_groimp/README.md @@ -0,0 +1,40 @@ +# Tutorial 2d: GroIMP + +**What you learn:** how a GroIMP graph (its nodes, their managed fields, the plant's scales and its `>` and `+` +edges) is written as encoding3d frames, and turned back into GroIMP nodes; and how GroIMP users export a whole +project, with its 3D geometry, from GroIMP itself. + +**What you need** (not encoding3d's requirements): Java 21 and Maven. The program uses GroIMP's Graph module, from +GroIMP's package registry (no GroIMP application), and GroIMP's adapter (`bindings/java/groimp/`), a proposal for +GroIMP kept in this repository until GroIMP adopts it. + +**How to run it:** `./run.sh`. It installs encoding3d's Java codec and GroIMP's adapter in Maven's local repository, +then compiles and runs the program (`src/main/java/tutorial/`), which writes its files in `outputs/`. It is in three +parts: + +1. **A GroIMP plant.** Node classes as an RGG model declares them: a `Plant`, `Internode`s and `Leaf`s, whose fields + GroIMP manages (`Managed.java` declares them, as GroIMP's XL compiler does for a model's classes). The plant is + made of its internodes (a `/` edge), each internode followed by the next (`>`) and bearing a leaf (`+`). It grows + by code standing for an RGG rule: a new internode and leaf on top, every leaf longer. + GroIMP's graph module has no geometry: in the GroIMP application, shape nodes (cylinders, …) place the organs in + space, and the plugin meshes them (below). So that viewers can draw this plant, each organ keeps its two ends + (`x1` … `z2`, declared as the vocabulary's `proximal_end_x` … `distal_end_z`): the internodes stacked upwards, the + leaves leaning out to alternate sides. +2. **A frame per step.** `GroImp.toDataStructure` writes the plant and its organs as two entity sets, the edges as + `parent` and `edge_type`, the managed fields as columns (`length` and `radius` declared in metres). Six steps make + a sequence. +3. **Back into GroIMP.** `GroImp.toGraph` rebuilds each row as a node of its class, its fields set. + +Open `outputs/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1): the stem and its leaves, +drawn from their ends, grow step by step. + +## From GroIMP itself: the plugin + +GroIMP's encoding3d plugin (`bindings/java/groimp-plugin/`, also a proposal for GroIMP) exports a GroIMP project with +its geometry: in GroIMP, **File > Export** of the 3D view to a `.glb` file, or headless, the command `export3d +plant.glb`. Its README says how to build and install it; `bindings/java/groimp-plugin/test/run_in_groimp.sh` grows +GroIMP's `simpleRapeseed` example headless and exports a frame after each of four runs, a log other platforms can +replay (tutorial 3). + +Try it yourself: add a field `area` to the leaves, manage it, declare it in `m2`, and check it in the frames with +`encoding3d info outputs/frames/frame_000005.glb`. diff --git a/tutorials/02d_groimp/pom.xml b/tutorials/02d_groimp/pom.xml new file mode 100644 index 0000000..781991a --- /dev/null +++ b/tutorials/02d_groimp/pom.xml @@ -0,0 +1,52 @@ + + + + 4.0.0 + org.openalea.tutorials + encoding3d-tutorial-groimp + 0.1.0 + + + 21 + UTF-8 + + + + + groimp + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d-groimp + 0.1.0 + + + de.grogra + graph + 3.0.0 + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + tutorial.GroImpTutorial + outputs + + + + + diff --git a/tutorials/02d_groimp/run.sh b/tutorials/02d_groimp/run.sh new file mode 100755 index 0000000..d39e544 --- /dev/null +++ b/tutorials/02d_groimp/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Runs tutorial 2d (README.md): Java 21 and Maven; encoding3d's Java codec and GroIMP's adapter installed in Maven's +# local repository first (from the repository's root, once). Its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +mvn -q -f ../../bindings/java/pom.xml install -DskipTests +mvn -q -f ../../bindings/java/groimp/pom.xml install -DskipTests +mvn -q compile exec:java +if command -v encoding3d > /dev/null; then encoding3d validate outputs/frames/scene.fspm.json | head -1; fi diff --git a/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java new file mode 100644 index 0000000..1c51d2f --- /dev/null +++ b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java @@ -0,0 +1,142 @@ +package tutorial; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Sequences; +import org.openalea.encoding3d.Variable; +import org.openalea.encoding3d.groimp.GroImp; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** Tutorial 2d: a GroIMP graph written and read as encoding3d (README.md explains each part). */ +public final class GroImpTutorial { + private GroImpTutorial() {} + + // ── 1. A GroIMP plant: node classes with managed fields, as an RGG model declares them ────────────────── + public static final class Plant extends Node { + String species = "Wheat"; + public static final NType $TYPE = Managed.type(Plant.class, "species"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Plant(); } + } + + // GroIMP's graph module has no geometry: in the GroIMP application, shape nodes (cylinders...) place the organs in + // space, and the plugin meshes them. Here each organ keeps its two ends (x1 ... z2), so that viewers draw it. + public static final class Internode extends Node { + double length = 0.02; + float radius = 0.002f; + double x1, y1, z1, x2, y2, z2 = 0.02; // m: its lower and upper ends + public static final NType $TYPE = Managed.type(Internode.class, "length", "radius", + "x1", "y1", "z1", "x2", "y2", "z2"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Internode(); } + } + + public static final class Leaf extends Node { + double length = 0.01; + float radius = 0.001f; + double x1, y1, z1, x2, y2, z2; // m: where it leaves its internode, and its tip + double side = 1; // which side it leans to: +1 or -1 (not managed) + public static final NType $TYPE = Managed.type(Leaf.class, "length", "radius", + "x1", "y1", "z1", "x2", "y2", "z2"); + + /** Its tip, from its base: leaning 45 degrees to its side. */ + void place() { + x2 = x1 + side * length * Math.sqrt(0.5); + y2 = y1; + z2 = z1 + length * Math.sqrt(0.5); + } + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Leaf(); } + } + + /** The plant grows as an RGG rule would make it: a new internode bearing a leaf on top, every leaf longer. */ + static Internode grow(Internode top, List leaves) { + for (Leaf leaf : leaves) { + leaf.length += 0.02; + leaf.place(); + } + Internode next = new Internode(); + next.x1 = top.x2; next.y1 = top.y2; next.z1 = top.z2; // on top of the previous one + next.x2 = next.x1; next.y2 = next.y1; next.z2 = next.z1 + next.length; + Leaf leaf = new Leaf(); + leaf.x1 = next.x2; leaf.y1 = next.y2; leaf.z1 = next.z2; + leaf.side = leaves.size() % 2 == 0 ? 1 : -1; // alternate sides + leaf.place(); + top.addEdgeBitsTo(next, Graph.SUCCESSOR_EDGE, null); // GroIMP's '>' edge + next.addEdgeBitsTo(leaf, Graph.BRANCH_EDGE, null); // GroIMP's '+' edge + leaves.add(leaf); + return next; + } + + /** How the graph is written: its scale below the plant is the vocabulary's Organ, lengths in metres. */ + static GroImp.Options options() { + GroImp.Options options = new GroImp.Options(); + options.id = "plant"; + options.scaleNames.put(2, "Organ"); + options.scaleRanks.put(2, 5L); + options.scaleScales.put(2, "Organ"); + options.labels.put("Internode", "StemInternode"); + options.variables.put("length", new Variable("m", "length", "extensive", "state")); + options.variables.put("radius", new Variable("m", "radius", "intensive", "state")); + options.variables.put("species", new Variable("1", null, "descriptor", "parameter")); + String[] ends = {"x1", "y1", "z1", "x2", "y2", "z2"}; + String[] standardNames = {"proximal_end_x", "proximal_end_y", "proximal_end_z", "distal_end_x", "distal_end_y", + "distal_end_z"}; + for (int index = 0; index < ends.length; index++) + options.variables.put(ends[index], new Variable("m", standardNames[index], "descriptor", "state")); + return options; + } + + public static void main(String[] arguments) throws Exception { + Path out = Path.of(arguments.length > 0 ? arguments[0] : "outputs"); + GraphManager graph = new GraphManager(new ServerConnection(null), "tutorial", false, false); + Node root = new Node(); + Plant plant = new Plant(); + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Internode top = new Internode(); + plant.addEdgeBitsTo(top, Graph.REFINEMENT_EDGE, null); // GroIMP's '/' edge: the plant is made of them + graph.setRoot(Graph.MAIN_GRAPH, root); + List leaves = new ArrayList<>(); + + // ── 2. A frame per step ───────────────────────────────────────────────────────────────────────────── + List frames = new ArrayList<>(); + for (int step = 0; step < 6; step++) { + if (step > 0) top = grow(top, leaves); + DataStructure written = GroImp.toDataStructure(root, options()); + Frame frame = new Frame(); + frame.dataStructures.add(written); + frame.time = step; + frame.step = step; + frames.add(frame); + EntitySet organs = written.entitySet("Organ"); + System.out.println("step " + step + ": " + organs.rowCount() + " organs, scales " + + written.entitySets.stream().map(set -> set.name).toList()); + } + Path index = Sequences.writeSequence(out.resolve("frames"), frames); + System.out.println("written: " + index); + + // ── 3. Back into GroIMP nodes ───────────────────────────────────────────────────────────────────────── + DataStructure last = Codec.read(out.resolve("frames").resolve("frame_000005.glb")).dataStructures.get(0); + Map> classes = Map.of("Plant", Plant::new, "Internode", Internode::new, "Leaf", Leaf::new); + Map nodes = new HashMap<>(); + Node rebuilt = GroImp.toGraph(last, label -> classes.get(label).get(), nodes); + new GraphManager(new ServerConnection(null), "rebuilt", false, false).setRoot(Graph.MAIN_GRAPH, rebuilt); + double leafLength = nodes.values().stream().filter(node -> node instanceof Leaf) + .mapToDouble(node -> ((Leaf) node).length).sum(); + System.out.println("rebuilt as GroIMP nodes: " + nodes.size() + " nodes, leaves " + Math.round(leafLength * 100) + + " cm long in all"); + } +} diff --git a/tutorials/02d_groimp/src/main/java/tutorial/Managed.java b/tutorials/02d_groimp/src/main/java/tutorial/Managed.java new file mode 100644 index 0000000..259fca7 --- /dev/null +++ b/tutorials/02d_groimp/src/main/java/tutorial/Managed.java @@ -0,0 +1,77 @@ +package tutorial; + +import java.lang.reflect.Field; + +import de.grogra.graph.impl.Node; +import de.grogra.graph.impl.Node.NType; +import de.grogra.reflect.Type; + +/** + * GroIMP's "managed fields": the fields of a node class GroIMP knows by name, as its XL compiler declares them for the + * classes of an RGG model. GroIMP's adapter writes them as columns. Here they are declared from plain Java fields by + * reflection, so that the tutorial's node classes stay short. + */ +final class Managed { + private Managed() {} + + /** The NType of {@code type}, with its fields {@code names} managed. */ + static NType type(Class type, String... names) { + NType nodeType = new NType(type); + for (String name : names) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + nodeType.addManagedField(new ManagedField(nodeType, field)); + } catch (NoSuchFieldException error) { + throw new IllegalArgumentException(type.getSimpleName() + " has no field " + name, error); + } + } + nodeType.validate(); + return nodeType; + } + + private static final class ManagedField extends NType.Field { + private final Field field; + + ManagedField(NType type, Field field) { + super(type, field.getName(), PUBLIC | SCO, typeOf(field.getType()), null); + this.field = field; + } + + private static Type typeOf(Class type) { + if (type == double.class) return Type.DOUBLE; + if (type == float.class) return Type.FLOAT; + if (type == int.class) return Type.INT; + if (type == boolean.class) return Type.BOOLEAN; + if (type == String.class) return Type.STRING; + throw new IllegalArgumentException("a managed field of type " + type); + } + + private Object read(Object node) { + try { + return field.get(node); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + private void write(Object node, Object value) { + try { + field.set(node, value); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + @Override public double getDouble(Object node) { return (double) read(node); } + @Override public void setDouble(Object node, double value) { write(node, value); } + @Override public float getFloat(Object node) { return (float) read(node); } + @Override public void setFloat(Object node, float value) { write(node, value); } + @Override public int getInt(Object node) { return (int) read(node); } + @Override public void setInt(Object node, int value) { write(node, value); } + @Override public boolean getBoolean(Object node) { return (boolean) read(node); } + @Override public void setBoolean(Object node, boolean value) { write(node, value); } + @Override public Object getObject(Object node) { return read(node); } + @Override protected void setObjectImpl(Object node, Object value) { write(node, value); } + } +} diff --git a/tutorials/03_metafspm_and_virtualplantlab/Project.toml b/tutorials/03_metafspm_and_virtualplantlab/Project.toml new file mode 100644 index 0000000..a8460d2 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/Project.toml @@ -0,0 +1,12 @@ +[deps] +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[sources] +encoding3d = {path = "../../bindings/julia/encoding3d"} + +[compat] +PlantGraphs = "1" +ZMQ = "1" +julia = "1.12" diff --git a/tutorials/03_metafspm_and_virtualplantlab/README.md b/tutorials/03_metafspm_and_virtualplantlab/README.md new file mode 100644 index 0000000..176275c --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/README.md @@ -0,0 +1,32 @@ +# Tutorial 3: metafspm and VirtualPlantLab, one plant + +**What you learn:** how two platforms, each in its own language and process, work on the same plant and exchange +their values at each step through encoding3d: a coupling session. + +- **metafspm** (Python) grows the roots: each tip elongates as fast as its sugar allows (structural: it creates the + segments). +- **VirtualPlantLab** (Julia, `carbon.jl`) is the carbon model: it splits each segment into two tissue layers, grown + by a PlantGraphs rule, and computes their sugar from the segment's mass (functional, adding entities of its own at + a finer scale). +- encoding3d's **coordinator** runs the session: it keeps the shared plant, passes each model the values it needs + (converting grams to milligrams, copying a segment's mass to its layers, averaging the layers' sugar back to their + segment), and tells each model when to run. + +Both models are toys: the tutorial shows the coupling, not plant science. + +**What you need** (not encoding3d's requirements): metafspm's environment, as in tutorial 2a, and Julia 1.12 or later +for VirtualPlantLab's side (this folder's `Project.toml`: encoding3d.jl from this repository, PlantGraphs, ZMQ). + +**How to run it:** `./run.sh` (`JULIA=/path/to/julia ./run.sh` if Julia is not on the path). It prepares the Julia +project, then runs `coupling.ipynb`, which starts the coordinator, launches `carbon.jl` and runs metafspm's scene. +Its files are in `outputs/`: the coordinator's record of the shared plant (`record/`) and metafspm's own frames +(`metafspm/`). The notebook explains: + +1. metafspm's root growth, with its sugar as an input; +2. VirtualPlantLab's model seen from metafspm: a `RemoteComponent`, computing nothing, whose state variables are the + other model's outputs, with metafspm's rights on the plant's scales and the links between their names; +3. the session: the coordinator, VirtualPlantLab's model launched, metafspm's scene run; +4. what happened, from the coordinator's record. + +Open `outputs/metafspm/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1) and colour the roots +by `C_hexose_root`. diff --git a/tutorials/03_metafspm_and_virtualplantlab/carbon.jl b/tutorials/03_metafspm_and_virtualplantlab/carbon.jl new file mode 100644 index 0000000..72da792 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/carbon.jl @@ -0,0 +1,69 @@ +# Tutorial 3, VirtualPlantLab's side: a carbon model (a toy) as a participant of an encoding3d coupling session. +# Launched by the coordinator (the notebook), which gives it the session's address. It splits each of metafspm's root +# segments into two layers, grown by a PlantGraphs rule, and gives each layer a sugar (hexose) concentration from its +# segment's structural mass in mg (metafspm sends grams: the coordinator converts): (rank + 1) / (1 + mass), computed +# on the graph by a query. +using encoding3d +using PlantGraphs +using ZMQ # the ZeroMQ transport (a folder works without it) + +struct Tissue <: Node end +struct Bud <: Node end +mutable struct Layer <: Node + segment::Int64 # its segment's id in the session + rank::Int # 0: the outer layer, 1: the inner one + hexose::Float64 +end +const NEW_SEGMENTS = Int64[] # the segments to split at the next rewriting + +# the rule: the bud adds two layers for each new segment, and stays to add more later +split_segments = Rule(Bud, rhs = bud -> begin + isempty(NEW_SEGMENTS) ? Bud() : Bud() + Tuple(Layer(segment, 0, 0.0) + Layer(segment, 1, 0.0) for segment in NEW_SEGMENTS) +end) +tissue = Graph(axiom = Tissue() + Bud(), rules = split_segments) + +# what it does in the session: it follows metafspm's segments, adds its own layers under them, needs their structural +# mass, gives their hexose concentration +declaration = Declaration("carbon"; platform = "VirtualPlantLab", + entity_sets = [EntitySetSpec("plant", "SubOrgan"), EntitySetSpec("plant", "Layer"; rights = "refines")], + variables = [VariableSpec("plant", "Layer", "struct_mass", "requires"; unit = "mg", kind = "extensive"), + VariableSpec("plant", "Layer", "hexose", "provides"; unit = "mol.g-1", kind = "massic_concentration")]) +participant = Participant(declaration; timeout = 600) +offset = participant.id_offset # the ids of the layers it creates start there +split = Set{Int64}() + +layers_of(tissue) = entity_set(from_graph(tissue; id = "plant", entity_set = "Layer", rank = 7, + keep = node -> node isa Layer), "Layer") + +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + # each segment's structural mass (mg), from its layers (the coordinator copies a segment's mass to its layers) + mass = Dict{Int64,Float64}() + if any(set -> set.name == "Layer", plant.entity_sets) + received, mine = entity_set(plant, "Layer"), layers_of(tissue) + segment_of = Dict(id + offset => segment for (id, segment) in zip(mine.columns["id"], mine.columns["segment"])) + for (layer, value) in zip(received.columns["id"], received.columns["struct_mass"]) + isnan(value) || (mass[segment_of[layer]] = value) + end + end + # the new segments split by the rule, then every layer's hexose by a query + empty!(NEW_SEGMENTS) + append!(NEW_SEGMENTS, [segment for segment in entity_set(plant, "SubOrgan").columns["id"] if !(segment in split)]) + union!(split, NEW_SEGMENTS) + rewrite!(tissue) + for layer in apply(tissue, Query(Layer)) + layer.hexose = (layer.rank + 1) / (1 + get(mass, layer.segment, 0.0)) + end + # its layers sent: ids in its range, under their segment + layers = layers_of(tissue) + columns = encoding3d.Columns( + "id" => layers.columns["id"] .+ offset, + "complex" => Vector{Int64}(layers.columns["segment"]), + "parent" => [parent == NO_ID ? NO_ID : parent + offset for parent in layers.columns["parent"]], + "edge_type" => layers.columns["edge_type"], + "label" => layers.columns["label"], + "hexose" => Vector{Float64}(layers.columns["hexose"])) + send_update(participant, Frame(data_structures = [DataStructure("plant"; + entity_sets = [EntitySet("Layer"; columns = columns, rank = 7)])])) +end +println("carbon: the session ended (", participant.stopped, ")") diff --git a/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb b/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb new file mode 100644 index 0000000..335614c --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb @@ -0,0 +1,319 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Tutorial 3: metafspm and VirtualPlantLab, one plant\n", + "\n", + "Two platforms work on the same root system, each in its own language and process, exchanging their values at each\n", + "step through encoding3d:\n", + "\n", + "- **metafspm** (Python) grows the roots: each segment's tip elongates as fast as its sugar allows (a **structural**\n", + " model: it creates the segments);\n", + "- **VirtualPlantLab** (Julia) is the carbon model: it splits each segment into two tissue layers, grown by a\n", + " PlantGraphs rule, and computes their sugar from the segment's mass (a **functional** model, which also adds\n", + " entities of its own, at a finer scale).\n", + "\n", + "Both models are toys, written for the tutorial: they show the coupling, not plant science.\n", + "\n", + "A **coordinator** (encoding3d's, in Python) runs the session: it keeps the shared plant, passes each model the values\n", + "it needs (converting units, copying a segment's mass to its layers, averaging the layers' sugar back to the segment),\n", + "and tells each model when to run. This notebook starts the coordinator, launches VirtualPlantLab's model\n", + "(`carbon.jl`), and runs metafspm's." + ] + }, + { + "cell_type": "code", + "id": "cell-1", + "metadata": {}, + "source": [ + "import os\n", + "import shutil\n", + "import threading\n", + "from dataclasses import dataclass\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import encoding3d\n", + "from openalea.encoding3d.coupling import Coordinator, Link\n", + "from openalea.metafspm.coupling.component import StructuralComponent, input_variable, parameter, state_variable\n", + "from openalea.metafspm.data_structure.configs import PropsConfig, ScalesConfig as scales\n", + "from openalea.metafspm.io.coupling import RemoteComponent\n", + "from openalea.metafspm.scene.scene import Scene, SceneRecorder\n", + "from openalea.metafspm.solve.decorator import actual, segmentation\n", + "\n", + "OUTPUTS = \"outputs\"\n", + "JULIA = os.environ.get(\"JULIA\", \"julia\") # the Julia that runs VirtualPlantLab's model\n", + "STEPS = 8" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## 1. metafspm's root growth\n", + "\n", + "As in tutorial 2a, a component grows one root axis, but its tip now elongates in proportion to its sugar,\n", + "`C_hexose_root`, an **input**: metafspm does not compute it, VirtualPlantLab will. Each segment's structural mass is\n", + "proportional to its length." + ] + }, + { + "cell_type": "code", + "id": "cell-3", + "metadata": {}, + "source": [ + "SEGMENT = 0.01 # m\n", + "DENSITY = 0.1 # g of structure per m of root\n", + "UNIT = dict(unit_comment=\"\", value_comment=\"\", references=\"\", DOI=[], min_value=None, max_value=None)\n", + "\n", + "\n", + "def segment_variable(unit, description, default=0., kind=\"descriptor\"):\n", + " return state_variable(unit=unit, description=description, default=default, scale=scales.SubOrgan, kind=kind, **UNIT)\n", + "\n", + "\n", + "@dataclass\n", + "class SugarLimitedRoot(StructuralComponent):\n", + " \"\"\"A root axis whose tip elongates with its sugar, cut every 1 cm (a toy).\"\"\"\n", + " C_hexose_root: float = input_variable(unit=\"mol.g-1\", description=\"sugar concentration\", default=1.,\n", + " scale=scales.SubOrgan, kind=\"massic_concentration\", **UNIT)\n", + " length: float = segment_variable(\"m\", \"the segment's length\", kind=\"extensive\")\n", + " struct_mass: float = segment_variable(\"g\", \"the segment's structural mass\", kind=\"extensive\")\n", + " radius: float = segment_variable(\"m\", \"the segment's radius\", default=0.0005, kind=\"intensive\")\n", + " is_apex: float = segment_variable(\"adim\", \"1 for the tip\")\n", + " x1: float = segment_variable(\"m\", \"proximal end\")\n", + " y1: float = segment_variable(\"m\", \"proximal end\")\n", + " z1: float = segment_variable(\"m\", \"proximal end\")\n", + " x2: float = segment_variable(\"m\", \"distal end\")\n", + " y2: float = segment_variable(\"m\", \"distal end\")\n", + " z2: float = segment_variable(\"m\", \"distal end\")\n", + " elongation_rate: float = parameter(unit=\"m.s-1.g.mol-1\", description=\"elongation per unit of sugar\",\n", + " default=2e-7, **UNIT)\n", + "\n", + " @classmethod\n", + " def initiate_plant(cls, g, plant, scenario):\n", + " s = g.scales\n", + " parent = plant\n", + " for scale, label in ((s.Axis, g.labels.Axis.Root), (s.GrowthUnit, g.labels.GrowthUnit.Root),\n", + " (s.Phytomer, g.labels.Phytomer.Root), (s.Organ, g.labels.Organ.RootInternode)):\n", + " parent = g.add_component(parent, **PropsConfig(scale=scale, edge_type=\"/\", label=label))\n", + " tip = g.add_component(parent, **PropsConfig(scale=s.SubOrgan, edge_type=\"/\", label=g.labels.SubOrgan.RootSegment))\n", + " x, y, z = (g.property(name)[plant] for name in (\"x\", \"y\", \"z\"))\n", + " for name, value in ((\"length\", 0.005), (\"struct_mass\", DENSITY * 0.005), (\"radius\", 0.0005), (\"is_apex\", 1.),\n", + " (\"x1\", x), (\"y1\", y), (\"z1\", z), (\"x2\", x), (\"y2\", y), (\"z2\", z - 0.005)):\n", + " g.property(name)[tip] = value\n", + "\n", + " def _tips(self):\n", + " return [v for v, flag in self.mtg.property(\"is_apex\").items() if flag]\n", + "\n", + " @actual\n", + " def _elongation(self):\n", + " g = self.mtg\n", + " length, mass, z2, sugar = (g.property(name) for name in (\"length\", \"struct_mass\", \"z2\", \"C_hexose_root\"))\n", + " rate = dict(zip(self.data_structure.entity_ids(\"node\").tolist(),\n", + " self.parameter_values(\"elongation_rate\").tolist()))\n", + " for tip in self._tips():\n", + " growth = rate[tip] * sugar[tip] * self.dt\n", + " length[tip] += growth\n", + " mass[tip] = DENSITY * length[tip]\n", + " z2[tip] -= growth\n", + "\n", + " @segmentation\n", + " def _segmentation(self):\n", + " g = self.mtg\n", + " props = {name: g.property(name) for name in (\"length\", \"struct_mass\", \"is_apex\", \"radius\", \"C_hexose_root\",\n", + " \"x1\", \"y1\", \"z1\", \"x2\", \"y2\", \"z2\")}\n", + " for tip in self._tips():\n", + " if props[\"length\"][tip] <= SEGMENT:\n", + " continue\n", + " rest = props[\"length\"][tip] - SEGMENT\n", + " props[\"length\"][tip], props[\"struct_mass\"][tip], props[\"is_apex\"][tip] = SEGMENT, DENSITY * SEGMENT, 0.\n", + " props[\"z2\"][tip] = props[\"z1\"][tip] - SEGMENT\n", + " x, y, z = props[\"x2\"][tip], props[\"y2\"][tip], props[\"z2\"][tip]\n", + " g.add_child(tip, **PropsConfig(scale=g.scales.SubOrgan, edge_type=\"<\", label=g.labels.SubOrgan.RootSegment,\n", + " length=rest, struct_mass=DENSITY * rest, is_apex=1.,\n", + " radius=props[\"radius\"][tip], C_hexose_root=props[\"C_hexose_root\"][tip],\n", + " x1=x, y1=y, z1=z, x2=x, y2=y, z2=z - rest))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## 2. VirtualPlantLab's model, seen from metafspm\n", + "\n", + "Inside metafspm, the other platform's model is a **RemoteComponent**: a component that computes nothing. Its state\n", + "variables are what the remote model gives (`C_hexose_root`), its inputs what metafspm sends it (`struct_mass`). It\n", + "also says what metafspm does with the plant's scales: metafspm **owns** the plant down to the segments (it creates\n", + "them); it **follows** the layers VirtualPlantLab adds (they are created in metafspm's graph too). And it **links**\n", + "metafspm's names to the other model's: the segment's sugar is the mean of its layers' `hexose`.\n", + "\n", + "Calling it at the end of each step sends metafspm's outputs and brings back the next values." + ] + }, + { + "cell_type": "code", + "id": "cell-5", + "metadata": {}, + "source": [ + "@dataclass\n", + "class VirtualPlantLabCarbon(RemoteComponent):\n", + " \"\"\"VirtualPlantLab's carbon model (carbon.jl), as a metafspm component.\"\"\"\n", + " participant = \"metafspm\"\n", + " rights = {\"Plant\": \"owns\", \"Axis\": \"owns\", \"GrowthUnit\": \"owns\", \"Phytomer\": \"owns\", \"Organ\": \"owns\",\n", + " \"SubOrgan\": \"owns\", \"Layer\": \"follows\"}\n", + " links = [Link(\"metafspm\", \"C_hexose_root\", \"carbon\", \"hexose\", aggregation=\"mean\")]\n", + " C_hexose_root: float = state_variable(unit=\"mol.g-1\", description=\"sugar concentration\", default=1.,\n", + " scale=scales.SubOrgan, kind=\"massic_concentration\", **UNIT)\n", + " struct_mass: float = input_variable(unit=\"g\", description=\"structural mass\", default=0., scale=scales.SubOrgan,\n", + " kind=\"extensive\", **UNIT)\n", + "\n", + "\n", + "class CoupledRoot:\n", + " \"\"\"The plant model: metafspm's growth, then the exchange with VirtualPlantLab.\"\"\"\n", + " initiators = (SugarLimitedRoot,)\n", + "\n", + " def __init__(self, data_structure, time_step):\n", + " self.growth = SugarLimitedRoot(data_structure=data_structure)\n", + " self.carbon = VirtualPlantLabCarbon(data_structure=data_structure)\n", + " self.components = [self.growth, self.carbon]\n", + " self.carbon.start() # joins the session: the initial state exchanged\n", + "\n", + " def run(self):\n", + " self.growth()\n", + " self.carbon()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "## 3. The session\n", + "\n", + "The coordinator gets the session's settings: how many steps, how long each is, the links it applies itself (the\n", + "layers need their segment's structural mass: `copy` it down), and a folder where it **records** the whole shared\n", + "plant at each step, as an encoding3d sequence. Then it launches VirtualPlantLab's model, which joins, and waits for\n", + "metafspm, which joins when its scene is built.\n", + "\n", + "The participants talk over ZeroMQ (in memory). `Coordinator(\"files:outputs/messages\", ...)` would make them\n", + "exchange files in a folder instead, with no other change." + ] + }, + { + "cell_type": "code", + "id": "cell-7", + "metadata": {}, + "source": [ + "for folder in (\"record\", \"metafspm\"):\n", + " shutil.rmtree(os.path.join(OUTPUTS, folder), ignore_errors=True)\n", + "coordinator = Coordinator(\"tcp://127.0.0.1:*\", steps=STEPS, interval=6., time_unit=\"h\", expect=[\"metafspm\"],\n", + " links=[Link(\"carbon\", \"struct_mass\", \"metafspm\", \"struct_mass\", aggregation=\"copy\")],\n", + " record=os.path.join(OUTPUTS, \"record\"), timeout=600)\n", + "coordinator.launch(\"carbon\", [JULIA, \"--project=.\", \"carbon.jl\"])\n", + "session = threading.Thread(target=coordinator.run)\n", + "session.start()\n", + "VirtualPlantLabCarbon.endpoint = coordinator.endpoint\n", + "print(\"coordinator at\", coordinator.endpoint)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "metafspm's scene: one plant, six-hour steps, its own frames recorded too. Each `scene.run()` is one step of the\n", + "session." + ] + }, + { + "cell_type": "code", + "id": "cell-9", + "metadata": {}, + "source": [ + "planting = pd.DataFrame([dict(plant=\"root\", model=CoupledRoot, x=0., y=0., z=0., rotation=0., scenario={})])\n", + "planting.attrs.update(xrange=0.1, yrange=0.1)\n", + "scene = Scene(planting=planting, time_step=6 * 3600.,\n", + " recorder=SceneRecorder(os.path.join(OUTPUTS, \"metafspm\"), summaries=False, frame_period=1))\n", + "for _ in range(STEPS):\n", + " scene.run()\n", + "session.join() # the coordinator stops the session after the last step\n", + "print(\"the session has ended\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## 4. What happened\n", + "\n", + "The coordinator's record holds the shared plant at each step: metafspm's segments with their mass, and\n", + "VirtualPlantLab's layers with their sugar." + ] + }, + { + "cell_type": "code", + "id": "cell-11", + "metadata": {}, + "source": [ + "rows = []\n", + "for frame in encoding3d.read_sequence(os.path.join(OUTPUTS, \"record\", \"scene.fspm.json\")):\n", + " plant = frame.data_structure(\"plant\")\n", + " segments, layers = plant.entity_set(\"SubOrgan\"), plant.entity_set(\"Layer\")\n", + " rows.append(dict(time_h=frame.time, segments=segments.count, layers=layers.count,\n", + " total_mass_mg=1000 * np.nansum(segments.columns[\"struct_mass\"]),\n", + " mean_hexose=np.nanmean(layers.columns[\"hexose\"]) if layers.count else np.nan))\n", + "pd.DataFrame(rows)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "The sugar of a layer falls as its segment gets heavier, so the roots grow more slowly; new segments start with the\n", + "most sugar (their layers are new: their mass reaches VirtualPlantLab one step later, the coupling being explicit).\n", + "metafspm counts the mass in grams and VirtualPlantLab in milligrams: the coordinator converts, as it declares. metafspm's own frames\n", + "(`outputs/metafspm/frames`) hold the roots with their geometry and sugar: open them in ParaView with encoding3d's\n", + "plugin and colour the roots by `C_hexose_root`.\n", + "\n", + "## Try it yourself\n", + "\n", + "- Exchange through a folder: `Coordinator(\"files:outputs/messages\", ...)`. The messages are deleted once read; add\n", + " `keep_messages=True` to keep them and look at them.\n", + "- Change VirtualPlantLab's sugar rule in `carbon.jl`, and see metafspm's roots grow faster or slower.\n", + "- Run two plants in the scene: VirtualPlantLab's model splits the segments of both." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/03_metafspm_and_virtualplantlab/run.sh b/tutorials/03_metafspm_and_virtualplantlab/run.sh new file mode 100755 index 0000000..f248391 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Runs tutorial 3 (README.md): metafspm's environment, and Julia 1.12 (JULIA, default julia) for VirtualPlantLab's side. +set -euo pipefail +cd "$(dirname "$0")" +"${JULIA:-julia}" --project=. -e 'using Pkg; Pkg.instantiate(); using encoding3d, PlantGraphs, ZMQ' +python -m pytest --nbmake coupling.ipynb diff --git a/tutorials/04_four_platforms/README.md b/tutorials/04_four_platforms/README.md new file mode 100644 index 0000000..6987279 --- /dev/null +++ b/tutorials/04_four_platforms/README.md @@ -0,0 +1,58 @@ +# Tutorial 4: four platforms, one plant + +**What you learn:** how four platforms, in four languages, run one plant together through encoding3d, each with the +model it is good at, two building the plant's structure and two computing its functions. + +| Platform | Its model (a toy) | Kind | In the shared plant | It gives | It needs | +|---|---|---|---|---|---| +| **CPlantBox** (C++, `cplantbox/roots.cpp`) | the root system, grown by CPlantBox's own rules (Anagallis' parameters) | structural | **owns** the plant, its root organs and their segments | each segment's length, radius and ends (cm) | — | +| **GroIMP** (Java, `groimp/.../Shoot.java`) | the shoot, a GroIMP graph of phytomers: a new one each time the roots have taken up 8 cm³ of water | structural | **owns** the phytomers, under CPlantBox's plant | each leaf's area (m²) and height (m) | the plant's water uptake (cm³/d) | +| **metafspm** (Python, `metafspm/uptake.py`) | root water uptake: through each segment's surface, more when the plant has more carbon | functional | **follows** the plant, organs and segments | each segment's uptake (cm³/d) | each segment's length and radius; the plant's assimilation | +| **VirtualPlantLab** (Julia, `virtualplantlab/light.jl`) | light (Beer–Lambert down the canopy) and photosynthesis (a light-use efficiency), on a PlantGraphs graph | functional | **follows** the plant and the phytomers | each leaf's absorbed light and assimilation; the plant's total (µmol/s) | each leaf's area and height | + +The loop: the leaves' light → their assimilation → the roots' uptake → the water the shoot has → new leaves → more +light. The models are toys, labelled as such: the tutorial shows how platforms work together, not plant science. + +## How they fit together + +- **One plant, shared.** The session's data structure, `plant`, holds every platform's entities: CPlantBox's plant, + root organs and segments, GroIMP's phytomers under the same plant. A platform declares the entity sets it works + on, and what it may do with them: **own** them (create them), or **follow** them (read them). No platform needs to + know the others' code or language. +- **Links, made by encoding3d's coordinator** (`session.py`): + - metafspm's `length` and `radius` are found by their **standard name** among CPlantBox's outputs (the same unit + here, cm; a different one would be converted); + - VirtualPlantLab's `area` and `height` by their **name** among GroIMP's; + - GroIMP's `water_uptake` of the plant is the **sum** of metafspm's uptake over the plant's segments (they are in + the plant, through their organs); + - metafspm's `plant_assimilation` of each segment is a **copy** of the plant's, given by VirtualPlantLab. +- **Stages.** At each daily step, CPlantBox grows the roots first, then GroIMP the shoot (it needs the plant + CPlantBox created), then metafspm and VirtualPlantLab compute together. +- **What each platform does with what it receives:** CPlantBox's adapter writes its plant; GroIMP's writes its graph; + metafspm builds its own graph from CPlantBox's roots (`RemoteComponent.join`) and computes with a metafspm + component; VirtualPlantLab rebuilds the phytomers as a PlantGraphs graph (`to_graph`) and queries it. + +## What you need + +Not encoding3d's requirements: everything tutorials 2a to 2d need. CPlantBox built (tutorial 2c, `CPLANTBOX` and +`CPLANTBOX_LIBRARY`), Java 21 and Maven (tutorial 2d), Julia 1.12 or later (`JULIA` if not on the path), and +metafspm's environment active (tutorial 2a), whose Python runs the coordinator and metafspm's side. + +## How to run it + +```bash +CPLANTBOX=/path/to/CPlantBox CPLANTBOX_LIBRARY=/path/to/CPlantBox-build/src/libCPlantBox.a ./run.sh +``` + +It builds CPlantBox's and GroIMP's sides, prepares the Julia project, then runs `session.py`, which launches the four +models, each in its own process, for ten days. Each prints what it does at each step, and the coordinator prints how +it linked them. The whole plant is recorded at each step in `outputs/record/`, with a `views.json` colouring the +roots by uptake: open `outputs/record/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1) and press +*Play*. + +## Try it yourself + +- Make phytomers cheaper (`WATER_PER_PHYTOMER` in `Shoot.java`) and watch the canopy close: assimilation saturates. +- Give metafspm's uptake a soil water potential, from a fifth participant owning a soil grid, linked to the roots + by a mapping (the coupling documentation, "Coupling platforms"). +- Replace one platform by the record of an earlier run: `encoding3d couple --replay` (the coupling documentation). diff --git a/tutorials/04_four_platforms/cplantbox/CMakeLists.txt b/tutorials/04_four_platforms/cplantbox/CMakeLists.txt new file mode 100644 index 0000000..f485b5a --- /dev/null +++ b/tutorials/04_four_platforms/cplantbox/CMakeLists.txt @@ -0,0 +1,19 @@ +# Tutorial 4, CPlantBox's side: the root architecture as a coupling session's participant (../README.md). +# +# cmake -S . -B build -DCPLANTBOX_SOURCE_DIR=/path/to/CPlantBox -DCPLANTBOX_LIBRARY=/path/to/libCPlantBox.a +cmake_minimum_required(VERSION 3.16) +project(encoding3d_tutorial_four_platforms_cplantbox LANGUAGES CXX) + +set(CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder") +set(CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox.a)") +if(NOT CPLANTBOX_SOURCE_DIR OR NOT CPLANTBOX_LIBRARY) + message(FATAL_ERROR "give CPlantBox: -DCPLANTBOX_SOURCE_DIR=... -DCPLANTBOX_LIBRARY=... (../README.md)") +endif() +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "${CPLANTBOX_SOURCE_DIR}" CACHE PATH "" FORCE) +set(ENCODING3D_CPLANTBOX_LIBRARY "${CPLANTBOX_LIBRARY}" CACHE FILEPATH "" FORCE) +set(ENCODING3D_BUILD_TESTS OFF CACHE BOOL "" FORCE) +add_subdirectory(../../../bindings/cpp encoding3d) + +add_executable(roots roots.cpp) +target_link_libraries(roots PRIVATE encoding3d::cplantbox) +target_compile_definitions(roots PRIVATE CPLANTBOX_PARAMETERS="${CPLANTBOX_SOURCE_DIR}/modelparameter/structural") diff --git a/tutorials/04_four_platforms/cplantbox/roots.cpp b/tutorials/04_four_platforms/cplantbox/roots.cpp new file mode 100644 index 0000000..959a502 --- /dev/null +++ b/tutorials/04_four_platforms/cplantbox/roots.cpp @@ -0,0 +1,90 @@ +// Tutorial 4, CPlantBox's side: the root architecture (structural). CPlantBox grows a root system by its own rules; +// each turn, the plant, its root organs and their segments are sent to the session (CPlantBox's adapter writes them, +// their ids renumbered in this participant's range), with each segment's length, radius and ends. +#include +#include +#include + +#include "encoding3d/coupling.hpp" +#include "encoding3d/cplantbox.hpp" +#include "encoding3d/encoding3d.hpp" + +#include "Plant.h" + +using namespace encoding3d; + +namespace { + +/// The declaration: CPlantBox owns the plant, its organs and segments, and provides the segments' sizes and ends. +json declaration() { + json variables = json::array(); + auto provides = [&](const char *name, const char *unit, const char *standard_name, const char *kind) { + variables.push_back({{"dataStructure", "plant"}, {"entitySet", "SubOrgan"}, {"name", name}, + {"direction", "provides"}, {"unit", unit}, {"standardName", standard_name}, + {"kind", kind}}); + }; + provides("length", "cm", "length", "extensive"); + provides("radius", "cm", "radius", "intensive"); + for (const auto &[name, standard_name] : {std::pair{"x1", "proximal_end_x"}, {"y1", "proximal_end_y"}, + {"z1", "proximal_end_z"}, {"x2", "distal_end_x"}, + {"y2", "distal_end_y"}, {"z2", "distal_end_z"}}) + provides(name, "cm", standard_name, "descriptor"); + json sets = json::array(); + for (const auto &[name, rank] : {std::pair{"Plant", 1}, {"Organ", 5}, {"SubOrgan", 6}}) + sets.push_back({{"dataStructure", "plant"}, {"name", name}, {"rank", rank}, {"rights", "owns"}}); + return {{"participant", "cplantbox"}, {"platform", "CPlantBox"}, {"entitySets", sets}, {"variables", variables}}; +} + +/// The session's id of an adapter's id: this participant's range, in order of appearance (stable while it grows). +std::int64_t session_id(std::map &ids, std::int64_t offset, std::int64_t adapter) { + if (adapter == NO_ID) return NO_ID; + auto found = ids.find(adapter); + if (found == ids.end()) found = ids.emplace(adapter, offset + std::int64_t(ids.size())).first; + return found->second; +} + +/// An entity set of the adapter's as the session's: its ids renumbered, its topology, the variables in *keep*. +EntitySet renumbered(const EntitySet &set, std::map &ids, std::int64_t offset, + const std::vector &keep) { + EntitySet out(set.name, set.rank); + for (const char *name : {"id", "complex", "parent"}) { + if (!set.has_column(name)) continue; + std::vector values; + for (const auto value : set.column(name).as()) values.push_back(session_id(ids, offset, value)); + out.set(name, Column(values)); + } + for (const char *name : {"label", "edge_type"}) + if (set.has_column(name)) out.set(name, set.column(name)); + for (const auto &name : keep) + if (set.has_column(name)) out.set(name, set.column(name)); + return out; +} + +} // namespace + +int main() { + auto plant = std::make_shared(1); + plant->readParameters(std::string(CPLANTBOX_PARAMETERS) + "/plant/Anagallis_femina_Leitner_2010.xml", "plant", + true, false); + plant->initialize(false); + plant->simulate(3., false); // a young root system to start from + + coupling::Participant participant(declaration(), "", json::array(), 600.0); + std::map ids; + while (auto turn = participant.next_turn()) { + if (turn->phase == "step") plant->simulate(turn->until - turn->time, false); // days, the session's unit + const auto written = cplantbox::to_data_structure(*plant); + DataStructure update("plant"); + update.entity_sets.push_back(renumbered(written.entity_set("Plant"), ids, participant.id_offset, {})); + update.entity_sets.push_back(renumbered(written.entity_set("Organ"), ids, participant.id_offset, {})); + update.entity_sets.push_back(renumbered(written.entity_set("SubOrgan"), ids, participant.id_offset, + {"length", "radius", "x1", "y1", "z1", "x2", "y2", "z2"})); + Frame frame; + frame.data_structures.push_back(update); + participant.send_update(frame); + std::cout << "cplantbox: day " << plant->getSimTime() << ", " << plant->getNumberOfSegments() << " segments" + << std::endl; + } + std::cout << "cplantbox: the session ended (" << participant.stopped.value_or("") << ")" << std::endl; + return 0; +} diff --git a/tutorials/04_four_platforms/groimp/pom.xml b/tutorials/04_four_platforms/groimp/pom.xml new file mode 100644 index 0000000..6632b7c --- /dev/null +++ b/tutorials/04_four_platforms/groimp/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + org.openalea.tutorials + encoding3d-tutorial-shoot + 0.1.0 + + + 21 + UTF-8 + + + + + groimp + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d-groimp + 0.1.0 + + + de.grogra + graph + 3.0.0 + + + org.zeromq + jeromq + 0.6.0 + + + diff --git a/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java new file mode 100644 index 0000000..259fca7 --- /dev/null +++ b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java @@ -0,0 +1,77 @@ +package tutorial; + +import java.lang.reflect.Field; + +import de.grogra.graph.impl.Node; +import de.grogra.graph.impl.Node.NType; +import de.grogra.reflect.Type; + +/** + * GroIMP's "managed fields": the fields of a node class GroIMP knows by name, as its XL compiler declares them for the + * classes of an RGG model. GroIMP's adapter writes them as columns. Here they are declared from plain Java fields by + * reflection, so that the tutorial's node classes stay short. + */ +final class Managed { + private Managed() {} + + /** The NType of {@code type}, with its fields {@code names} managed. */ + static NType type(Class type, String... names) { + NType nodeType = new NType(type); + for (String name : names) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + nodeType.addManagedField(new ManagedField(nodeType, field)); + } catch (NoSuchFieldException error) { + throw new IllegalArgumentException(type.getSimpleName() + " has no field " + name, error); + } + } + nodeType.validate(); + return nodeType; + } + + private static final class ManagedField extends NType.Field { + private final Field field; + + ManagedField(NType type, Field field) { + super(type, field.getName(), PUBLIC | SCO, typeOf(field.getType()), null); + this.field = field; + } + + private static Type typeOf(Class type) { + if (type == double.class) return Type.DOUBLE; + if (type == float.class) return Type.FLOAT; + if (type == int.class) return Type.INT; + if (type == boolean.class) return Type.BOOLEAN; + if (type == String.class) return Type.STRING; + throw new IllegalArgumentException("a managed field of type " + type); + } + + private Object read(Object node) { + try { + return field.get(node); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + private void write(Object node, Object value) { + try { + field.set(node, value); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + @Override public double getDouble(Object node) { return (double) read(node); } + @Override public void setDouble(Object node, double value) { write(node, value); } + @Override public float getFloat(Object node) { return (float) read(node); } + @Override public void setFloat(Object node, float value) { write(node, value); } + @Override public int getInt(Object node) { return (int) read(node); } + @Override public void setInt(Object node, int value) { write(node, value); } + @Override public boolean getBoolean(Object node) { return (boolean) read(node); } + @Override public void setBoolean(Object node, boolean value) { write(node, value); } + @Override public Object getObject(Object node) { return read(node); } + @Override protected void setObjectImpl(Object node, Object value) { write(node, value); } + } +} diff --git a/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java new file mode 100644 index 0000000..4d9a1e2 --- /dev/null +++ b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java @@ -0,0 +1,128 @@ +package tutorial; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openalea.encoding3d.Column; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Variable; +import org.openalea.encoding3d.coupling.Participant; +import org.openalea.encoding3d.coupling.Turn; +import org.openalea.encoding3d.groimp.GroImp; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** + * Tutorial 4, GroIMP's side: the shoot architecture (structural). The shoot is a GroIMP graph of phytomers (an + * internode and its leaf), one on top of the other; a new one grows each time the plant's roots have taken up enough + * water since the last, and every leaf expands. Each turn, GroIMP's adapter writes the graph, and its phytomers are + * sent to the session under the plant CPlantBox created. + */ +public final class Shoot { + private Shoot() {} + + static final double WATER_PER_PHYTOMER = 8.; // cm3 of water taken up for a new phytomer (a toy) + + /** A phytomer, as an RGG model declares it: its leaf's area and its height, managed by GroIMP. */ + public static final class Phytomer extends Node { + double area = 1e-4; // m2 + double height; // m + public static final NType $TYPE = Managed.type(Phytomer.class, "area", "height"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Phytomer(); } + } + + static ObjectNode declaration(ObjectMapper json) { + ObjectNode declaration = json.createObjectNode().put("participant", "groimp").put("platform", "GroIMP"); + ArrayNode sets = declaration.putArray("entitySets"); + sets.addObject().put("dataStructure", "plant").put("name", "Plant").put("rank", 1).put("rights", "follows"); + sets.addObject().put("dataStructure", "plant").put("name", "Phytomer").put("rank", 4).put("rights", "owns"); + ArrayNode variables = declaration.putArray("variables"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Phytomer").put("name", "area") + .put("direction", "provides").put("unit", "m2").put("kind", "extensive"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Phytomer").put("name", "height") + .put("direction", "provides").put("unit", "m").put("kind", "descriptor"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Plant").put("name", "water_uptake") + .put("direction", "requires").put("unit", "cm3/d").put("kind", "extensive"); + return declaration; + } + + static GroImp.Options options() { + GroImp.Options options = new GroImp.Options(); + options.id = "plant"; + options.scaleNames.put(2, "Phytomer"); + options.scaleRanks.put(2, 4L); + options.variables.put("area", new Variable("m2", null, "extensive", "state")); + options.variables.put("height", new Variable("m", null, "descriptor", "state")); + return options; + } + + public static void main(String[] arguments) throws Exception { + ObjectMapper json = new ObjectMapper(); + GraphManager graph = new GraphManager(new ServerConnection(null), "shoot", false, false); + Node root = new Node(); + Node plant = new Node(); // the plant node GroIMP's graph hangs the phytomers on + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Phytomer top = new Phytomer(); + plant.addEdgeBitsTo(top, Graph.REFINEMENT_EDGE, null); + graph.setRoot(Graph.MAIN_GRAPH, root); + List phytomers = new ArrayList<>(List.of(top)); + Map ids = new HashMap<>(); // GroIMP node id: session id + double water = 0; + + try (Participant participant = new Participant(declaration(json), null, null, 600.0)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + EntitySet plants = turn.frame().dataStructure("plant").entitySet("Plant"); + long plantId = plants.column("id").longs()[0]; // CPlantBox's plant + if (turn.phase().equals("step")) { + double uptake = plants.column("water_uptake").doubles()[0]; + water += Double.isNaN(uptake) ? 0 : uptake * (turn.until() - turn.time()); + for (Phytomer phytomer : phytomers) phytomer.area *= 1.15; // the leaves expand + while (water >= WATER_PER_PHYTOMER) { // a new phytomer on top + water -= WATER_PER_PHYTOMER; + Phytomer next = new Phytomer(); + next.height = top.height + 0.03; + top.addEdgeBitsTo(next, Graph.SUCCESSOR_EDGE, null); + phytomers.add(next); + top = next; + } + } + // the graph written by GroIMP's adapter, its phytomers sent under the session's plant + DataStructure written = GroImp.toDataStructure(root, options()); + EntitySet set = written.entitySet("Phytomer"); + long[] nodes = set.column("id").longs(); + long[] session = new long[nodes.length], complexes = new long[nodes.length]; + long[] parents = set.column("parent").longs().clone(); + for (int row = 0; row < nodes.length; row++) { + final long node = nodes[row]; + session[row] = ids.computeIfAbsent(node, key -> participant.idOffset + ids.size()); + complexes[row] = plantId; + } + for (int row = 0; row < parents.length; row++) if (parents[row] >= 0) parents[row] = ids.get(parents[row]); + EntitySet out = new EntitySet("Phytomer", 4L); + out.set("id", Column.of(session)).set("complex", Column.of(complexes)).set("parent", Column.of(parents)) + .set("edge_type", set.column("edge_type")).set("label", set.column("label")) + .set("area", set.column("area")).set("height", set.column("height")); + DataStructure update = new DataStructure("plant"); + update.entitySets.add(out); + Frame frame = new Frame(); + frame.dataStructures.add(update); + participant.sendUpdate(frame); + System.out.println("groimp: " + phytomers.size() + " phytomers, water left " + Math.round(water * 1000) / 1000.0 + + " cm3"); + } + System.out.println("groimp: the session ended (" + participant.stopped + ")"); + } + } +} diff --git a/tutorials/04_four_platforms/metafspm/uptake.py b/tutorials/04_four_platforms/metafspm/uptake.py new file mode 100644 index 0000000..337c07a --- /dev/null +++ b/tutorials/04_four_platforms/metafspm/uptake.py @@ -0,0 +1,62 @@ +""" +Tutorial 4, metafspm's side: root water uptake (functional, a toy). metafspm has no plant of its own here: it joins +the session and builds its data structure from the roots CPlantBox grows (RemoteComponent.join), then its component +computes each segment's uptake from its surface and the plant's carbon status (the more the leaves assimilate, the more +the roots take up), at each step. +""" +from dataclasses import dataclass + +import numpy as np + +from openalea.metafspm.coupling.component import FunctionalComponent, input_variable, state_variable +from openalea.metafspm.data_structure.configs import ScalesConfig as scales +from openalea.metafspm.io.coupling import RemoteComponent +from openalea.metafspm.solve.decorator import actual + +UNIT = dict(unit_comment="", value_comment="", references="", DOI=[], min_value=None, max_value=None) +CONDUCTANCE = 0.5 # cm3 of water per cm2 of root surface per day, at full carbon status (a toy) +HALF_ASSIMILATION = 5. # umol/s of assimilation for half the full uptake + + +def segment(variable, unit, description, kind): + return variable(unit=unit, description=description, default=0., scale=scales.SubOrgan, kind=kind, **UNIT) + + +@dataclass +class OtherPlatforms(RemoteComponent): + """The other platforms' models, as seen from metafspm: CPlantBox's roots (followed: their segments' sizes) and + VirtualPlantLab's photosynthesis (the plant's assimilation, copied to each segment); metafspm's uptake sent.""" + participant = "metafspm" + rights = {"Plant": "follows", "Organ": "follows", "SubOrgan": "follows"} + length: float = segment(state_variable, "cm", "the segment's length (CPlantBox's)", "extensive") + radius: float = segment(state_variable, "cm", "the segment's radius (CPlantBox's)", "intensive") + plant_assimilation: float = segment(state_variable, "umol/s", "the plant's assimilation (VirtualPlantLab's)", + "extensive") + water_uptake: float = segment(input_variable, "cm3/d", "the segment's water uptake (metafspm's)", "extensive") + + +@dataclass +class RootWaterUptake(FunctionalComponent): + """Each segment takes up water through its surface, more when the plant has more carbon.""" + length: float = segment(input_variable, "cm", "the segment's length", "extensive") + radius: float = segment(input_variable, "cm", "the segment's radius", "intensive") + plant_assimilation: float = segment(input_variable, "umol/s", "the plant's assimilation", "extensive") + water_uptake: float = segment(state_variable, "cm3/d", "the segment's water uptake", "extensive") + + @actual + def _water_uptake(self, length, radius, plant_assimilation): + carbon_status = plant_assimilation / (plant_assimilation + HALF_ASSIMILATION) + return CONDUCTANCE * 2 * np.pi * radius * length * np.nan_to_num(carbon_status) + + +if __name__ == "__main__": + data_structure = OtherPlatforms.join() # the session's address from the coordinator, which launched it + uptake = RootWaterUptake(data_structure=data_structure) + others = OtherPlatforms(data_structure=data_structure) + others.start() + while others.stopped is None: + uptake() + print(f"metafspm: {data_structure.n_nodes()} segments, uptake {np.sum(data_structure.get('water_uptake')):.4f} " + "cm3/d", flush=True) + others() + print(f"metafspm: the session ended ({others.stopped})") diff --git a/tutorials/04_four_platforms/run.sh b/tutorials/04_four_platforms/run.sh new file mode 100755 index 0000000..8aba9ae --- /dev/null +++ b/tutorials/04_four_platforms/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Runs tutorial 4 (README.md): builds the CPlantBox and GroIMP participants, prepares the Julia project, then runs the +# session. Needs: CPLANTBOX and CPLANTBOX_LIBRARY (as tutorial 2c), Java 21 and Maven, Julia 1.12 (JULIA, default +# julia), and metafspm's environment active (its python runs the coordinator and metafspm's side). +set -euo pipefail +cd "$(dirname "$0")" +: "${CPLANTBOX:?set CPLANTBOX to the CPlantBox source folder}" "${CPLANTBOX_LIBRARY:?set CPLANTBOX_LIBRARY to libCPlantBox.a}" +cmake -S cplantbox -B cplantbox/build -DCMAKE_BUILD_TYPE=Release -DCPLANTBOX_SOURCE_DIR="$CPLANTBOX" \ + -DCPLANTBOX_LIBRARY="$CPLANTBOX_LIBRARY" > /dev/null +cmake --build cplantbox/build -j 4 > /dev/null +mvn -q -f ../../bindings/java/pom.xml install -DskipTests +mvn -q -f ../../bindings/java/groimp/pom.xml install -DskipTests +mvn -q -f groimp/pom.xml package dependency:copy-dependencies -DincludeScope=runtime +"${JULIA:-julia}" --project=virtualplantlab -e 'using Pkg; Pkg.instantiate(); using encoding3d, PlantGraphs, ZMQ' +python session.py "$PWD/cplantbox/build/roots" "$PWD/groimp/target/classes:$PWD/groimp/target/dependency/*" \ + "${JULIA:-julia}" "$(command -v python)" +if command -v encoding3d > /dev/null; then encoding3d validate outputs/record/scene.fspm.json | head -1; fi diff --git a/tutorials/04_four_platforms/session.py b/tutorials/04_four_platforms/session.py new file mode 100644 index 0000000..31475cb --- /dev/null +++ b/tutorials/04_four_platforms/session.py @@ -0,0 +1,44 @@ +""" +Tutorial 4: the session. encoding3d's coordinator launches the four platforms' models, each in its own process, and +runs ten daily steps: CPlantBox grows the roots, then GroIMP the shoot, then metafspm (root water uptake) and +VirtualPlantLab (light and photosynthesis) compute together. The shared plant is recorded at each step in +outputs/record (an encoding3d sequence). + + python session.py CPLANTBOX JAVA_CLASSPATH JULIA METAFSPM_PYTHON +""" +import os +import sys + +from openalea.encoding3d import read_index, read +from openalea.encoding3d.coupling import Coordinator, Link +from openalea.encoding3d.views import VIEWS_FILE, check_views, write_views + +here = os.path.dirname(os.path.abspath(__file__)) +cplantbox, classpath, julia, python = sys.argv[1:5] + +coordinator = Coordinator( + "tcp://127.0.0.1:*", steps=10, interval=1., time_unit="d", timeout=600, + stages=[["cplantbox"], ["groimp"], ["metafspm", "virtualplantlab"]], + links=[Link("metafspm", "plant_assimilation", "virtualplantlab", "plant_assimilation", aggregation="copy"), + Link("groimp", "water_uptake", "metafspm", "water_uptake", aggregation="sum")], + record=os.path.join(here, "outputs", "record")) +coordinator.launch("cplantbox", [cplantbox]) +coordinator.launch("groimp", ["java", "-cp", classpath, "tutorial.Shoot"]) +coordinator.launch("virtualplantlab", [julia, f"--project={os.path.join(here, 'virtualplantlab')}", + os.path.join(here, "virtualplantlab", "light.jl")]) +coordinator.launch("metafspm", [python, os.path.join(here, "metafspm", "uptake.py")]) +store = coordinator.run() +for coupling in coordinator.couplings: + source = f"{coupling.provider}.{coupling.source.name}" if coupling.source else "-" + print(f" {coupling.receiver}.{coupling.variable.name} <- {source} ({coupling.how}" + f"{', ' + coupling.aggregation if coupling.aggregation else ''}, factor {coupling.factor:g})") +# how to look at the record: the roots (CPlantBox's segments) as tubes, coloured by metafspm's uptake +record = os.path.join(here, "outputs", "record") +views = {"encoding3d_views": 1, "active": "uptake", "configurations": {"uptake": {"layers": [ + {"name": "roots", "source": {"data_structure": "plant", "kind": "segments", "entity_set": "SubOrgan"}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_uptake"}, "colormap": "Viridis (matplotlib)", + "tube": {"radius": "radius", "scale": 1}}]}}} +write_views(os.path.join(record, VIEWS_FILE), views) +last = read(read_index(os.path.join(record, "scene.fspm.json"))["frames"][-1]["path"]) +print(check_views(views, last)) +print("recorded in", record) diff --git a/tutorials/04_four_platforms/virtualplantlab/Project.toml b/tutorials/04_four_platforms/virtualplantlab/Project.toml new file mode 100644 index 0000000..8e1400e --- /dev/null +++ b/tutorials/04_four_platforms/virtualplantlab/Project.toml @@ -0,0 +1,12 @@ +[deps] +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[sources] +encoding3d = {path = "../../../bindings/julia/encoding3d"} + +[compat] +PlantGraphs = "1" +ZMQ = "1" +julia = "1.12" diff --git a/tutorials/04_four_platforms/virtualplantlab/light.jl b/tutorials/04_four_platforms/virtualplantlab/light.jl new file mode 100644 index 0000000..171adb5 --- /dev/null +++ b/tutorials/04_four_platforms/virtualplantlab/light.jl @@ -0,0 +1,60 @@ +# Tutorial 4, VirtualPlantLab's side: light and photosynthesis (functional, a toy). Each turn, the shoot's phytomers +# (GroIMP's, sent by the session) become a PlantGraphs graph (to_graph); a query then gives each leaf the light left +# under the leaves above it (Beer-Lambert) and its photosynthesis (a light-use efficiency). The plant's total +# assimilation goes to the plant, for metafspm. +using encoding3d +using PlantGraphs +using ZMQ + +const INCIDENT = 1000.0 # umol/s of light (PAR) falling on the plant's ground area +const GROUND = 0.01 # m2: the plant's ground area +const EXTINCTION = 0.6 # the canopy's light extinction coefficient +const EFFICIENCY = 0.05 # mol CO2 fixed per mol of light absorbed + +mutable struct Leaf <: Node # a phytomer's leaf, as the session sends it (to_graph fills its fields) + area::Float64 # m2 + height::Float64 # m +end + +declaration = Declaration("virtualplantlab"; platform = "VirtualPlantLab", + entity_sets = [EntitySetSpec("plant", "Plant"; rank = 1), EntitySetSpec("plant", "Phytomer"; rank = 4)], + variables = [VariableSpec("plant", "Phytomer", "area", "requires"; unit = "m2", kind = "extensive"), + VariableSpec("plant", "Phytomer", "height", "requires"; unit = "m", kind = "descriptor"), + VariableSpec("plant", "Phytomer", "absorbed_light", "provides"; unit = "umol/s", kind = "extensive"), + VariableSpec("plant", "Phytomer", "assimilation", "provides"; unit = "umol/s", kind = "extensive"), + VariableSpec("plant", "Plant", "plant_assimilation", "provides"; unit = "umol/s", kind = "extensive")]) +participant = Participant(declaration; timeout = 600) + +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + if !any(set -> set.name == "Phytomer", plant.entity_sets) || row_count(entity_set(plant, "Phytomer")) == 0 + send_update(participant, Frame()) + continue + end + shoot = entity_set(plant, "Phytomer") + # the phytomers as a VirtualPlantLab graph: each row a Leaf node, its fields from the columns + static, graph_ids = to_graph(plant; entity_set = "Phytomer", types = Dict("Phytomer" => Leaf)) + canopy = Graph(axiom = static) + # Beer-Lambert down the canopy, by a query: a leaf receives the light the leaves above it let through + leaves = sort(apply(canopy, Query(Leaf)); by = leaf -> -leaf.height) + absorbed, assimilation = IdDict{Leaf,Float64}(), IdDict{Leaf,Float64}() + area_above = 0.0 + for leaf in leaves + light = INCIDENT * exp(-EXTINCTION * area_above / GROUND) + absorbed[leaf] = light * (1 - exp(-EXTINCTION * leaf.area / GROUND)) + assimilation[leaf] = EFFICIENCY * absorbed[leaf] + area_above += leaf.area + end + # sent: each phytomer's light and assimilation (its row's leaf, by its id), the plant's total + leaf_of = [static_graph(canopy)[graph_ids[id]].data for id in shoot.columns["id"]] # the graph holds a copy + phytomers = EntitySet("Phytomer"; rank = 4, columns = encoding3d.Columns( + "id" => shoot.columns["id"], + "absorbed_light" => [absorbed[leaf] for leaf in leaf_of], + "assimilation" => [assimilation[leaf] for leaf in leaf_of])) + total = sum(values(assimilation)) + plants = EntitySet("Plant"; rank = 1, columns = encoding3d.Columns( + "id" => entity_set(plant, "Plant").columns["id"], "plant_assimilation" => [total])) + send_update(participant, Frame(data_structures = [DataStructure("plant"; entity_sets = [plants, phytomers])])) + println("virtualplantlab: ", length(leaves), " leaves, assimilation ", round(total; digits = 2), " umol/s") +end +println("virtualplantlab: the session ended (", participant.stopped, ")") diff --git a/tutorials/README.md b/tutorials/README.md new file mode 100644 index 0000000..178cfa2 --- /dev/null +++ b/tutorials/README.md @@ -0,0 +1,18 @@ +# Tutorials + +Step by step, from encoding3d on its own to four platforms running one plant together. Each tutorial is a folder with +its explanation (`README.md`), its code, and `run.sh`, which runs it from start to end. + +| Tutorial | What it shows | What it needs | +|---|---|---| +| `01_encoding3d_alone` | writing, checking, reading and viewing a growing root system's files, with no platform | Python, `openalea.encoding3d` | +| `02a_metafspm` | a metafspm scene recorded as frames, read by encoding3d alone, logged smaller, rebuilt in metafspm | metafspm's environment | +| `02b_virtualplantlab` | a VirtualPlantLab plant grown by rules, written with its geometry, rebuilt and grown on; an MTG to and from encoding3d | Julia 1.12, VirtualPlantLab's packages, MTG.jl | +| `02c_cplantbox` | a CPlantBox plant written day by day with its geometry, soil grid and mapping; frames back as CPlantBox root systems | C++17, CMake, CPlantBox built | +| `02d_groimp` | a GroIMP graph written step by step and rebuilt as GroIMP nodes; GroIMP's plugin exporting a project with its geometry | Java 21, Maven (GroIMP's Graph module) | +| `03_metafspm_and_virtualplantlab` | a coupling session: metafspm's root growth and VirtualPlantLab's carbon model on one plant, over ZeroMQ, recorded | metafspm's environment, Julia 1.12 | +| `04_four_platforms` | CPlantBox's roots and GroIMP's shoot (structural), metafspm's water uptake and VirtualPlantLab's light and photosynthesis (functional), one plant, one session | everything tutorials 2a to 2d need | + +**encoding3d depends on no platform.** Tutorial 1 needs encoding3d only, and encoding3d's CI runs it. The tutorials +using a platform (metafspm, VirtualPlantLab, CPlantBox, GroIMP) list what they need in their own folder, outside +encoding3d's requirements, and are run by hand.