From ed52992cd916032fcd5822b6c7d0ca7ad9f4de9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 11:30:58 +0100 Subject: [PATCH 01/39] Documentation to the OpenAlea guidelines: doc/ (renamed from docs/), conf.py on the guidelines' template reading the package metadata, the full extension list, a notebook (doc/notebooks, run by nbmake in CI and by nbsphinx), the doc extra, conda/environment.yml, CONTRIBUTING.md, CODE_OF_CONDUCT.md and CHANGELOG.md --- .github/workflows/conformance.yml | 8 +- .gitignore | 4 +- .readthedocs.yml | 2 +- CHANGELOG.md | 11 + CODE_OF_CONDUCT.md | 8 + CONTRIBUTING.md | 29 ++ README.md | 8 +- conda/environment.yml | 13 +- {docs => doc}/_static/images/archisimple.png | Bin .../_static/images/granap_section.png | Bin .../_static/images/mecha_section.png | Bin {docs => doc}/_static/images/root_system.png | Bin {docs => doc}/_static/images/wheat_shoot.png | Bin {docs => doc}/_static/openalea_web.svg | 0 {docs => doc}/api.md | 0 {docs => doc}/conf.py | 75 +++-- {docs => doc}/design_note.md | 2 +- {docs => doc}/governance.md | 0 {docs => doc}/implementations.md | 0 {docs => doc}/index.md | 1 + doc/notebooks/fspm_tf_in_python.ipynb | 293 ++++++++++++++++++ {docs => doc}/overview.md | 0 {docs => doc}/platforms.md | 0 {docs => doc}/python.md | 0 {docs => doc}/specification.md | 0 {docs => doc}/using.md | 0 {docs => doc}/vocabulary.md | 0 pyproject.toml | 6 +- 28 files changed, 417 insertions(+), 43 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md rename {docs => doc}/_static/images/archisimple.png (100%) rename {docs => doc}/_static/images/granap_section.png (100%) rename {docs => doc}/_static/images/mecha_section.png (100%) rename {docs => doc}/_static/images/root_system.png (100%) rename {docs => doc}/_static/images/wheat_shoot.png (100%) rename {docs => doc}/_static/openalea_web.svg (100%) rename {docs => doc}/api.md (100%) rename {docs => doc}/conf.py (62%) rename {docs => doc}/design_note.md (99%) rename {docs => doc}/governance.md (100%) rename {docs => doc}/implementations.md (100%) rename {docs => doc}/index.md (96%) create mode 100644 doc/notebooks/fspm_tf_in_python.ipynb rename {docs => doc}/overview.md (100%) rename {docs => doc}/platforms.md (100%) rename {docs => doc}/python.md (100%) rename {docs => doc}/specification.md (100%) rename {docs => doc}/using.md (100%) rename {docs => doc}/vocabulary.md (100%) diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index dae2e39..fc4938a 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -43,9 +43,13 @@ jobs: with: python-version: "3.12" - name: Install openalea.fspm_tf and the documentation's tools - run: python -m pip install numpy jsonschema sphinx pydata-sphinx-theme myst-parser sphinx-favicon sphinx-copybutton . + run: | + sudo apt-get install -y pandoc + python -m pip install numpy jsonschema pytest sphinx ".[doc]" + - name: The notebooks run + run: python -m pytest --nbmake doc/notebooks - name: Build the documentation, warnings as errors - run: python -m sphinx -b html docs docs/_build -W --keep-going + run: python -m sphinx -b html doc doc/_build -W --keep-going # The reference codec through the action other codecs use: its dumps of the conformance files, and the files it writes python-codec-conformance: diff --git a/.gitignore b/.gitignore index b92e6f0..a108694 100644 --- a/.gitignore +++ b/.gitignore @@ -116,5 +116,5 @@ examples/**/outputs/ *.report.json # the documentation build, and its pages generated from spec/vocabulary -docs/_build/ -docs/_generated/ +doc/_build/ +doc/_generated/ diff --git a/.readthedocs.yml b/.readthedocs.yml index d27053d..a160bb4 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,4 +9,4 @@ conda: environment: conda/environment.yml sphinx: - configuration: docs/conf.py + configuration: doc/conf.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e63645b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +## Unreleased: specification 0.1 (draft) + +- The specification (`spec/specification.md`), its JSON Schemas and the first vocabulary (41 standard names, the + labels with their Plant Ontology terms, the enumerations, a column of names per platform). +- `openalea.fspm_tf`, the Python reference codec (`bindings/python/`): reading and writing frames and sequences, the + validator, display copies, the comparison with expected values, the `fspm-tf` command line. +- The conformance files (`conformance/`), their expected values, and the GitHub action checking a codec against them. +- The ParaView reader plugin (`paraview/`). +- The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..3ea8cf2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,8 @@ +# Code of conduct + +fspm-TF is developed by several plant-modelling teams together. Everyone taking part (in issues, pull requests, +reviews, meetings) is expected to follow the +[Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/): +be welcoming and respectful, give and accept constructive criticism, and focus on what is best for the community. + +Unacceptable behaviour can be reported to the maintainers listed in `pyproject.toml`; reports are kept confidential. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..f451b1d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,29 @@ +# Contributing + +fspm-TF is a format shared by several FSPM platforms. Contributions are welcome from each of them: to the +specification, to the vocabulary, to a codec, to the tools. + +## What goes where + +- **`spec/`**: the agreement between the platforms (the specification, its JSON Schemas, the vocabulary). A change + there is proposed as a pull request, with the conformance scene showing it (`conformance/generate.py` and the + files it writes), the validator's check and the reference codec's update. See + [Changing the format](doc/governance.md). +- **The vocabulary** (`spec/vocabulary/`): a new standard name comes with its definition, canonical unit and kind, + and is agreed by at least two platforms. Each platform fills its own column of names. +- **`bindings//`**: one codec per language, each with its own build, tests and release, maintained by its + team. A codec passes the conformance check (`conformance/README.md`) in its CI. +- **Adapters** (a platform's model to and from the tables) stay in the platforms' own repositories. + +## Developing the Python package + +```bash +mamba env create -f conda/environment.yml # the package, editable, with the tests' and the docs' tools +mamba activate fspm_tf +pytest # the package's tests (bindings/python/test) +pytest --nbmake doc/notebooks # the documentation's notebooks run +python -m sphinx -b html doc doc/_build -W # the documentation, warnings as errors +python conformance/generate.py # after a change to the conformance scenes +``` + +Commit with explicit messages, one subject per commit, and open the pull request on the development branch. diff --git a/README.md b/README.md index 11e0f41..dde125b 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,11 @@ interface for coupling components across platforms. Status: specification 0.1, draft. -- [The documentation](docs/index.md): for users (viewing, checking, Python) and for platform developers +- [The documentation](doc/index.md): for users (viewing, checking, Python) and for platform developers (implementing a codec, conformance, changing the format), built with Sphinx (`.readthedocs.yml`; locally: - `python -m sphinx -b html docs docs/_build`) + `python -m sphinx -b html doc doc/_build`) - [The specification](spec/specification.md) -- [The design note](docs/design_note.md): the review of existing formats and the decisions taken +- [The design note](doc/design_note.md): the review of existing formats and the decisions taken - [Implementations](implementations.md) ## The repository @@ -33,7 +33,7 @@ change there, released as a new version. Everything else implements, checks or d | `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | | `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | -| `docs/design_note.md` | how the decisions were reached | no | +| `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | ## The Python package diff --git a/conda/environment.yml b/conda/environment.yml index 32f55e5..dcd0b88 100644 --- a/conda/environment.yml +++ b/conda/environment.yml @@ -4,16 +4,11 @@ channels: - conda-forge dependencies: - python + - pip + - pandoc # the documentation's notebooks (nbsphinx) - numpy - jsonschema - - pip - pytest - # the documentation (docs/, also built by readthedocs from this file) - - sphinx - - pydata-sphinx-theme - - myst-parser - - sphinx-favicon - - sphinx-copybutton - # the package itself, from the sources (its dependencies are the conda ones above) + # the package from the sources, with the documentation's and the tests' tools (also what readthedocs builds with) - pip: - - -e .. + - -e ..[doc,test] diff --git a/docs/_static/images/archisimple.png b/doc/_static/images/archisimple.png similarity index 100% rename from docs/_static/images/archisimple.png rename to doc/_static/images/archisimple.png diff --git a/docs/_static/images/granap_section.png b/doc/_static/images/granap_section.png similarity index 100% rename from docs/_static/images/granap_section.png rename to doc/_static/images/granap_section.png diff --git a/docs/_static/images/mecha_section.png b/doc/_static/images/mecha_section.png similarity index 100% rename from docs/_static/images/mecha_section.png rename to doc/_static/images/mecha_section.png diff --git a/docs/_static/images/root_system.png b/doc/_static/images/root_system.png similarity index 100% rename from docs/_static/images/root_system.png rename to doc/_static/images/root_system.png diff --git a/docs/_static/images/wheat_shoot.png b/doc/_static/images/wheat_shoot.png similarity index 100% rename from docs/_static/images/wheat_shoot.png rename to doc/_static/images/wheat_shoot.png diff --git a/docs/_static/openalea_web.svg b/doc/_static/openalea_web.svg similarity index 100% rename from docs/_static/openalea_web.svg rename to doc/_static/openalea_web.svg diff --git a/docs/api.md b/doc/api.md similarity index 100% rename from docs/api.md rename to doc/api.md diff --git a/docs/conf.py b/doc/conf.py similarity index 62% rename from docs/conf.py rename to doc/conf.py index c06c8d0..e46ff8a 100644 --- a/docs/conf.py +++ b/doc/conf.py @@ -5,56 +5,85 @@ import json import os import sys -from importlib.metadata import PackageNotFoundError, metadata +from importlib.metadata import metadata -import pydata_sphinx_theme # noqa: F401 (Pydata theme: https://pydata-sphinx-theme.readthedocs.io) +import pydata_sphinx_theme # noqa: F401 (Pydata theme: https://pydata-sphinx-theme.readthedocs.io/en/stable/index.html) HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(HERE) -sys.path.insert(0, os.path.join(ROOT, "bindings", "python", "src")) # the package, installed or not +# The package's metadata (pyproject.toml), as every OpenAlea documentation reads it +package = "openalea.fspm_tf" +meta = metadata(package) project = "fspm-TF" -try: - release = metadata("openalea.fspm_tf").get("version") -except PackageNotFoundError: - release = "0.1" +release = meta.get("version") version = ".".join(release.split(".")[:3]) -author = "Tristan Gérault" -copyright = "Cecill-C INRAE / INRIA / CIRAD" +author = meta["Author-email"].split(" <")[0] +desc = meta["Summary"] +urls = {key.strip(): value.strip() for key, value in (item.split(",") for item in meta.get_all("Project-URL"))} + +# The package's root, for autodoc when it is not installed (its sources are in bindings/python/src) +sys.path.insert(0, os.path.join(ROOT, "bindings", "python", "src")) +# -- General configuration ------------------------------------------------ extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - "sphinx.ext.viewcode", - "sphinx.ext.intersphinx", - "sphinx_copybutton", - "sphinx_favicon", - "myst_parser", + "sphinx.ext.autodoc", # support for automatic inclusion of docstrings + "sphinx.ext.autosummary", # generates autodoc summaries + "sphinx.ext.doctest", # inclusion and testing of doctest code snippets + "sphinx.ext.intersphinx", # support for linking to other projects + "sphinx.ext.mathjax", # support for math equations + "sphinx.ext.ifconfig", # support for conditional content + "sphinx.ext.viewcode", # support for links to source code + "sphinx.ext.coverage", # includes doc coverage stats in the documentation + "sphinx.ext.todo", # support for todo items + "sphinx.ext.napoleon", # support for numpy and google style docstrings + "sphinx_favicon", # support for favicon + "sphinx_copybutton", # support for copybutton in code blocks + "nbsphinx", # for integrating jupyter notebooks + "myst_parser", # for parsing .md files ] -myst_enable_extensions = ["colon_fence", "deflist"] -myst_heading_anchors = 3 templates_path = ["_templates"] -exclude_patterns = ["_build", "_templates"] +autosummary_generate = True +exclude_patterns = ["build", "_build", "_templates", "**.ipynb_checkpoints"] source_suffix = {".rst": "restructuredtext", ".md": "markdown"} master_doc = "index" +copyright = "Cecill-C INRAE / INRIA / CIRAD" language = "en" pygments_style = "sphinx" +todo_include_todos = False +myst_enable_extensions = ["colon_fence", "deflist"] +myst_heading_anchors = 3 autodoc_member_order = "bysource" -autodoc_default_options = {"members": True, "undoc-members": False} +nbsphinx_execute = "always" # the notebooks run at each build, as nbmake runs them in the tests +# -- Options for HTML output ---------------------------------------------- html_theme = "pydata_sphinx_theme" html_theme_options = { "header_links_before_dropdown": 6, - "icon_links": [{"name": "GitHub", "url": "https://github.com/openalea/fspm-TF", "icon": "fa-brands fa-github"}], + "icon_links": [{"name": "GitHub", "url": urls["Repository"], "icon": "fa-brands fa-github"}], + "show_version_warning_banner": True, "footer_start": ["copyright"], "footer_center": ["sphinx-version"], - "secondary_sidebar_items": {"**/*": ["page-toc", "edit-this-page", "sourcelink"]}, + "secondary_sidebar_items": {"**/*": ["page-toc", "edit-this-page", "sourcelink"], "examples/no-sidebar": []}, "use_edit_page_button": True, } html_static_path = ["_static"] html_logo = "_static/openalea_web.svg" html_favicon = "_static/openalea_web.svg" -html_context = {"github_user": "openalea", "github_repo": "fspm-TF", "github_version": "main", "doc_path": "docs"} +html_domain_indices = True +html_use_index = True +html_split_index = False +html_show_sourcelink = True +html_show_sphinx = True +html_show_copyright = True +htmlhelp_basename = "fspm_tf_documentation" +html_context = {"github_user": "openalea", "github_repo": "fspm-TF", "github_version": "main", "doc_path": "doc"} + +# -- Options for LaTeX, manual page and Texinfo output --------------------- +latex_elements = {} +latex_documents = [(master_doc, "fspm_tf.tex", "fspm-TF Documentation", "INRAE / inria / CIRAD", "manual")] +man_pages = [(master_doc, "fspm_tf", "fspm-TF Documentation", [author], 1)] +texinfo_documents = [(master_doc, "fspm_tf", "fspm-TF Documentation", author, "fspm_tf", desc, "Miscellaneous")] intersphinx_mapping = {"python": ("https://docs.python.org/3", None)} diff --git a/docs/design_note.md b/doc/design_note.md similarity index 99% rename from docs/design_note.md rename to doc/design_note.md index d82fea8..3ac46c2 100644 --- a/docs/design_note.md +++ b/doc/design_note.md @@ -446,7 +446,7 @@ This repository is platform-neutral, separate from metafspm, like RSML. ``` fspm-TF/ README.md - docs/design_note.md this note + doc/design_note.md this note spec/ the agreement, the only folder discussed with the platforms: specification.md the specification schema/ JSON Schema of FSPM_scene and of the index file diff --git a/docs/governance.md b/doc/governance.md similarity index 100% rename from docs/governance.md rename to doc/governance.md diff --git a/docs/implementations.md b/doc/implementations.md similarity index 100% rename from docs/implementations.md rename to doc/implementations.md diff --git a/docs/index.md b/doc/index.md similarity index 96% rename from docs/index.md rename to doc/index.md index 960faa5..beb2083 100644 --- a/docs/index.md +++ b/doc/index.md @@ -32,6 +32,7 @@ For those who run models and want to look at, keep or share their results. fspm-TF in five minutes Viewing and checking files Reading and writing in Python +A notebook: fspm-TF in Python ``` ## Implementing fspm-TF diff --git a/doc/notebooks/fspm_tf_in_python.ipynb b/doc/notebooks/fspm_tf_in_python.ipynb new file mode 100644 index 0000000..58e68e1 --- /dev/null +++ b/doc/notebooks/fspm_tf_in_python.ipynb @@ -0,0 +1,293 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "514c77a2", + "metadata": {}, + "source": [ + "# fspm-TF in Python\n", + "\n", + "This notebook reads, inspects and writes fspm-TF files with `openalea.fspm_tf`, the format's Python reference codec.\n", + "It uses the repository's conformance files, small reference scenes in `conformance/`, as examples.\n", + "\n", + "- a **frame** is one time step of a scene: one `.glb` file;\n", + "- it holds **data structures** (a plant population, a soil grid...), each made of **entity sets** (one per scale: Plant,\n", + " Axis, SubOrgan...), which are tables of numpy columns;\n", + "- every variable column comes with a **declaration**: unit, standard name, kind, role." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "438f5812", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import fspm_tf\n", + "\n", + "CONFORMANCE = os.path.join(\"..\", \"..\", \"conformance\") # the repository's reference scenes" + ] + }, + { + "cell_type": "markdown", + "id": "31adf4b2", + "metadata": {}, + "source": [ + "## Reading a frame\n", + "\n", + "`fspm_tf.read` gives a `Frame`: its time, and its data structures." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1dd1105d", + "metadata": {}, + "outputs": [], + "source": [ + "frame = fspm_tf.read(os.path.join(CONFORMANCE, \"soil_and_roots.glb\"))\n", + "print(frame.time, frame.time_unit, frame.step)\n", + "for data_structure in frame.data_structures:\n", + " print(data_structure.id, data_structure.kind, [entity_set.name for entity_set in data_structure.entity_sets])" + ] + }, + { + "cell_type": "markdown", + "id": "024ae15f", + "metadata": {}, + "source": [ + "Each entity set is a table: one row per entity, one numpy column per variable. Pandas shows it directly (vector\n", + "columns, one per row, aside)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cded09d7", + "metadata": {}, + "outputs": [], + "source": [ + "roots = frame.data_structure(\"root_system\")\n", + "segments = roots.entity_set(\"SubOrgan\")\n", + "pd.DataFrame({name: values for name, values in segments.columns.items() if np.ndim(values) == 1})" + ] + }, + { + "cell_type": "markdown", + "id": "1562b906", + "metadata": {}, + "source": [ + "The columns `id`, `label`, `complex`, `parent` and `edge_type` are the multiscale graph: `complex` is the entity\n", + "containing a segment one scale up (its axis), `parent` the segment it follows, and `edge_type` how (in line, or as a\n", + "branch)." + ] + }, + { + "cell_type": "markdown", + "id": "dcfbaf6c", + "metadata": {}, + "source": [ + "## What the variables mean\n", + "\n", + "Each variable is declared by its writer: its unit (a UCUM code), its standard name in the shared vocabulary, how it\n", + "combines (its kind) and its role." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e8d75be", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame({name: vars(variable) for name, variable in segments.variables.items()}).T[\n", + " [\"unit\", \"standard_name\", \"kind\", \"role\", \"description\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2b30a2d", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.fspm_tf import vocabulary\n", + "\n", + "vocabulary.standard_names()[\"water_potential\"][\"definition\"], vocabulary.standard_names()[\"water_potential\"][\"canonical_unit\"]" + ] + }, + { + "cell_type": "markdown", + "id": "9a180b10", + "metadata": {}, + "source": [ + "## The soil, and which cells each root segment lies in\n", + "\n", + "A grid is a data structure too, and a **mapping** says which soil cell each segment is in, with a weight." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "970cdb4d", + "metadata": {}, + "outputs": [], + "source": [ + "soil = frame.data_structure(\"soil\")\n", + "print(soil.grid)\n", + "mapping = frame.mappings[0]\n", + "pd.DataFrame(mapping.columns)" + ] + }, + { + "cell_type": "markdown", + "id": "34d6e326", + "metadata": {}, + "source": [ + "## Geometry\n", + "\n", + "A mesh's vertices carry the row of their entity at every scale they represent: here each triangle knows its segment,\n", + "its axis and its plant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7a4bb1c", + "metadata": {}, + "outputs": [], + "source": [ + "mesh = roots.meshes[0]\n", + "print(len(mesh.positions) // 3, \"triangles; rows by scale:\", list(mesh.feature_rows))\n", + "first_triangle_segment = segments.columns[\"id\"][mesh.feature_rows[\"SubOrgan\"][0]]\n", + "first_triangle_segment" + ] + }, + { + "cell_type": "markdown", + "id": "a5089917", + "metadata": {}, + "source": [ + "## A sequence of frames\n", + "\n", + "A simulation writes a frame every so many steps, with an index, `scene.fspm.json`. Entities keep their ids from\n", + "frame to frame." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c18ea1c8", + "metadata": {}, + "outputs": [], + "source": [ + "index = os.path.join(CONFORMANCE, \"growing_sequence\", \"scene.fspm.json\")\n", + "rows = []\n", + "for growing in fspm_tf.read_sequence(index):\n", + " elements = growing.data_structures[0].entity_set(\"SubOrgan\")\n", + " rows.append({\"time (d)\": growing.time, \"segments\": elements.count, \"root length (m)\": elements.columns[\"length\"].sum()})\n", + "pd.DataFrame(rows)" + ] + }, + { + "cell_type": "markdown", + "id": "851aa18e", + "metadata": {}, + "source": [ + "## Writing and checking a frame\n", + "\n", + "Writing builds the same objects. The validator checks a file against the specification: ids, links, units against\n", + "the vocabulary, tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f88d3c6b", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.fspm_tf.validate import validate_bytes\n", + "\n", + "plant = fspm_tf.EntitySet(\"Plant\", rank=1, columns=dict(id=np.array([1]), label=np.array([\"Wheat\"], dtype=object)))\n", + "segments = fspm_tf.EntitySet(\"SubOrgan\", rank=6, columns=dict(\n", + " id=np.array([10, 11, 12]), label=np.array([\"RootSegment\"] * 3, dtype=object), complex=np.array([1, 1, 1]),\n", + " parent=np.array([-1, 10, 11]), edge_type=np.array([\"none\", \"successor\", \"successor\"], dtype=object),\n", + " length=np.array([0.01, 0.01, 0.004])),\n", + " variables={\"length\": fspm_tf.Variable(unit=\"m\", standard_name=\"length\", kind=\"extensive\")})\n", + "written = fspm_tf.Frame([fspm_tf.DataStructure(\"roots\", entity_sets=[plant, segments])], time=0., time_unit=\"d\")\n", + "data = fspm_tf.to_bytes(written)\n", + "print(len(data), \"bytes;\", validate_bytes(data))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b4ffd9e", + "metadata": {}, + "outputs": [], + "source": [ + "again = fspm_tf.from_bytes(data)\n", + "again.data_structure(\"roots\").entity_set(\"SubOrgan\").columns[\"length\"]" + ] + }, + { + "cell_type": "markdown", + "id": "077e016f", + "metadata": {}, + "source": [ + "A wrong unit is caught: a length in seconds does not convert to the standard name's metres." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e9e76d8", + "metadata": {}, + "outputs": [], + "source": [ + "segments.variables[\"length\"] = fspm_tf.Variable(unit=\"s\", standard_name=\"length\", kind=\"extensive\")\n", + "print(validate_bytes(fspm_tf.to_bytes(written)))" + ] + }, + { + "cell_type": "markdown", + "id": "1ebb1400", + "metadata": {}, + "source": [ + "## For other viewers: display copies\n", + "\n", + "Viewers that do not read the tables (Blender, a web viewer, ParaView without fspm-TF's plugin) colour shapes by\n", + "per-vertex attributes. A display copy adds one, from the tables." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff9605db", + "metadata": {}, + "outputs": [], + "source": [ + "from openalea.fspm_tf.display import add_display\n", + "\n", + "shown = add_display(fspm_tf.read(os.path.join(CONFORMANCE, \"root_system.glb\")), [\"water_potential\"])\n", + "mesh = shown.data_structures[0].meshes[0]\n", + "mesh.display[\"water_potential\"][:6], mesh.display_sources" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/overview.md b/doc/overview.md similarity index 100% rename from docs/overview.md rename to doc/overview.md diff --git a/docs/platforms.md b/doc/platforms.md similarity index 100% rename from docs/platforms.md rename to doc/platforms.md diff --git a/docs/python.md b/doc/python.md similarity index 100% rename from docs/python.md rename to doc/python.md diff --git a/docs/specification.md b/doc/specification.md similarity index 100% rename from docs/specification.md rename to doc/specification.md diff --git a/docs/using.md b/doc/using.md similarity index 100% rename from docs/using.md rename to doc/using.md diff --git a/docs/vocabulary.md b/doc/vocabulary.md similarity index 100% rename from docs/vocabulary.md rename to doc/vocabulary.md diff --git a/pyproject.toml b/pyproject.toml index 634f4df..4a25d11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,11 +60,15 @@ dev = [ "pytest-cov >=3", ] doc = [ - "sphinx", "pydata-sphinx-theme", "myst-parser", "sphinx-favicon", + "ipykernel", "sphinx-copybutton", + "ipython_genutils", + "nbsphinx", + "nbmake", # the notebooks are tested: pytest --nbmake doc/notebooks + "pandas", # the notebooks' tables ] [project.scripts] From c9256a788f242dcda68dc480b5679234e8b02913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 12:01:12 +0100 Subject: [PATCH 02/39] FspmTF.jl, the Julia codec (bindings/julia/FspmTF): the GLB container, the data model, reading and writing frames and sequences, the conformance dump (every conformance file read as expected; what it writes passes fspm-tf validate and glTF-Validator with no warning); a MultiScaleTreeGraph.jl package extension (from_mtg, to_mtg: MTG.jl's test files rebuilt exactly, another platform's graph read as an MTG); its CI (Julia 1.10 and latest, the conformance action); doc/julia.md, implementations --- .github/workflows/julia-codec.yml | 55 ++ .gitignore | 3 + CHANGELOG.md | 2 + README.md | 2 +- bindings/julia/FspmTF/Project.toml | 23 + .../ext/FspmTFMultiScaleTreeGraphExt.jl | 396 ++++++++++++++ bindings/julia/FspmTF/src/FspmTF.jl | 51 ++ bindings/julia/FspmTF/src/codec.jl | 486 ++++++++++++++++++ bindings/julia/FspmTF/src/dump.jl | 94 ++++ bindings/julia/FspmTF/src/glb.jl | 117 +++++ bindings/julia/FspmTF/src/model.jl | 299 +++++++++++ bindings/julia/FspmTF/src/sequence.jl | 63 +++ bindings/julia/FspmTF/test/Project.toml | 8 + bindings/julia/FspmTF/test/runtests.jl | 203 ++++++++ bindings/julia/README.md | 13 + bindings/julia/conformance_run.jl | 28 + doc/index.md | 1 + doc/julia.md | 92 ++++ doc/platforms.md | 2 +- doc/python.md | 3 +- implementations.md | 7 +- 21 files changed, 1943 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/julia-codec.yml create mode 100644 bindings/julia/FspmTF/Project.toml create mode 100644 bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl create mode 100644 bindings/julia/FspmTF/src/FspmTF.jl create mode 100644 bindings/julia/FspmTF/src/codec.jl create mode 100644 bindings/julia/FspmTF/src/dump.jl create mode 100644 bindings/julia/FspmTF/src/glb.jl create mode 100644 bindings/julia/FspmTF/src/model.jl create mode 100644 bindings/julia/FspmTF/src/sequence.jl create mode 100644 bindings/julia/FspmTF/test/Project.toml create mode 100644 bindings/julia/FspmTF/test/runtests.jl create mode 100644 bindings/julia/README.md create mode 100644 bindings/julia/conformance_run.jl create mode 100644 doc/julia.md diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml new file mode 100644 index 0000000..ef928cd --- /dev/null +++ b/.github/workflows/julia-codec.yml @@ -0,0 +1,55 @@ +# The Julia codec (bindings/julia/FspmTF): its tests, on the oldest supported Julia and the latest release, and the +# conformance check other codecs run (its dumps of the conformance files compared with their expected values, the files +# it writes validated by fspm-tf validate and Khronos' glTF-Validator). OpenAlea's conda CI builds the Python package +# only. +name: Julia codec + +on: + push: + branches: + - main + pull_request: + +jobs: + test: + name: Julia ${{ matrix.julia-version }}, ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + julia-version: ["1.10", "1"] + os: [ubuntu-latest] + include: + - julia-version: "1" + os: windows-latest + - julia-version: "1" + os: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.julia-version }} + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + with: + project: bindings/julia/FspmTF + - uses: julia-actions/julia-runtest@v1 + with: + project: bindings/julia/FspmTF + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - name: Dump and write the conformance scenes with the Julia codec + run: | + julia --project=bindings/julia/FspmTF -e 'using Pkg; Pkg.instantiate()' + julia --project=bindings/julia/FspmTF bindings/julia/conformance_run.jl conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written diff --git a/.gitignore b/.gitignore index a108694..64f4752 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ examples/**/outputs/ # the documentation build, and its pages generated from spec/vocabulary doc/_build/ doc/_generated/ + +# Julia: environments resolved locally (the codec is a library: no committed manifest) +bindings/julia/**/Manifest*.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index e63645b..f9ef5ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,4 +8,6 @@ validator, display copies, the comparison with expected values, the `fspm-tf` command line. - The conformance files (`conformance/`), their expected values, and the GitHub action checking a codec against them. - The ParaView reader plugin (`paraview/`). +- FspmTF.jl, the Julia codec (`bindings/julia/FspmTF/`): conformant reading and writing, sequences, and a + MultiScaleTreeGraph.jl package extension (`from_mtg`, `to_mtg`); its CI, `.github/workflows/julia-codec.yml`. - The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/README.md b/README.md index dde125b..913eebd 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ change there, released as a new version. Everything else implements, checks or d | `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | | `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it | no | +| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | | `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | diff --git a/bindings/julia/FspmTF/Project.toml b/bindings/julia/FspmTF/Project.toml new file mode 100644 index 0000000..7a621ce --- /dev/null +++ b/bindings/julia/FspmTF/Project.toml @@ -0,0 +1,23 @@ +name = "FspmTF" +uuid = "b7fd6adf-cc0c-448b-9b1a-23349074d866" +authors = ["Tristan Gérault"] +version = "0.1.0" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" + +[weakdeps] +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" + +[extensions] +FspmTFMultiScaleTreeGraphExt = "MultiScaleTreeGraph" + +[compat] +Dates = "1" +JSON = "1" +MultiScaleTreeGraph = "0.16" +OrderedCollections = "1, 2" +julia = "1.10" + diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl new file mode 100644 index 0000000..ab91414 --- /dev/null +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -0,0 +1,396 @@ +""" +A MultiScaleTreeGraph (MTG.jl) as an fspm-TF `graph` data structure, and back: + +| fspm-TF | MultiScaleTreeGraph | +|---|---| +| one entity set per scale ≥ 1, `rank` the scale | the nodes of that scale | +| `id` | `node_id` | +| `label` | the node's `symbol` | +| `complex` | its first ancestor at a smaller scale (none for the scene) | +| `parent`, `edge_type` | its parent, when at the same scale and linked by `<` (successor) or `+` (branch) | +| relation `bearer` | its parent otherwise, when it is not its complex linked by `/` (e.g. an axis borne by an internode) | +| `mtg_index` | the node's `index` | +| variables | its attributes | +| scalars | the scene's (the scale 0 root's) attributes | + +What fspm-TF 0.1 cannot say is kept in the data structure's `extras["MultiScaleTreeGraph"]`, so that `to_mtg` rebuilds +the same MTG: the attributes a node does not have (fspm-TF columns have a value on every row; the missing value +written is NaN, `false`, 0 or ""), the types written as text (dates), the scene's node and its MTG metadata +(`symbols`, `scales`, `description`), and the order of children when it is not the order of their ids. +""" +module FspmTFMultiScaleTreeGraphExt + +using Dates +using FspmTF +using FspmTF: Columns, Declarations, json_object, check_identifier +using MultiScaleTreeGraph +using OrderedCollections: OrderedDict + +const PLATFORM = "MultiScaleTreeGraph" +const METADATA_ATTRIBUTES = (:symbols, :scales, :description) +const LINK_NAMES = Dict(:< => "successor", :+ => "branch", :/ => "decomposition") +const LINKS = Dict(name => link for (link, name) in LINK_NAMES) +const TEXT_TYPES = Dict("Date" => Date, "DateTime" => DateTime, "Time" => Time) +const INDEX_COLUMN = "mtg_index" +const INDEX_VARIABLE = Variable(unit = "1", kind = "descriptor", role = "state", + description = "the node's index in its MTG code (1 in Internode1)") + +parent_node(node) = isroot(node) ? nothing : parent(node) + +"""The nodes of the MTG in depth-first order, children in their order.""" +function preorder(root) + nodes, stack = Any[], Any[root] + while !isempty(stack) + node = pop!(stack) + push!(nodes, node) + append!(stack, reverse(children(node))) + end + return nodes +end + +"""The first ancestor of `node` at a smaller scale than its own, or nothing.""" +function complex_node(node) + ancestor = parent_node(node) + while ancestor !== nothing && scale(ancestor) >= scale(node) + ancestor = parent_node(ancestor) + end + return ancestor +end + +default_set_name(scale_number, symbols) = + length(symbols) == 1 && occursin(r"^[a-zA-Z_][a-zA-Z0-9_]*$", symbols[1]) ? symbols[1] : "Scale$scale_number" + +# ── Attributes as columns ─────────────────────────────────────────────── + +const TextType = Union{Date,DateTime,Time} + +"""A column of `values` (`nothing` where a node has no such attribute): the column, its missing rows (0-based), +the type written as text (or nothing); nothing for values fspm-TF cannot hold.""" +function attribute_column(values::Vector) + present = [value for value in values if value !== nothing] + missing_rows = [row - 1 for (row, value) in enumerate(values) if value === nothing] + text_type = nothing + if all(value -> value isa Bool, present) + column = Bool[value === nothing ? false : value for value in values] + elseif all(value -> value isa Integer && !(value isa Bool), present) + column = Int64[value === nothing ? 0 : value for value in values] + elseif all(value -> value isa Real && !(value isa Bool), present) + column = Float64[value === nothing ? NaN : value for value in values] + elseif all(value -> value isa Union{AbstractString,Symbol}, present) + column = String[value === nothing ? "" : string(value) for value in values] + elseif !isempty(present) && all(value -> value isa TextType, present) && length(Set(typeof.(present))) == 1 + text_type = string(nameof(typeof(first(present)))) + column = String[value === nothing ? "" : string(value) for value in values] + elseif all(value -> value isa AbstractVector{<:Real}, present) && length(Set(length.(present))) == 1 && + length(first(present)) >= 2 + width = length(first(present)) + integer = all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) + column = integer ? zeros(Int64, length(values), width) : fill(NaN, length(values), width) + for (row, value) in enumerate(values) + value === nothing || (column[row, :] .= value) + end + else + return nothing + end + return column, missing_rows, text_type +end + +"""Columns of `nodes`' attributes, by name in order of first appearance; the missing rows and text types of each. +`skip`: attributes not written; the MTG's root's metadata (MTG.jl's `symbols`, `scales`, `description`) neither.""" +function attribute_columns(nodes, set_name, variables, skip = ()) + attribute_of(node, name) = + isroot(node) && name in METADATA_ATTRIBUTES ? nothing : attribute(node, name; default = nothing) + names = Symbol[] + for node in nodes, name in attribute_names(node) + name in names || name in skip || (isroot(node) && name in METADATA_ATTRIBUTES) || push!(names, name) + end + columns, declarations = Columns(), Declarations() + missing_rows, text_types = json_object(), json_object() + for name in names + values = Any[attribute_of(node, name) for node in nodes] + converted = attribute_column(values) + if converted === nothing + @warn "attribute '$name' of '$set_name' not written: fspm-TF holds numbers, booleans, text, dates " * + "and fixed-size numeric vectors" + continue + end + column_name = string(name) + check_identifier(column_name, "attribute of '$set_name'") + column, missing_of_column, text_type = converted + columns[column_name] = column + isempty(missing_of_column) || (missing_rows[column_name] = missing_of_column) + text_type === nothing || (text_types[column_name] = text_type) + haskey(variables, column_name) && (declarations[column_name] = variables[column_name]) + end + return columns, declarations, missing_rows, text_types +end + +# ── MTG → data structure ──────────────────────────────────────────────── + +""" + from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) + -> DataStructure + +The MTG whose root is `mtg` as a `graph` data structure `id`. `scales`: the name of each scale's entity set +(scale => name, e.g. `Dict(1 => "Plant", 2 => "Axis", 3 => "Organ")`; by default the scale's symbol when it has +only one, `Scale` otherwise). `variables`: the declarations of the attributes (name => `Variable`: unit, standard +name, kind, role), MTG.jl having no units. A scale 0 root (MTG.jl's `Scene`) is the data structure itself, its +attributes the scalars. +""" +function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,String}(), + variables = Dict{String,Variable}(), platform = PLATFORM, model = nothing) + variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) + nodes = preorder(mtg) + scene = scale(mtg) == 0 ? mtg : nothing + extras = json_object() + by_scale = OrderedDict{Int,Vector{Any}}() + for node in nodes + node === scene && continue + scale(node) >= 1 || throw(ArgumentError("node $(node_id(node)): scale $(scale(node)) below a root's")) + push!(get!(by_scale, scale(node), Any[]), node) + end + sort!(by_scale) + set_names = Dict{Int,String}() + for (scale_number, scale_nodes) in by_scale + name = get(scales, scale_number, nothing) + name === nothing && (name = default_set_name(scale_number, unique(string.(symbol.(scale_nodes))))) + check_identifier(name, "the entity set name of scale $scale_number") + set_names[scale_number] = name + end + + entity_sets, missing_rows, text_types = EntitySet[], json_object(), json_object() + bearers = OrderedDict{Tuple{String,String},Vector{Tuple{Int64,Int64,String}}}() + for (scale_number, scale_nodes) in by_scale + set_name = set_names[scale_number] + columns = Columns("id" => Int64[node_id(node) for node in scale_nodes], + "label" => String[string(symbol(node)) for node in scale_nodes]) + complexes = [complex_node(node) for node in scale_nodes] + if scale_number != first(keys(by_scale)) + columns["complex"] = Int64[complex === nothing || complex === scene ? NO_ID : node_id(complex) + for complex in complexes] + end + parents, edge_types = Int64[], String[] + for (node, complex) in zip(scale_nodes, complexes) + tree_parent = parent_node(node) + if tree_parent !== nothing && scale(tree_parent) == scale(node) && link(node) in (:<, :+) + push!(parents, node_id(tree_parent)) + push!(edge_types, LINK_NAMES[link(node)]) + continue + end + push!(parents, NO_ID) + push!(edge_types, "none") + tree_parent === nothing && continue + tree_parent === complex && link(node) == :/ && continue # its complex, by decomposition + tree_parent === scene && link(node) == :/ && continue # the scene's component + tree_parent === scene && + throw(ArgumentError("node $(node_id(node)) is linked to the scene by '$(link(node))', not '/'")) + pair = (set_name, set_names[scale(tree_parent)]) + push!(get!(bearers, pair, Tuple{Int64,Int64,String}[]), + (node_id(node), node_id(tree_parent), LINK_NAMES[link(node)])) + end + columns["parent"], columns["edge_type"] = parents, edge_types + columns[INDEX_COLUMN] = Int64[index(node) for node in scale_nodes] + attributes, declarations, missing_of_set, text_types_of_set = + attribute_columns(scale_nodes, set_name, variables, (Symbol(INDEX_COLUMN),)) + merge!(columns, attributes) + declarations = merge(Declarations(INDEX_COLUMN => INDEX_VARIABLE), declarations) + isempty(missing_of_set) || (missing_rows[set_name] = missing_of_set) + isempty(text_types_of_set) || (text_types[set_name] = text_types_of_set) + push!(entity_sets, EntitySet(set_name; columns = columns, variables = declarations, rank = scale_number)) + end + + relations = Relation[] + for ((source_set, target_set), edges) in bearers + name = length(bearers) == 1 ? "bearer" : "bearer_$(source_set)_$(target_set)" + push!(relations, Relation(name; source_set = source_set, target_set = target_set, + columns = Columns("source" => Int64[edge[1] for edge in edges], + "target" => Int64[edge[2] for edge in edges], + "relation" => String[edge[3] for edge in edges]))) + end + + metadata = json_object() # MTG.jl's, on the root + for name in METADATA_ATTRIBUTES + name in attribute_names(mtg) || continue + value = attribute(mtg, name; default = nothing) # nothing kept: e.g. no description + metadata[string(name)] = value isa MultiScaleTreeGraph.ColumnTable ? + json_object(string(column) => getproperty(value, column) for column in names(value)) : value + end + isempty(metadata) || (extras["metadata"] = metadata) + scalars = nothing + if scene !== nothing + extras["scene"] = json_object("id" => node_id(scene), "symbol" => string(symbol(scene)), + "index" => index(scene), "link" => string(link(scene))) + columns, declarations, missing_of_scene, text_types_of_scene = + attribute_columns([scene], "scalars", variables) + if !isempty(columns) + scalars = Table("scalars"; columns = columns, variables = declarations) + isempty(missing_of_scene) || (missing_rows["scalars"] = missing_of_scene) + isempty(text_types_of_scene) || (text_types["scalars"] = text_types_of_scene) + end + end + isempty(missing_rows) || (extras["missing"] = missing_rows) + isempty(text_types) || (extras["text_types"] = text_types) + order = [node_id(node) for node in nodes] + order == [node_id(node) for node in preorder_by_id(nodes)] || (extras["order"] = order) + + return DataStructure(id; kind = "graph", entity_sets = entity_sets, relations = relations, scalars = scalars, + platform = platform, model = model, extras = json_object(PLATFORM => extras)) +end + +"""The depth-first order of the same tree, children sorted by id: the order to_mtg rebuilds without `order`.""" +function preorder_by_id(nodes) + ordered, stack = Any[], Any[first(nodes)] + while !isempty(stack) + node = pop!(stack) + push!(ordered, node) + append!(stack, sort(children(node); by = node_id, rev = true)) + end + return ordered +end + +# ── Data structure → MTG ──────────────────────────────────────────────── + +attribute_value(column::AbstractVector, row) = column[row] +attribute_value(column::AbstractMatrix, row) = collect(column[row, :]) + +function restore(value, text_type) + text_type === nothing && return value + return TEXT_TYPES[text_type](value) +end + +"""The attributes of each row of `table`: Dict(column => value), without the rows' missing ones.""" +function row_attributes(table, missing_of_table, text_types_of_table, skip) + missing_rows = Dict(name => Set(rows) for (name, rows) in pairs(missing_of_table)) + rows = [Dict{Symbol,Any}() for _ in 1:row_count(table)] + for (name, column) in table.columns + name in skip && continue + absent = get(missing_rows, name, Set{Int}()) + text_type = get(text_types_of_table, name, nothing) + for row in 1:row_count(table) + (row - 1) in absent && continue + rows[row][Symbol(name)] = restore(attribute_value(column, row), text_type) + end + end + return rows +end + +""" + to_mtg(data_structure) -> Node + +The `graph` data structure as an MTG (its root node). A data structure written by `from_mtg` is rebuilt as it was: +ids, symbols, scales, indices, links, attributes, the order of children. Another one (e.g. from metafspm) becomes an +MTG with one scale per entity set (its rank), each entity's symbol its label (else its entity set's name), its +variables as attributes, under a `Scene` root holding the scalars. +""" +function FspmTF.to_mtg(data_structure::DataStructure) + data_structure.kind == "graph" || + throw(ArgumentError("data structure '$(data_structure.id)' is a $(data_structure.kind), not a graph")) + extras = get(data_structure.extras, PLATFORM, json_object()) + missing_rows, text_types = get(extras, "missing", json_object()), get(extras, "text_types", json_object()) + entity_sets = [set for set in data_structure.entity_sets if set.rank !== nothing] + sort!(entity_sets; by = set -> set.rank) + + # Every entity: its scale, symbol, index, attributes; its parent in the tree and the link to it + encoding, attributes, tree_parent = Dict{Int64,Any}(), Dict{Int64,Dict{Symbol,Any}}(), Dict{Int64,Tuple{Int64,Symbol}}() + reserved = (FspmTF.ENTITY_SET_RESERVED..., INDEX_COLUMN) + for set in entity_sets + ids = set.columns["id"] + labels = get(set.columns, "label", fill(set.name, row_count(set))) + indices = get(set.columns, INDEX_COLUMN, zeros(Int64, row_count(set))) + rows = row_attributes(set, get(missing_rows, set.name, json_object()), get(text_types, set.name, json_object()), + reserved) + for row in 1:row_count(set) + encoding[ids[row]] = (symbol = labels[row], index = indices[row], scale = set.rank) + attributes[ids[row]] = rows[row] + end + complexes = get(set.columns, "complex", fill(NO_ID, row_count(set))) + for row in 1:row_count(set) # complex: by decomposition, '/' + complexes[row] == NO_ID || (tree_parent[ids[row]] = (complexes[row], :/)) + end + end + for relation in data_structure.relations # bearers: the link of their label + startswith(relation.name, "bearer") || continue + labels = get(relation.columns, "relation", fill("branch", row_count(relation))) + for (source, target, label) in zip(relation.columns["source"], relation.columns["target"], labels) + tree_parent[source] = (target, LINKS[label]) + end + end + for set in entity_sets # parents at the same scale, '<' or '+' + haskey(set.columns, "parent") || continue + edge_types = set.columns["edge_type"] + for (id, parent_id, edge_type) in zip(set.columns["id"], set.columns["parent"], edge_types) + parent_id == NO_ID || (tree_parent[id] = (parent_id, edge_type == "successor" ? :< : :+)) + end + end + + # The root: the scene written by from_mtg, or a new scene when there are several roots or scalars + roots = sort([id for id in keys(encoding) if !haskey(tree_parent, id)]) + scene_entry = get(extras, "scene", nothing) + scene_attributes = data_structure.scalars === nothing ? Dict{Symbol,Any}() : + only(row_attributes(data_structure.scalars, get(missing_rows, "scalars", json_object()), + get(text_types, "scalars", json_object()), ())) + if scene_entry === nothing && length(roots) == 1 && isempty(scene_attributes) + root_id = only(roots) + root_encoding = encoding[root_id] + root = Node(root_id, MutableNodeMTG(:/, Symbol(root_encoding.symbol), root_encoding.index, + root_encoding.scale), attributes[root_id]) + roots = Int64[] + else + scene_entry === nothing && + (scene_entry = json_object("id" => maximum(keys(encoding); init = -1) + 1, "symbol" => "Scene", + "index" => 0, "link" => "/")) + root = Node(Int(scene_entry["id"]), MutableNodeMTG(Symbol(scene_entry["link"]), Symbol(scene_entry["symbol"]), + Int(scene_entry["index"]), 0), scene_attributes) + root_id = node_id(root) + end + children_of = Dict{Int64,Vector{Int64}}() + for (id, (parent_id, _)) in tree_parent + push!(get!(children_of, parent_id, Int64[]), id) + end + isempty(roots) || (children_of[root_id] = vcat(roots, get(children_of, root_id, Int64[]))) + rank_of = haskey(extras, "order") ? Dict(Int64(id) => position for (position, id) in enumerate(extras["order"])) : + nothing + for list in values(children_of) + rank_of === nothing ? sort!(list) : sort!(list; by = id -> get(rank_of, id, typemax(Int))) + end + stack = Any[root] + while !isempty(stack) + node = pop!(stack) + created = Any[] + for id in get(children_of, node_id(node), Int64[]) + child_encoding = encoding[id] + child_link = haskey(tree_parent, id) ? tree_parent[id][2] : :/ + push!(created, Node(Int(id), node, MutableNodeMTG(child_link, Symbol(child_encoding.symbol), + child_encoding.index, child_encoding.scale), + attributes[id])) + end + append!(stack, reverse(created)) + end + + metadata = get(extras, "metadata", nothing) + if metadata !== nothing + metadata_values = Pair{Symbol,Any}[] + haskey(metadata, "symbols") && push!(metadata_values, :symbols => String.(metadata["symbols"])) + haskey(metadata, "scales") && push!(metadata_values, :scales => Int.(metadata["scales"])) + if haskey(metadata, "description") && metadata["description"] === nothing + push!(metadata_values, :description => nothing) + elseif haskey(metadata, "description") + description = metadata["description"] + column_names = Symbol.(collect(keys(description))) + columns = AbstractVector[column_values(value) for value in values(description)] + push!(metadata_values, :description => MultiScaleTreeGraph.ColumnTable(column_names, columns)) + end + append!(root, (; metadata_values...)) + elseif scene_entry !== nothing + symbols = unique([(encoding[id].symbol, encoding[id].scale) for id in sort(collect(keys(encoding)))]) + append!(root, (symbols = vcat([string(symbol(root))], [string(item[1]) for item in symbols]), + scales = vcat([0], [item[2] for item in symbols]))) + end + return root +end + +column_values(value::AbstractVector) = + all(item -> item isa AbstractVector, value) ? [String.(item) for item in value] : + all(item -> item isa AbstractString, value) ? String.(value) : collect(value) + +end diff --git a/bindings/julia/FspmTF/src/FspmTF.jl b/bindings/julia/FspmTF/src/FspmTF.jl new file mode 100644 index 0000000..13b8249 --- /dev/null +++ b/bindings/julia/FspmTF/src/FspmTF.jl @@ -0,0 +1,51 @@ +""" + FspmTF + +Reading and writing fspm-TF files in Julia: whole functional-structural plant model scenes (multiscale plant graphs, +their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 +binary (GLB) file per time step. The specification is `spec/specification.md` in the fspm-TF repository. + + read_frame(path) -> Frame write_frame(path, frame) + from_bytes(data) -> Frame to_bytes(frame) -> Vector{UInt8} + read_index(path), read_sequence(path) write_sequence(directory, frames) + frame_to_dict(frame), write_dump(path, frame) the layout of the conformance files' expected values + +With MultiScaleTreeGraph loaded, `from_mtg` and `to_mtg` convert an MTG to and from a data structure. +""" +module FspmTF + +using JSON +using OrderedCollections: OrderedDict + +"""The version of the specification this package reads and writes.""" +const SPECIFICATION_VERSION = "0.1" + +include("glb.jl") +include("model.jl") +include("codec.jl") +include("sequence.jl") +include("dump.jl") + +export Frame, DataStructure, EntitySet, Relation, Table, Mapping, Grid, Mesh, Variable, NO_ID, EDGE_TYPES +export read_frame, write_frame, from_bytes, to_bytes, read_index, read_sequence, write_sequence, write_index +export frame_to_dict, write_dump, row_count, variable_names, entity_ids, entity_set, data_structure +export from_mtg, to_mtg + +""" + from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) + -> DataStructure + +A MultiScaleTreeGraph (its root node) as a `graph` data structure. Defined when MultiScaleTreeGraph is loaded +(the package extension `FspmTFMultiScaleTreeGraphExt`). +""" +function from_mtg end + +""" + to_mtg(data_structure) -> Node + +A `graph` data structure as a MultiScaleTreeGraph (its root node). Defined when MultiScaleTreeGraph is loaded (the +package extension `FspmTFMultiScaleTreeGraphExt`). +""" +function to_mtg end + +end diff --git a/bindings/julia/FspmTF/src/codec.jl b/bindings/julia/FspmTF/src/codec.jl new file mode 100644 index 0000000..de6de16 --- /dev/null +++ b/bindings/julia/FspmTF/src/codec.jl @@ -0,0 +1,486 @@ +# Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), +# meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data +# structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, +# standard names, kinds and roles. + +const EXTENSION = "FSPM_scene" +const METADATA = "EXT_structural_metadata" +const FEATURES = "EXT_mesh_features" +const EDGE_TYPE_ENUM = "FSPM_edge_type" +const IDENTIFIER = r"^[a-zA-Z_][a-zA-Z0-9_]*$" +const ARRAY_BUFFER, ELEMENT_ARRAY_BUFFER = 34962, 34963 +const FLOAT, UNSIGNED_INT, UNSIGNED_SHORT, UNSIGNED_BYTE = 5126, 5125, 5123, 5121 +const MAX_FLOAT_FEATURE = 2^24 # feature rows are stored as Float32 vertex attributes +# The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about +# x, column-major). Readers of the tables use the scene's frame, not this node. +const Z_UP_TO_Y_UP = [1.0, 0.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0] + +const COMPONENT_TYPES = Dict{DataType,String}(T => uppercase(string(T)) for T in COLUMN_TYPES) +const TYPES_OF_COMPONENTS = Dict{String,DataType}(name => T for (T, name) in COMPONENT_TYPES) +# EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; +Inf +# and -Inf as the largest finite value and the next one after the most negative, declared in the property's extras +# (FSPM_infinity). +no_data_float(::Type{T}) where {T<:AbstractFloat} = -floatmax(T) +infinity_stand_ins(::Type{T}) where {T<:AbstractFloat} = (positive = floatmax(T), negative = nextfloat(-floatmax(T))) + +json_object(pairs...) = OrderedDict{String,Any}(pairs...) + +# ── Writing ────────────────────────────────────────────────────────────── + +""" + to_bytes(frame) -> Vector{UInt8} + +The GLB file of `frame`. +""" +function to_bytes(frame::Frame) + buffers = BufferBuilder() + schema = json_object("id" => "fspm_scene", "classes" => json_object(), "enums" => json_object( + EDGE_TYPE_ENUM => json_object("valueType" => "UINT8", + "values" => [json_object("name" => name, "value" => value - 1) + for (value, name) in enumerate(EDGE_TYPES)]))) + tables, variables = Any[], json_object() + document = json_object("asset" => json_object("version" => "2.0", "generator" => "FspmTF.jl $SPECIFICATION_VERSION"), + "scene" => 0, "scenes" => [json_object("nodes" => [0])], + "nodes" => Any[json_object("name" => "fspm_scene", "matrix" => Z_UP_TO_Y_UP, + "children" => Int[])]) + accessors, meshes = Any[], Any[] + + function add_table(table::AbstractTable, class_id::AbstractString) + check_identifier(class_id, "class") + properties, stored = encode_table(table, class_id, buffers, schema) + schema["classes"][class_id] = json_object("name" => table.name, "properties" => properties) + push!(tables, json_object("name" => table.name, "class" => class_id, "count" => row_count(table), + "properties" => stored)) + declared = json_object(name => declaration_json(table.variables[name]) + for name in variable_names(table) if haskey(table.variables, name)) + isempty(declared) || (variables[class_id] = declared) + return length(tables) - 1 + end + + data_structures = Any[] + for data_structure in frame.data_structures + check_identifier(data_structure.id, "data structure id") + entry = json_object("id" => data_structure.id, "kind" => data_structure.kind) + data_structure.platform === nothing || (entry["platform"] = data_structure.platform) + data_structure.model === nothing || (entry["model"] = data_structure.model) + entry["origin"] = vec(data_structure.origin) # column-major, as glTF matrices + entry["entitySets"] = [json_object("name" => set.name, "rank" => set.rank, + "propertyTable" => add_table(set, "$(data_structure.id)__$(set.name)")) + for set in data_structure.entity_sets] + if !isempty(data_structure.relations) + entry["relations"] = [json_object("name" => relation.name, "sourceSet" => relation.source_set, + "targetSet" => relation.target_set, + "propertyTable" => add_table(relation, "$(data_structure.id)__$(relation.name)")) + for relation in data_structure.relations] + end + data_structure.scalars === nothing || + (entry["scalars"] = add_table(data_structure.scalars, "$(data_structure.id)__scalars")) + data_structure.grid === nothing || (entry["grid"] = grid_json(data_structure.grid)) + isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) + node = json_object("name" => data_structure.id) + data_structure.origin == identity_matrix() || (node["matrix"] = entry["origin"]) + push!(document["nodes"], node) + push!(document["nodes"][1]["children"], length(document["nodes"]) - 1) + entry["node"] = length(document["nodes"]) - 1 + if !isempty(data_structure.meshes) + node["children"], entry["meshes"] = Int[], Any[] + for mesh in data_structure.meshes + push!(meshes, encode_mesh(mesh, data_structure, entry, buffers, accessors)) + push!(document["nodes"], json_object("name" => mesh.name, "mesh" => length(meshes) - 1)) + push!(node["children"], length(document["nodes"]) - 1) + mesh_entry = json_object("name" => mesh.name, "mesh" => length(meshes) - 1) + if !isempty(mesh.display) + mesh_entry["display"] = [json_object("attribute" => display_attribute(name), "variable" => name, + "entitySet" => get(mesh.display_sources, name, nothing)) + for name in keys(mesh.display)] + end + push!(entry["meshes"], mesh_entry) + end + end + push!(data_structures, entry) + end + + mappings = Any[] + for mapping in frame.mappings + check_identifier(mapping.name, "mapping name") + entry = json_object("name" => mapping.name, + "source" => json_object("dataStructure" => mapping.source[1], "entitySet" => mapping.source[2]), + "target" => json_object("dataStructure" => mapping.target[1], "entitySet" => mapping.target[2])) + mapping.method === nothing || (entry["method"] = mapping.method) + entry["propertyTable"] = add_table(mapping, "mapping__$(mapping.name)") + push!(mappings, entry) + end + + scene = json_object("version" => SPECIFICATION_VERSION, + "frame" => json_object("time" => Float64(frame.time), "timeUnit" => frame.time_unit, + "step" => frame.step)) + frame.vocabulary_version === nothing || (scene["vocabulary"] = frame.vocabulary_version) + isempty(frame.writer) || (scene["writer"] = frame.writer) + scene["dataStructures"] = data_structures + isempty(mappings) || (scene["mappings"] = mappings) + isempty(variables) || (scene["variables"] = variables) + + other_extensions = get(frame.unknown, "extensions", json_object()) + extensions = json_object(METADATA => json_object("schema" => schema, "propertyTables" => tables), + EXTENSION => scene) + merge!(extensions, other_extensions) + document["extensionsUsed"] = vcat([METADATA, EXTENSION], isempty(meshes) ? String[] : [FEATURES], + collect(String, keys(other_extensions))) + document["extensions"] = extensions + haskey(frame.unknown, "extras") && (document["extras"] = frame.unknown["extras"]) + if !isempty(meshes) + document["meshes"], document["accessors"] = meshes, accessors + end + if !isempty(buffers.views) + # the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + append!(buffers.data, zeros(UInt8, padding(length(buffers.data)))) + document["buffers"] = [json_object("byteLength" => length(buffers.data))] + document["bufferViews"] = buffers.views + end + return pack(document, buffers.data) +end + +""" + write_frame(path, frame) + +Write `frame` to the GLB file `path`. +""" +write_frame(path::AbstractString, frame::Frame) = (write(path, to_bytes(frame)); path) + +check_identifier(name, what) = (name isa AbstractString && occursin(IDENTIFIER, name)) || + throw(ArgumentError("$what '$name' must be an identifier (letters, digits, '_'; not starting with a digit)")) + +function declaration_json(variable::Variable) + declaration = json_object("unit" => variable.unit, "role" => variable.role) + variable.standard_name === nothing || (declaration["standardName"] = variable.standard_name) + variable.kind === nothing || (declaration["kind"] = variable.kind) + variable.minimum === nothing || (declaration["minimum"] = variable.minimum) + variable.maximum === nothing || (declaration["maximum"] = variable.maximum) + isempty(variable.references) || (declaration["references"] = variable.references) + isempty(variable.doi) || (declaration["doi"] = variable.doi) + return declaration +end + +"""Row-major values of a column: a vector column's rows one after the other.""" +row_major(column::AbstractVector) = column +row_major(column::AbstractMatrix) = vec(permutedims(column)) + +function encode_table(table::AbstractTable, class_id::AbstractString, buffers::BufferBuilder, schema) + properties, stored = json_object(), json_object() + reserved_names = reserved(table) + for (name, column) in table.columns + check_identifier(name, "column of '$(table.name)'") + description = declaration(table, name).description + property = json_object() + description === nothing || (property["description"] = description) + element = eltype(column) + if name in ENUM_COLUMNS && name in reserved_names + enum_name, codes = enum_codes(name, column, class_id, schema) + property["type"], property["enumType"] = "ENUM", enum_name + stored[name] = json_object("values" => add!(buffers, codes)) + elseif element <: AbstractString + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': text vectors are not supported")) + property["type"] = "STRING" + encoded = [Vector{UInt8}(codeunits(value)) for value in column] + offsets = vcat(0, cumsum(Int64[length(value) for value in encoded])) + offset_type = offsets[end] < 2^32 ? UInt32 : UInt64 + stored[name] = json_object("values" => add!(buffers, reduce(vcat, encoded; init = UInt8[])), + "stringOffsets" => add!(buffers, offset_type.(offsets)), + "stringOffsetType" => uppercase(string(offset_type))) + elseif element === Bool + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': boolean vectors are not supported")) + property["type"] = "BOOLEAN" + packed = zeros(UInt8, cld(length(column), 8)) + for (row, value) in enumerate(column) + value && (packed[(row-1)÷8+1] |= UInt8(1) << ((row - 1) % 8)) # bit order: little + end + stored[name] = json_object("values" => add!(buffers, packed)) + else + haskey(COMPONENT_TYPES, element) || + throw(ArgumentError("column '$name' of '$(table.name)': type $element is not supported")) + property["type"], property["componentType"] = "SCALAR", COMPONENT_TYPES[element] + if column isa AbstractMatrix + size(column, 2) >= 2 || + throw(ArgumentError("column '$name' of '$(table.name)': vectors have at least 2 components")) + property["array"], property["count"] = true, size(column, 2) + end + data = column + if element <: AbstractFloat + if any(isinf, column) + stand_ins = infinity_stand_ins(element) + any(value -> value == stand_ins.positive || value == stand_ins.negative, column) && + throw(ArgumentError("column '$name' of '$(table.name)' holds both infinities and the values " * + "standing for them ($stand_ins)")) + property["extras"] = json_object("FSPM_infinity" => json_object( + "positive" => Float64(stand_ins.positive), "negative" => Float64(stand_ins.negative))) + data = map(value -> value == Inf ? stand_ins.positive : + value == -Inf ? stand_ins.negative : value, data) + end + if any(isnan, data) + no_data = no_data_float(element) + any(==(no_data), data) && + throw(ArgumentError("column '$name' of '$(table.name)' holds both NaN and $no_data, the " * + "value standing for NaN")) + property["noData"] = Float64(no_data) + data = map(value -> isnan(value) ? no_data : value, data) + end + elseif name in ("complex", "parent") && name in reserved_names + property["noData"] = NO_ID + end + stored[name] = json_object("values" => add!(buffers, row_major(data))) + end + name == "id" && table isa EntitySet && (property["required"] = true) + properties[name] = property + end + return properties, stored +end + +function enum_codes(name, column, class_id, schema) + if name == "edge_type" + unknown = setdiff(Set(column), Set(EDGE_TYPES)) + isempty(unknown) || + throw(ArgumentError("edge_type of '$class_id': unknown values $(sort(collect(unknown))) " * + "(allowed: $EDGE_TYPES)")) + return EDGE_TYPE_ENUM, UInt8[findfirst(==(value), EDGE_TYPES) - 1 for value in column] + end + names = sort(unique(column)) + enum_name = "$(class_id)__$name" + value_type = length(names) <= 256 ? UInt8 : UInt16 + schema["enums"][enum_name] = json_object("valueType" => uppercase(string(value_type)), + "values" => [json_object("name" => label, "value" => code - 1) + for (code, label) in enumerate(names)]) + code_of = Dict(label => code - 1 for (code, label) in enumerate(names)) + return enum_name, value_type[code_of[value] for value in column] +end + +function grid_json(grid::Grid) + entry = json_object("kind" => grid.kind, "shape" => grid.shape, "origin" => grid.origin, "spacing" => grid.spacing, + "axes" => grid.axes, "periodic" => grid.periodic, "zUp" => grid.z_up, "cells" => grid.cells) + grid.kind == "adaptive" && (entry["maxLevel"] = grid.max_level) + return entry +end + +display_attribute(name::AbstractString) = "_" * uppercase(name) + +function encode_mesh(mesh::Mesh, data_structure::DataStructure, entry, buffers::BufferBuilder, accessors) + function accessor(values, accessor_type, component_type, target; bounds = false) + buffer_view = add!(buffers, row_major(values); target = target) + item = json_object("bufferView" => buffer_view, "componentType" => component_type, + "count" => size(values, 1), "type" => accessor_type) + if bounds + item["min"] = Float64.(vec(minimum(values; dims = 1))) + item["max"] = Float64.(vec(maximum(values; dims = 1))) + end + push!(accessors, item) + return length(accessors) - 1 + end + + positions = Float32.(mesh.positions) + attributes = json_object("POSITION" => accessor(positions, "VEC3", FLOAT, ARRAY_BUFFER; bounds = true)) + tables = Dict(set_entry["name"] => set_entry["propertyTable"] for set_entry in entry["entitySets"]) + feature_ids = Any[] + for (set_index, (set_name, rows)) in enumerate(mesh.feature_rows) + haskey(tables, set_name) || + throw(ArgumentError("mesh '$(mesh.name)': no entity set '$set_name' in '$(data_structure.id)'")) + count = row_count(entity_set(data_structure, set_name)) + count >= MAX_FLOAT_FEATURE && + throw(ArgumentError("mesh '$(mesh.name)': $count entities in '$set_name', feature IDs stop at 2^24")) + any(row -> row < NO_ID || row >= count, rows) && + throw(ArgumentError("mesh '$(mesh.name)': feature rows of '$set_name' outside its $count rows")) + stored = Float32[row == NO_ID ? count : row for row in rows] # null: one past the last row + attributes["_FEATURE_ID_$(set_index - 1)"] = accessor(stored, "SCALAR", FLOAT, ARRAY_BUFFER) + feature = json_object("featureCount" => length(unique(filter(!=(NO_ID), rows))), "attribute" => set_index - 1, + "propertyTable" => tables[set_name], "label" => set_name) + any(==(NO_ID), rows) && (feature["nullFeatureId"] = count) + push!(feature_ids, feature) + end + for (name, values) in mesh.display + length(values) == size(positions, 1) || + throw(ArgumentError("mesh '$(mesh.name)': display copy '$name' has $(length(values)) values, " * + "$(size(positions, 1)) vertices")) + attributes[display_attribute(name)] = accessor(Float32.(values), "SCALAR", FLOAT, ARRAY_BUFFER) + end + primitive = json_object("attributes" => attributes, "mode" => 4) + mesh.indices === nothing || + (primitive["indices"] = accessor(UInt32.(mesh.indices), "SCALAR", UNSIGNED_INT, ELEMENT_ARRAY_BUFFER)) + isempty(feature_ids) || (primitive["extensions"] = json_object(FEATURES => json_object("featureIds" => feature_ids))) + return json_object("name" => mesh.name, "primitives" => [primitive]) +end + +# ── Reading ────────────────────────────────────────────────────────────── + +""" + from_bytes(data) -> Frame + +The frame of a GLB file's bytes. +""" +function from_bytes(data::AbstractVector{UInt8}) + document, binary = unpack(data) + extensions = get(document, "extensions", json_object()) + haskey(extensions, EXTENSION) || throw(ArgumentError("not an fspm-TF file: no $EXTENSION extension")) + scene = extensions[EXTENSION] + metadata = get(extensions, METADATA, json_object("schema" => json_object(), "propertyTables" => Any[])) + variables = get(scene, "variables", json_object()) + + function table(index, make) + description = metadata["propertyTables"][index+1] + columns, declarations = decode_table(document, binary, description, metadata["schema"], variables) + return make(String(get(description, "name", description["class"])), columns, declarations) + end + + frame_entry = get(scene, "frame", json_object()) + frame = Frame(time = Float64(get(frame_entry, "time", 0.0)), time_unit = get(frame_entry, "timeUnit", "d"), + step = Int(get(frame_entry, "step", 0)), writer = get(scene, "writer", json_object()), + vocabulary_version = get(scene, "vocabulary", nothing)) + for entry in get(scene, "dataStructures", Any[]) + origin = haskey(entry, "origin") ? reshape(Float64.(entry["origin"]), 4, 4) : identity_matrix() + data_structure = DataStructure(entry["id"]; kind = get(entry, "kind", "graph"), + platform = get(entry, "platform", nothing), model = get(entry, "model", nothing), + extras = get(entry, "extras", json_object()), origin = origin) + for item in get(entry, "entitySets", Any[]) + rank = get(item, "rank", nothing) + push!(data_structure.entity_sets, + table(item["propertyTable"], (name, columns, declarations) -> + EntitySet(name, columns, declarations, rank === nothing ? nothing : Int(rank)))) + end + for item in get(entry, "relations", Any[]) + push!(data_structure.relations, + table(item["propertyTable"], (name, columns, declarations) -> + Relation(name, columns, declarations, item["sourceSet"], item["targetSet"]))) + end + haskey(entry, "scalars") && (data_structure.scalars = table(entry["scalars"], Table)) + if haskey(entry, "grid") + grid = entry["grid"] + data_structure.grid = Grid(kind = grid["kind"], shape = Int.(grid["shape"]), + origin = Float64.(grid["origin"]), spacing = Float64.(grid["spacing"]), + axes = String.(grid["axes"]), periodic = Bool.(grid["periodic"]), + z_up = get(grid, "zUp", true), max_level = get(grid, "maxLevel", 0), + cells = grid["cells"]) + end + tables = Dict(item["propertyTable"] => item["name"] for item in get(entry, "entitySets", Any[])) + for mesh_entry in get(entry, "meshes", Any[]) + push!(data_structure.meshes, decode_mesh(document, binary, mesh_entry, tables)) + end + push!(frame.data_structures, data_structure) + end + for entry in get(scene, "mappings", Any[]) + source = (entry["source"]["dataStructure"], entry["source"]["entitySet"]) + target = (entry["target"]["dataStructure"], entry["target"]["entitySet"]) + push!(frame.mappings, table(entry["propertyTable"], (name, columns, declarations) -> + Mapping(name, columns, declarations, source, target, get(entry, "method", nothing)))) + end + other_extensions = json_object(name => value for (name, value) in extensions + if !(name in (METADATA, EXTENSION, FEATURES))) + isempty(other_extensions) || (frame.unknown["extensions"] = other_extensions) + haskey(document, "extras") && (frame.unknown["extras"] = document["extras"]) + return frame +end + +""" + read_frame(path) -> Frame + +The frame of the GLB file `path`. +""" +read_frame(path::AbstractString) = from_bytes(read(path)) + +function decode_table(document, binary, description, schema, variables) + class_id = description["class"] + properties = schema["classes"][class_id]["properties"] + count = Int(description["count"]) + columns, declarations = Columns(), Declarations() + for (name, stored) in description["properties"] + property = properties[name] + raw = view_bytes(document, binary, stored["values"]) + type = property["type"] + if type == "ENUM" + enum = schema["enums"][property["enumType"]] + codes = read_values(TYPES_OF_COMPONENTS[get(enum, "valueType", "UINT16")], raw, count) + name_of = Dict(Int(item["value"]) => String(item["name"]) for item in enum["values"]) + column = String[name_of[Int(code)] for code in codes] + elseif type == "STRING" + offset_type = TYPES_OF_COMPONENTS[get(stored, "stringOffsetType", "UINT32")] + offsets = read_values(offset_type, view_bytes(document, binary, stored["stringOffsets"]), count + 1) + column = String[String(raw[offsets[row]+1:offsets[row+1]]) for row in 1:count] + elseif type == "BOOLEAN" + column = Bool[(raw[(row-1)÷8+1] >> ((row - 1) % 8)) & 0x01 == 0x01 for row in 1:count] + elseif type == "SCALAR" + element = TYPES_OF_COMPONENTS[property["componentType"]] + vector = get(property, "array", false) + vector && !haskey(property, "count") && + throw(ArgumentError("'$class_id.$name': variable-length arrays are not part of fspm-TF 0.1")) + width = vector ? Int(property["count"]) : 1 + values = read_values(element, raw, count * width) + if element <: AbstractFloat + if haskey(property, "noData") + no_data = element(property["noData"]) + values = map(value -> value == no_data ? element(NaN) : value, values) + end + infinity = get(get(property, "extras", json_object()), "FSPM_infinity", nothing) + if infinity !== nothing + positive, negative = element(infinity["positive"]), element(infinity["negative"]) + values = map(value -> value == positive ? element(Inf) : + value == negative ? element(-Inf) : value, values) + end + end + column = vector ? permutedims(reshape(values, width, count)) : values + else + throw(ArgumentError("'$class_id.$name': property type $type is not part of fspm-TF 0.1")) + end + columns[name] = column + declared = get(get(variables, class_id, json_object()), name, nothing) + if declared !== nothing || haskey(property, "description") + declared = declared === nothing ? json_object() : declared + declarations[name] = Variable(unit = get(declared, "unit", "1"), + standard_name = get(declared, "standardName", nothing), + kind = get(declared, "kind", nothing), role = get(declared, "role", "state"), + description = get(property, "description", nothing), + minimum = get(declared, "minimum", nothing), + maximum = get(declared, "maximum", nothing), + references = String.(get(declared, "references", String[])), + doi = String.(get(declared, "doi", String[]))) + end + end + return columns, declarations +end + +function accessor_values(document, binary, index) + accessor = document["accessors"][index+1] + width = Dict("SCALAR" => 1, "VEC2" => 2, "VEC3" => 3, "VEC4" => 4)[accessor["type"]] + element = Dict(FLOAT => Float32, UNSIGNED_INT => UInt32, UNSIGNED_SHORT => UInt16, + UNSIGNED_BYTE => UInt8)[accessor["componentType"]] + raw = view_bytes(document, binary, accessor["bufferView"]) + buffer_view = document["bufferViews"][accessor["bufferView"]+1] + item_size = sizeof(element) * width + stride = get(buffer_view, "byteStride", item_size) + start = get(accessor, "byteOffset", 0) + count = Int(accessor["count"]) + values = stride == item_size ? read_values(element, raw, count * width, start) : + reduce(vcat, [read_values(element, raw, width, start + item * stride) for item in 0:count-1]; + init = element[]) + return width > 1 ? permutedims(reshape(values, width, count)) : values +end + +function decode_mesh(document, binary, mesh_entry, tables) + primitive = document["meshes"][mesh_entry["mesh"]+1]["primitives"][1] + attributes = primitive["attributes"] + positions = Float64.(accessor_values(document, binary, attributes["POSITION"])) + feature_rows = OrderedDict{String,Vector{Int64}}() + for feature in get(get(get(primitive, "extensions", json_object()), FEATURES, json_object()), "featureIds", Any[]) + rows = Int64.(accessor_values(document, binary, attributes["_FEATURE_ID_$(feature["attribute"])"])) + if haskey(feature, "nullFeatureId") + null = feature["nullFeatureId"] + rows = Int64[row == null ? NO_ID : row for row in rows] + end + feature_rows[tables[feature["propertyTable"]]] = rows + end + display, sources = OrderedDict{String,Vector{Float32}}(), Dict{String,String}() + for item in get(mesh_entry, "display", Any[]) + display[item["variable"]] = accessor_values(document, binary, attributes[item["attribute"]]) + get(item, "entitySet", nothing) === nothing || (sources[item["variable"]] = item["entitySet"]) + end + indices = haskey(primitive, "indices") ? Int64.(accessor_values(document, binary, primitive["indices"])) : nothing + return Mesh(mesh_entry["name"]; positions = positions, feature_rows = feature_rows, indices = indices, + display = display, display_sources = sources) +end diff --git a/bindings/julia/FspmTF/src/dump.jl b/bindings/julia/FspmTF/src/dump.jl new file mode 100644 index 0000000..0742b3a --- /dev/null +++ b/bindings/julia/FspmTF/src/dump.jl @@ -0,0 +1,94 @@ +# A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): +# every table's columns as lists, NaN as null, ±Inf as "Infinity" / "-Infinity", positions as their Float32 values. + +json_value(value::AbstractFloat) = + isnan(value) ? nothing : isinf(value) ? (value > 0 ? "Infinity" : "-Infinity") : Float64(value) +json_value(value) = value + +json_column(column::AbstractVector) = [json_value(value) for value in column] +json_column(column::AbstractMatrix) = [[json_value(value) for value in row] for row in eachrow(column)] + +"""The name of a column's type in the expected values: numpy's (`float64`, `int64`, `uint8`, `bool`...), or +`string`.""" +dtype_name(column::AbstractArray) = eltype(column) <: AbstractString ? "string" : lowercase(string(eltype(column))) + +function table_dict(table::AbstractTable) + entry = json_object("name" => table.name, "count" => row_count(table), + "columns" => json_object(name => json_object("dtype" => dtype_name(column), + "values" => json_column(column)) + for (name, column) in table.columns)) + declarations = json_object() + for (name, variable) in table.variables + declared = json_object("unit" => variable.unit, "role" => variable.role) + variable.standard_name === nothing || (declared["standard_name"] = variable.standard_name) + variable.kind === nothing || (declared["kind"] = variable.kind) + variable.description === nothing || (declared["description"] = variable.description) + variable.minimum === nothing || (declared["minimum"] = variable.minimum) + variable.maximum === nothing || (declared["maximum"] = variable.maximum) + isempty(variable.references) || (declared["references"] = variable.references) + isempty(variable.doi) || (declared["doi"] = variable.doi) + declarations[name] = declared + end + isempty(declarations) || (entry["variables"] = declarations) + return entry +end + +""" + frame_to_dict(frame) -> OrderedDict + +`frame` as plain JSON values, in the layout of the conformance files' expected values. +""" +function frame_to_dict(frame::Frame) + data_structures = Any[] + for data_structure in frame.data_structures + entry = json_object("id" => data_structure.id, "kind" => data_structure.kind, + "origin" => [collect(row) for row in eachrow(data_structure.origin)], + "entity_sets" => [merge!(table_dict(set), json_object("rank" => set.rank)) + for set in data_structure.entity_sets]) + data_structure.platform === nothing || (entry["platform"] = data_structure.platform) + data_structure.model === nothing || (entry["model"] = data_structure.model) + isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) + if !isempty(data_structure.relations) + entry["relations"] = [merge!(table_dict(relation), json_object("source_set" => relation.source_set, + "target_set" => relation.target_set)) + for relation in data_structure.relations] + end + data_structure.scalars === nothing || (entry["scalars"] = table_dict(data_structure.scalars)) + if data_structure.grid !== nothing + grid = data_structure.grid + entry["grid"] = json_object("kind" => grid.kind, "shape" => grid.shape, "origin" => grid.origin, + "spacing" => grid.spacing, "axes" => grid.axes, "periodic" => grid.periodic, + "z_up" => grid.z_up, "max_level" => grid.max_level, "cells" => grid.cells) + end + if !isempty(data_structure.meshes) + entry["meshes"] = map(data_structure.meshes) do mesh + item = json_object("name" => mesh.name, + "positions_float32" => json_column(Float64.(Float32.(mesh.positions))), + "feature_rows" => json_object(name => rows for (name, rows) in mesh.feature_rows)) + mesh.indices === nothing || (item["indices"] = mesh.indices) + isempty(mesh.display) || (item["display"] = json_object(name => json_column(Float64.(values)) + for (name, values) in mesh.display)) + item + end + end + push!(data_structures, entry) + end + mappings = [merge!(table_dict(mapping), json_object("source" => collect(mapping.source), + "target" => collect(mapping.target), + "method" => mapping.method)) + for mapping in frame.mappings] + return json_object("time" => frame.time, "time_unit" => frame.time_unit, "step" => frame.step, + "data_structures" => data_structures, "mappings" => mappings) +end + +""" + write_dump(path, frame) + +Write `frame` as JSON in the layout of the conformance files' expected values, for `fspm-tf compare`. +""" +function write_dump(path::AbstractString, frame::Frame) + open(path, "w") do io + JSON.json(io, frame_to_dict(frame)) + end + return path +end diff --git a/bindings/julia/FspmTF/src/glb.jl b/bindings/julia/FspmTF/src/glb.jl new file mode 100644 index 0000000..873bd1d --- /dev/null +++ b/bindings/julia/FspmTF/src/glb.jl @@ -0,0 +1,117 @@ +# The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary +# chunk padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. + +const GLB_MAGIC = 0x46546C67 # "glTF" +const GLB_VERSION = 0x00000002 +const JSON_CHUNK = 0x4E4F534A # "JSON" +const BINARY_CHUNK = 0x004E4942 # "BIN\0" +const ALIGNMENT = 8 + +padding(length::Integer) = mod(-length, ALIGNMENT) + +function write_word!(io::IO, value::UInt32) + write(io, htol(value)) +end + +""" + pack(document, binary) -> Vector{UInt8} + +The bytes of a GLB file from its JSON `document` (any value JSON.jl writes) and its `binary` buffer. +""" +function pack(document, binary::AbstractVector{UInt8}) + text = Vector{UInt8}(codeunits(JSON.json(document))) + append!(text, fill(UInt8(' '), padding(length(text)))) + binary = vcat(binary, zeros(UInt8, padding(length(binary)))) + total = 12 + 8 + length(text) + (isempty(binary) ? 0 : 8 + length(binary)) + io = IOBuffer(sizehint = total) + write_word!(io, GLB_MAGIC) + write_word!(io, GLB_VERSION) + write_word!(io, UInt32(total)) + write_word!(io, UInt32(length(text))) + write_word!(io, JSON_CHUNK) + write(io, text) + if !isempty(binary) + write_word!(io, UInt32(length(binary))) + write_word!(io, BINARY_CHUNK) + write(io, binary) + end + return take!(io) +end + +read_word(data::AbstractVector{UInt8}, offset::Integer) = ltoh(reinterpret(UInt32, data[offset+1:offset+4])[1]) + +""" + unpack(data) -> (document, binary) + +The JSON document (a `JSON.Object`, keys in their order) and the binary buffer of a GLB file's bytes. +""" +function unpack(data::AbstractVector{UInt8}) + length(data) < 20 && throw(ArgumentError("not a GLB file: too short")) + read_word(data, 0) == GLB_MAGIC || throw(ArgumentError("not a GLB file: wrong magic")) + version = read_word(data, 4) + version == GLB_VERSION || throw(ArgumentError("GLB version $(Int(version)), 2 expected")) + total = read_word(data, 8) + total == length(data) || + throw(ArgumentError("GLB header length $(Int(total)), the file holds $(length(data)) bytes")) + offset, document, binary = 12, nothing, UInt8[] + while offset < total + chunk_length, chunk_type = read_word(data, offset), read_word(data, offset + 4) + chunk = view(data, offset+9:offset+8+Int(chunk_length)) + if chunk_type == JSON_CHUNK + document = JSON.parse(String(copy(chunk))) + elseif chunk_type == BINARY_CHUNK + binary = Vector{UInt8}(chunk) + end + offset += 8 + Int(chunk_length) + end + document === nothing && throw(ArgumentError("GLB file without a JSON chunk")) + return document, binary +end + +""" + BufferBuilder() + +Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. +""" +mutable struct BufferBuilder + data::Vector{UInt8} + views::Vector{OrderedDict{String,Any}} +end +BufferBuilder() = BufferBuilder(UInt8[], OrderedDict{String,Any}[]) + +little_endian_bytes(values::AbstractArray{T}) where {T<:Union{Number,Bool}} = + Vector{UInt8}(reinterpret(UInt8, htol.(vec(collect(values))))) + +""" + add!(buffers, data; target=nothing) -> Int + +Append `data` (bytes, or an array of numbers written little-endian) to the buffer; the 0-based index of its buffer +view. +""" +function add!(buffers::BufferBuilder, data::AbstractVector{UInt8}; target = nothing) + append!(buffers.data, zeros(UInt8, padding(length(buffers.data)))) + buffer_view = OrderedDict{String,Any}("buffer" => 0, "byteOffset" => length(buffers.data), + "byteLength" => length(data)) + target === nothing || (buffer_view["target"] = target) + append!(buffers.data, data) + push!(buffers.views, buffer_view) + return length(buffers.views) - 1 +end +add!(buffers::BufferBuilder, data::AbstractArray; target = nothing) = + add!(buffers, little_endian_bytes(data); target = target) + +function view_bytes(document, binary::Vector{UInt8}, index::Integer) + buffer_view = document["bufferViews"][index+1] + get(buffer_view, "buffer", 0) == 0 || + throw(ArgumentError("fspm-TF files keep every buffer view in the GLB binary chunk")) + start = get(buffer_view, "byteOffset", 0) + return view(binary, start+1:start+buffer_view["byteLength"]) +end + +"""The `count` values of type `T` stored little-endian in `bytes` from byte `offset` (0-based).""" +function read_values(::Type{T}, bytes::AbstractVector{UInt8}, count::Integer, offset::Integer = 0) where {T} + size = sizeof(T) + length(bytes) >= offset + count * size || + throw(ArgumentError("a buffer view holds $(length(bytes)) bytes, $(offset + count * size) needed")) + return ltoh.(collect(reinterpret(T, bytes[offset+1:offset+count*size]))) +end diff --git a/bindings/julia/FspmTF/src/model.jl b/bindings/julia/FspmTF/src/model.jl new file mode 100644 index 0000000..3a52deb --- /dev/null +++ b/bindings/julia/FspmTF/src/model.jl @@ -0,0 +1,299 @@ +# The fspm-TF data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +# +# Frame one time step of a scene +# DataStructure one plant population's graph, or one environment model's grid +# EntitySet the entities of one scale (or a grid's cells): a table of columns +# Relation an edge table between entity sets (cell adjacency, a branch's bearer, ...) +# Table the data structure's scalars: a table of one row +# Grid a regular or adaptive grid's layout +# Mesh triangles, each vertex carrying the row of its entity in one or several entity sets +# Mapping source entity ↔ target entity weights between two data structures +# +# A column is a Vector (one value per row: numbers, Bool, String) or a Matrix (one row per entity, one column per +# component: a fixed-size vector per entity). + +"""The id of no entity: the `complex` or `parent` of an entity that has none, the feature row of a vertex with none.""" +const NO_ID = -1 +"""The values of `edge_type`: the root of its scale, MTG `<`, MTG `+`.""" +const EDGE_TYPES = ("none", "successor", "branch") +const ENUM_COLUMNS = ("label", "edge_type", "relation") +const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index") +const DATA_STRUCTURE_KINDS = ("graph", "regular_grid", "adaptive_grid", "table") +const ROLES = ("state", "input", "parameter", "derived", "setting") +const COLUMN_TYPES = (Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64) + +""" + Variable(; unit="1", standard_name=nothing, kind=nothing, role="state", description=nothing, + minimum=nothing, maximum=nothing, references=String[], doi=String[]) + +The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind +(`extensive`, `intensive`, ...: how it maps between scales and grids), its role and its provenance. +""" +Base.@kwdef mutable struct Variable + unit::String = "1" + standard_name::Union{Nothing,String} = nothing + kind::Union{Nothing,String} = nothing + role::String = "state" + description::Union{Nothing,String} = nothing + minimum::Union{Nothing,Real} = nothing + maximum::Union{Nothing,Real} = nothing + references::Vector{String} = String[] + doi::Vector{String} = String[] +end + +function Base.:(==)(left::Variable, right::Variable) + all(isequal(getfield(left, name), getfield(right, name)) for name in fieldnames(Variable)) +end + +const Columns = OrderedDict{String,AbstractArray} +const Declarations = OrderedDict{String,Variable} +const JSONObject = AbstractDict{String,Any} + +"""The tables of a frame: `Table` (scalars), `EntitySet`, `Relation`, `Mapping`. Each has `name`, `columns` and +`variables`.""" +abstract type AbstractTable end + +""" + Table(name; columns=Columns(), variables=Declarations()) + +Columns of equal length and the declarations of its variables: a data structure's scalars (one row). +""" +mutable struct Table <: AbstractTable + name::String + columns::Columns + variables::Declarations +end + +""" + EntitySet(name; columns, variables, rank=nothing) + +The entities of one scale of a graph (`rank`: the scale's rank, coarser first), or a grid's cells (`rank` nothing). +Reserved columns: `id` (Int64, stable across frames, never reused), `label` (String), `complex` (the id of the +containing entity at a coarser scale, `NO_ID` if none), `parent` (the predecessor's id at the same scale, `NO_ID` if +none), `edge_type` (`"none"`, `"successor"`, `"branch"`); an adaptive grid's cells: `level` and `index`. +""" +mutable struct EntitySet <: AbstractTable + name::String + columns::Columns + variables::Declarations + rank::Union{Nothing,Int} +end + +""" + Relation(name; columns, variables, source_set, target_set) + +An edge table: `source` and `target` entity ids (in the entity sets `source_set` and `target_set`), an optional `id` +per edge, an optional `relation` label per edge, and edge variables. +""" +mutable struct Relation <: AbstractTable + name::String + columns::Columns + variables::Declarations + source_set::String + target_set::String +end + +""" + Mapping(name; columns, variables, source, target, method=nothing) + +Weights between the entities of two data structures: `source` and `target` ids, `weight`. `source` and `target`: +(data structure id, entity set name). +""" +mutable struct Mapping <: AbstractTable + name::String + columns::Columns + variables::Declarations + source::Tuple{String,String} + target::Tuple{String,String} + method::Union{Nothing,String} +end + +# The columns whose meaning the specification fixes, per kind of table +const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index") +const RELATION_RESERVED = ("id", "source", "target", "relation") +const MAPPING_RESERVED = ("source", "target", "weight") +reserved(::Table) = () +reserved(::EntitySet) = ENTITY_SET_RESERVED +reserved(::Relation) = RELATION_RESERVED +reserved(::Mapping) = MAPPING_RESERVED + +as_column(name::AbstractString, values::AbstractVector{<:AbstractString}) = Vector{String}(values) +as_column(name::AbstractString, values::AbstractVector{Bool}) = Vector{Bool}(values) +as_column(name::AbstractString, values::AbstractVector{T}) where {T<:Number} = Vector{T}(values) +as_column(name::AbstractString, values::AbstractMatrix{T}) where {T<:Number} = Matrix{T}(values) +as_column(name::AbstractString, values::Number) = [values] +as_column(name::AbstractString, values::AbstractString) = [String(values)] +function as_column(name::AbstractString, values::AbstractVector) + isempty(values) && return Float64[] + if all(value -> value isa AbstractString, values) + return Vector{String}(values) + end + promoted = [values...] # the narrowest common type + eltype(promoted) <: Union{Number,Bool} && return promoted + throw(ArgumentError("column '$name': numbers, booleans or strings per row (got $(eltype(promoted)))")) +end +as_column(name::AbstractString, values) = + throw(ArgumentError("column '$name': a vector, or a matrix of one row per entity (got $(typeof(values)))")) + +function prepare_columns(table_name::AbstractString, columns, reserved_names) + prepared = Columns() + for (name, values) in pairs(columns) + name = String(name) + column = as_column(name, values) + if name in ID_COLUMNS && name in reserved_names && eltype(column) <: Integer && !(eltype(column) <: Bool) + column = convert(Array{Int64}, column) # ids are Int64, whatever the platform's integers + end + prepared[name] = column + end + counts = Dict(name => size(column, 1) for (name, column) in prepared) + length(Set(values(counts))) > 1 && + throw(ArgumentError("table '$table_name': columns of different lengths $counts")) + return prepared +end + +prepare_variables(variables) = Declarations(String(name) => variable for (name, variable) in pairs(variables)) + +Table(name::AbstractString; columns = Columns(), variables = Declarations()) = + Table(String(name), prepare_columns(name, columns, ()), prepare_variables(variables)) +EntitySet(name::AbstractString; columns = Columns(), variables = Declarations(), rank = nothing) = + EntitySet(String(name), prepare_columns(name, columns, ENTITY_SET_RESERVED), prepare_variables(variables), rank) +Relation(name::AbstractString; columns = Columns(), variables = Declarations(), source_set = "", target_set = "") = + Relation(String(name), prepare_columns(name, columns, RELATION_RESERVED), prepare_variables(variables), + String(source_set), String(target_set)) +Mapping(name::AbstractString; columns = Columns(), variables = Declarations(), source = ("", ""), + target = ("", ""), method = nothing) = + Mapping(String(name), prepare_columns(name, columns, MAPPING_RESERVED), prepare_variables(variables), + (String(source[1]), String(source[2])), (String(target[1]), String(target[2])), method) + +"""The number of rows of a table.""" +row_count(table::AbstractTable) = isempty(table.columns) ? 0 : size(first(values(table.columns)), 1) + +"""The names of a table's variable columns (its columns but the reserved ones).""" +variable_names(table::AbstractTable) = [name for name in keys(table.columns) if !(name in reserved(table))] + +"""The declaration of a column (an undeclared one: unit "1", role "state").""" +declaration(table::AbstractTable, name::AbstractString) = get(table.variables, name, Variable()) + +"""The ids of an entity set's entities: its `id` column, or a grid's row numbers (from 0) when it has none.""" +entity_ids(entity_set::EntitySet) = + haskey(entity_set.columns, "id") ? entity_set.columns["id"] : collect(Int64, 0:row_count(entity_set)-1) + +""" + Grid(; kind="regular", shape, origin, spacing, axes=["x","y","z"], periodic, z_up=true, max_level=0, + cells="Cell") + +A grid's layout. Regular: `shape` cells of size `spacing` from `origin`, along `axes`, `periodic` per axis, z +pointing up or down (`z_up`). Adaptive: the same for its base level, refined up to `max_level`, its leaf cells (an +entity set with `level` and `index` columns) explicit. `cells`: the name of the cells' entity set. +""" +Base.@kwdef mutable struct Grid + kind::String = "regular" + shape::Vector{Int} = Int[] + origin::Vector{Float64} = Float64[] + spacing::Vector{Float64} = Float64[] + axes::Vector{String} = ["x", "y", "z"] + periodic::Vector{Bool} = Bool[] + z_up::Bool = true + max_level::Int = 0 + cells::String = "Cell" +end + +""" + Mesh(name; positions, feature_rows=OrderedDict(), indices=nothing, display=OrderedDict(), + display_sources=Dict()) + +Triangles: `positions` (V×3, in the data structure's frame), three consecutive vertices per triangle, or `indices` +(3T, 0-based) into them; `feature_rows`: entity set name => the row (0-based) of each vertex's entity in that set, +`NO_ID` for none; `display`: variable name => its Float32 copy per vertex, from the entity set `display_sources[name]`. +""" +mutable struct Mesh + name::String + positions::Matrix{Float64} + feature_rows::OrderedDict{String,Vector{Int64}} + indices::Union{Nothing,Vector{Int64}} + display::OrderedDict{String,Vector{Float32}} + display_sources::Dict{String,String} +end + +function Mesh(name::AbstractString; positions, feature_rows = OrderedDict{String,Vector{Int64}}(), + indices = nothing, display = OrderedDict{String,Vector{Float32}}(), + display_sources = Dict{String,String}()) + positions = Matrix{Float64}(positions isa AbstractVector ? permutedims(reshape(positions, 3, :)) : positions) + size(positions, 2) == 3 || throw(ArgumentError("mesh '$name': positions are V×3")) + rows = OrderedDict{String,Vector{Int64}}(String(set) => Vector{Int64}(vec(values)) for (set, values) in pairs(feature_rows)) + for (set, values) in rows + length(values) == size(positions, 1) || + throw(ArgumentError("mesh '$name': $(length(values)) feature rows for '$set', $(size(positions, 1)) vertices")) + end + Mesh(String(name), positions, rows, indices === nothing ? nothing : Vector{Int64}(vec(indices)), + OrderedDict{String,Vector{Float32}}(String(variable) => Vector{Float32}(vec(values)) + for (variable, values) in pairs(display)), + Dict{String,String}(String(variable) => String(set) for (variable, set) in pairs(display_sources))) +end + +""" + DataStructure(id; kind="graph", entity_sets=EntitySet[], relations=Relation[], scalars=nothing, grid=nothing, + meshes=Mesh[], origin=identity, platform=nothing, model=nothing, extras=OrderedDict()) + +One plant population's graph, or one environment model's grid. `origin`: its frame in the scene (a 4×4 matrix, +`origin[row, column]`, or 16 numbers row by row). `extras`: JSON values for the writing platform (e.g. what it needs to rebuild its own data +structure); other readers keep them without interpreting them. +""" +mutable struct DataStructure + id::String + kind::String + entity_sets::Vector{EntitySet} + relations::Vector{Relation} + scalars::Union{Nothing,Table} + grid::Union{Nothing,Grid} + meshes::Vector{Mesh} + origin::Matrix{Float64} + platform::Union{Nothing,String} + model::Union{Nothing,String} + extras::AbstractDict{String,Any} +end + +identity_matrix() = Matrix{Float64}([i == j for i in 1:4, j in 1:4]) + +function DataStructure(id::AbstractString; kind = "graph", entity_sets = EntitySet[], relations = Relation[], + scalars = nothing, grid = nothing, meshes = Mesh[], origin = identity_matrix(), + platform = nothing, model = nothing, extras = OrderedDict{String,Any}()) + origin = Matrix{Float64}(origin isa AbstractVector ? permutedims(reshape(origin, 4, 4)) : origin) + size(origin) == (4, 4) || throw(ArgumentError("data structure '$id': the origin is a 4×4 matrix")) + DataStructure(String(id), String(kind), collect(EntitySet, entity_sets), collect(Relation, relations), scalars, + grid, collect(Mesh, meshes), origin, platform, model, extras) +end + +"""The entity set `name` of a data structure.""" +function entity_set(data_structure::DataStructure, name::AbstractString) + for candidate in data_structure.entity_sets + candidate.name == name && return candidate + end + throw(KeyError("data structure '$(data_structure.id)' has no entity set '$name'")) +end + +""" + Frame(; data_structures=DataStructure[], mappings=Mapping[], time=0.0, time_unit="d", step=0, + writer=OrderedDict(), vocabulary_version=nothing, unknown=OrderedDict()) + +A scene at one time step (`time` in `time_unit`, `step`), or a parameter set. `unknown`: the content of a read file +this package does not interpret (other root extensions, the root's `extras`), written back as it was. +""" +Base.@kwdef mutable struct Frame + data_structures::Vector{DataStructure} = DataStructure[] + mappings::Vector{Mapping} = Mapping[] + time::Float64 = 0.0 + time_unit::String = "d" + step::Int = 0 + writer::AbstractDict{String,Any} = OrderedDict{String,Any}() + vocabulary_version::Union{Nothing,String} = nothing + unknown::AbstractDict{String,Any} = OrderedDict{String,Any}() +end + +"""The data structure `id` of a frame.""" +function data_structure(frame::Frame, id::AbstractString) + for candidate in frame.data_structures + candidate.id == id && return candidate + end + throw(KeyError("the frame has no data structure '$id'")) +end diff --git a/bindings/julia/FspmTF/src/sequence.jl b/bindings/julia/FspmTF/src/sequence.jl new file mode 100644 index 0000000..691e9b7 --- /dev/null +++ b/bindings/julia/FspmTF/src/sequence.jl @@ -0,0 +1,63 @@ +# Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. + +const INDEX_FILE = "scene.fspm.json" + +""" + write_index(path, entries; time_unit="d") + +Write the index of a sequence: `entries`, one `(time, step, file)` per frame (a NamedTuple or a dictionary), in time +order. +""" +function write_index(path::AbstractString, entries; time_unit::AbstractString = "d") + field(entry, name) = entry isa NamedTuple ? getproperty(entry, name) : entry[string(name)] + frames = [json_object("time" => Float64(field(entry, :time)), "step" => Int(field(entry, :step)), + "file" => String(field(entry, :file))) for entry in entries] + open(path, "w") do io + JSON.json(io, json_object("fspmIndex" => SPECIFICATION_VERSION, "timeUnit" => time_unit, "frames" => frames); + pretty = 1) + write(io, "\n") + end + return path +end + +""" + write_sequence(directory, frames; name="frame") -> String + +Write `frames` to `directory`, one GLB each (`name_000000.glb`, ...), and their index; the index's path. +""" +function write_sequence(directory::AbstractString, frames; name::AbstractString = "frame") + mkpath(directory) + entries, time_units = NamedTuple[], Set{String}() + for (number, frame) in enumerate(frames) + file = string(name, "_", lpad(number - 1, 6, '0'), ".glb") + write_frame(joinpath(directory, file), frame) + push!(entries, (time = frame.time, step = frame.step, file = file)) + push!(time_units, frame.time_unit) + end + length(time_units) > 1 && + throw(ArgumentError("the frames' times are in several units: $(sort(collect(time_units)))")) + path = joinpath(directory, INDEX_FILE) + write_index(path, entries; time_unit = isempty(time_units) ? "d" : first(time_units)) + return path +end + +""" + read_index(path) -> JSON.Object + +The index of a sequence: its frames' times, steps and files, each frame with the absolute `"path"` of its file. +""" +function read_index(path::AbstractString) + index = JSON.parse(read(path, String)) + directory = dirname(abspath(path)) + for entry in get(index, "frames", Any[]) + entry["path"] = joinpath(directory, entry["file"]) + end + return index +end + +""" + read_sequence(path) + +The frames of the sequence whose index is `path`, read one after the other as they are iterated. +""" +read_sequence(path::AbstractString) = (read_frame(entry["path"]) for entry in read_index(path)["frames"]) diff --git a/bindings/julia/FspmTF/test/Project.toml b/bindings/julia/FspmTF/test/Project.toml new file mode 100644 index 0000000..5e2d591 --- /dev/null +++ b/bindings/julia/FspmTF/test/Project.toml @@ -0,0 +1,8 @@ +[deps] +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +JSON = "1" +MultiScaleTreeGraph = "0.16" diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl new file mode 100644 index 0000000..2156734 --- /dev/null +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -0,0 +1,203 @@ +using FspmTF +using JSON +using MultiScaleTreeGraph +using Test + +const CONFORMANCE = normpath(joinpath(@__DIR__, "..", "..", "..", "..", "conformance")) +const MTG_FILES = joinpath(pkgdir(MultiScaleTreeGraph), "test", "files") + +conformance_files() = sort([relpath(joinpath(folder, name), CONFORMANCE) + for (folder, _, files) in walkdir(CONFORMANCE) for name in files if endswith(name, ".glb")]) + +"""The differences between two JSON values, as `fspm-tf compare` finds them: objects by key, arrays in order, +numbers exactly (1 and 1.0 equal).""" +function differences(actual, expected, path = "\$") + if expected isa AbstractDict + actual isa AbstractDict || return ["$path: an object expected"] + found = ["$path: missing key '$key'" for key in keys(expected) if !haskey(actual, key)] + append!(found, ["$path: unexpected key '$key'" for key in keys(actual) if !haskey(expected, key)]) + for key in keys(expected) + haskey(actual, key) && append!(found, differences(actual[key], expected[key], "$path.$key")) + end + return found + elseif expected isa AbstractVector + actual isa AbstractVector || return ["$path: an array expected"] + length(actual) == length(expected) || return ["$path: $(length(actual)) items, $(length(expected)) expected"] + return reduce(vcat, [differences(left, right, "$path[$index]") + for (index, (left, right)) in enumerate(zip(actual, expected))]; init = String[]) + elseif expected isa Number && !(expected isa Bool) + actual isa Number && !(actual isa Bool) && Float64(actual) == Float64(expected) && return String[] + return ["$path: $actual, $expected expected"] + end + isequal(actual, expected) && typeof(actual) == typeof(expected) && return String[] + return ["$path: $(repr(actual)), $(repr(expected)) expected"] +end + +as_json(frame) = JSON.parse(JSON.json(frame_to_dict(frame))) +expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] * ".expected.json"), String)) + +@testset "FspmTF" begin + @testset "GLB container: the conformance files' JSON and binary chunks repacked as they were" begin + for relative in conformance_files() + data = read(joinpath(CONFORMANCE, relative)) + document, binary = FspmTF.unpack(data) + again, binary_again = FspmTF.unpack(FspmTF.pack(document, binary)) + @test binary_again == binary + @test isempty(differences(again, document)) + @test length(binary) % 8 == 0 + end + @test_throws ArgumentError FspmTF.unpack(UInt8[1, 2, 3]) + end + + @testset "Reading: every conformance file as its expected values ($relative)" for relative in conformance_files() + frame = read_frame(joinpath(CONFORMANCE, relative)) + @test isempty(differences(as_json(frame), expected_of(relative))) + end + + @testset "Writing: what is written reads back as the expected values ($relative)" for relative in conformance_files() + frame = from_bytes(to_bytes(read_frame(joinpath(CONFORMANCE, relative)))) + @test isempty(differences(as_json(frame), expected_of(relative))) + end + + @testset "Columns of every type, NaN, infinities, vectors, text" begin + columns = FspmTF.Columns("id" => Int32[0, 1, 2], "label" => ["Leaf", "Internode", "Leaf"], + "parent" => [-1, 0, 1], "edge_type" => ["none", "successor", "branch"], + "with_nan" => [1.0, NaN, -2.5], "with_infinities" => Float32[Inf, 1, -Inf], + "flag" => [true, false, true], "name" => ["é", "", "a b"], + "position" => [1.0 2.0 3.0; 4.0 5.0 6.0; 7.0 8.0 9.0]) + for T in FspmTF.COLUMN_TYPES + columns["column_$(lowercase(string(T)))"] = T[1, 2, 3] + end + frame = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = columns, rank = 1, + variables = Dict("with_nan" => Variable(unit = "m", standard_name = "length", kind = "extensive", + description = "a length", minimum = 0, + references = ["a reference"])))], + origin = [1.0 0 0 0.5; 0 1 0 0; 0 0 1 -0.1; 0 0 0 1], extras = Dict{String,Any}("platform" => [1, 2]))], + time = 2.5, time_unit = "h", step = 3) + read_back = from_bytes(to_bytes(frame)) + organ = entity_set(data_structure(read_back, "plants"), "Organ") + @test eltype(organ.columns["id"]) == Int64 # ids are Int64 + @test isequal(organ.columns["with_nan"], [1.0, NaN, -2.5]) + @test organ.columns["with_infinities"] == Float32[Inf, 1, -Inf] + @test organ.columns["flag"] == [true, false, true] + @test organ.columns["name"] == ["é", "", "a b"] + @test organ.columns["position"] == [1.0 2.0 3.0; 4.0 5.0 6.0; 7.0 8.0 9.0] + @test organ.columns["label"] == ["Leaf", "Internode", "Leaf"] + for T in FspmTF.COLUMN_TYPES + @test organ.columns["column_$(lowercase(string(T)))"] == T[1, 2, 3] + @test eltype(organ.columns["column_$(lowercase(string(T)))"]) == T + end + @test organ.variables["with_nan"] == frame.data_structures[1].entity_sets[1].variables["with_nan"] + @test data_structure(read_back, "plants").origin == frame.data_structures[1].origin + @test data_structure(read_back, "plants").extras["platform"] == [1, 2] + @test (read_back.time, read_back.time_unit, read_back.step) == (2.5, "h", 3) + end + + @testset "Meshes: positions, feature rows (none included), indices, display copies" begin + set = EntitySet("SubOrgan"; columns = Dict("id" => [10, 11]), rank = 1) + mesh = Mesh("roots"; positions = [0.0 0 0; 1 0 0; 0 1 0; 0 0 1], feature_rows = Dict("SubOrgan" => [0, 1, 1, -1]), + indices = [0, 1, 2, 0, 2, 3], display = Dict("radius" => Float32[1, 2, 2, 0]), + display_sources = Dict("radius" => "SubOrgan")) + frame = Frame(data_structures = [DataStructure("roots"; entity_sets = [set], meshes = [mesh])]) + read_back = from_bytes(to_bytes(frame)).data_structures[1].meshes[1] + @test read_back.positions == mesh.positions + @test read_back.feature_rows == mesh.feature_rows + @test read_back.indices == mesh.indices + @test read_back.display == mesh.display && read_back.display_sources == mesh.display_sources + end + + @testset "What the specification forbids is refused" begin + bad_column = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0], "1st" => [1.0]))])]) + @test_throws ArgumentError to_bytes(bad_column) + @test_throws ArgumentError to_bytes(Frame(data_structures = [DataStructure("bad id")])) + bad_edge = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0], "parent" => [-1], "edge_type" => ["root"]))])]) + @test_throws ArgumentError to_bytes(bad_edge) + both = Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Organ"; columns = Dict("id" => [0, 1], "x" => [NaN, -floatmax(Float64)]))])]) + @test_throws ArgumentError to_bytes(both) + @test_throws ArgumentError EntitySet("Organ"; columns = Dict("id" => [0, 1], "x" => [1.0])) + end + + @testset "Sequences: frames and their index" begin + folder = mktempdir() + frames = [Frame(data_structures = [DataStructure("plants"; entity_sets = [ + EntitySet("Plant"; columns = Dict("id" => [0], "biomass" => [Float64(step)]), rank = 1)])], + time = step / 24, time_unit = "d", step = step) for step in 0:2] + index = write_sequence(folder, frames) + entries = read_index(index)["frames"] + @test [entry["step"] for entry in entries] == [0, 1, 2] + @test [frame.data_structures[1].entity_sets[1].columns["biomass"][1] for frame in read_sequence(index)] == + [0.0, 1.0, 2.0] + read_back = read_index(joinpath(CONFORMANCE, "growing_sequence", "scene.fspm.json")) + @test all(isfile(entry["path"]) for entry in read_back["frames"]) + end + + @testset "MultiScaleTreeGraph: $name, exported, written, read and rebuilt as it was" for name in + ("simple_plant.mtg", "palm.mtg", "simple_plant-P1U1.mtg") + mtg = read_mtg(joinpath(MTG_FILES, name)) + variables = Dict("Length" => Variable(unit = "m", standard_name = "length", kind = "extensive")) + data_structure_of_mtg = from_mtg(mtg; id = "plant", variables = variables) + frame = from_bytes(to_bytes(Frame(data_structures = [data_structure_of_mtg]))) + rebuilt = to_mtg(frame.data_structures[1]) + nodes, rebuilt_nodes = traverse(mtg, node -> node), traverse(rebuilt, node -> node) + @test length(nodes) == length(rebuilt_nodes) + for (node, rebuilt_node) in zip(nodes, rebuilt_nodes) + @test node_id(node) == node_id(rebuilt_node) + @test (isroot(node) ? nothing : node_id(parent(node))) == + (isroot(rebuilt_node) ? nothing : node_id(parent(rebuilt_node))) + @test node_mtg(node) == node_mtg(rebuilt_node) + @test [node_id(child) for child in children(node)] == [node_id(child) for child in children(rebuilt_node)] + attributes = Dict(pairs(node_attributes(node))) + rebuilt_attributes = Dict(pairs(node_attributes(rebuilt_node))) + if isroot(node) + for key in (:symbols, :scales) + @test string.(attributes[key]) == string.(rebuilt_attributes[key]) + end + description, rebuilt_description = attributes[:description], rebuilt_attributes[:description] + if description === nothing + @test rebuilt_description === nothing + else + @test names(description) == names(rebuilt_description) + @test all(isequal(getproperty(description, column), getproperty(rebuilt_description, column)) + for column in names(description)) + end + filter!(item -> !(item.first in (:symbols, :scales, :description)), attributes) + filter!(item -> !(item.first in (:symbols, :scales, :description)), rebuilt_attributes) + end + @test isequal(attributes, rebuilt_attributes) + end + end + + @testset "MultiScaleTreeGraph: the bearer of a branch at another scale" begin + scene = Node(1, MutableNodeMTG(:/, :Scene, 0, 0), Dict{Symbol,Any}()) + plant = Node(2, scene, MutableNodeMTG(:/, :Plant, 1, 1), Dict{Symbol,Any}()) + axis = Node(3, plant, MutableNodeMTG(:/, :Axis, 1, 2), Dict{Symbol,Any}()) + internode = Node(4, axis, MutableNodeMTG(:/, :Internode, 1, 3), Dict{Symbol,Any}(:length => 0.1)) + branch = Node(5, internode, MutableNodeMTG(:+, :Axis, 2, 2), Dict{Symbol,Any}()) # borne by an internode + Node(6, branch, MutableNodeMTG(:/, :Internode, 1, 3), Dict{Symbol,Any}(:length => 0.2)) + written = from_mtg(scene; scales = Dict(1 => "Plant", 2 => "Axis", 3 => "SubOrgan")) + @test only(written.relations).name == "bearer" + @test only(written.relations).columns["source"] == [5] && only(written.relations).columns["target"] == [4] + axes = entity_set(written, "Axis") + @test axes.columns["parent"] == [-1, -1] && axes.columns["complex"] == [2, 2] + rebuilt = to_mtg(from_bytes(to_bytes(Frame(data_structures = [written]))).data_structures[1]) + rebuilt_branch = get_node(rebuilt, 5) + @test node_id(parent(rebuilt_branch)) == 4 && link(rebuilt_branch) == :+ + @test attribute(get_node(rebuilt, 6), :length) == 0.2 + end + + @testset "MultiScaleTreeGraph: another platform's graph (the conformance root system) as an MTG" begin + root_system = read_frame(joinpath(CONFORMANCE, "root_system.glb")).data_structures[1] + mtg = to_mtg(root_system) + nodes = traverse(mtg, node -> node) + @test length(nodes) == sum(row_count(set) for set in root_system.entity_sets) + 1 # + the scene + segments = entity_set(root_system, "SubOrgan") + for (id, parent_id) in zip(segments.columns["id"], segments.columns["parent"]) + parent_id == NO_ID || @test node_id(parent(get_node(mtg, Int(id)))) == parent_id + end + @test attribute(get_node(mtg, Int(segments.columns["id"][1])), :length) == segments.columns["length"][1] + end +end diff --git a/bindings/julia/README.md b/bindings/julia/README.md new file mode 100644 index 0000000..3c473da --- /dev/null +++ b/bindings/julia/README.md @@ -0,0 +1,13 @@ +# FspmTF.jl + +The Julia codec of fspm-TF: reading and writing frames and sequences (spec 0.1), and, as a package extension, a +MultiScaleTreeGraph.jl MTG to and from a `graph` data structure (`from_mtg`, `to_mtg`). + +- `FspmTF/src/`: the GLB container (`glb.jl`), the data model (`model.jl`), reading and writing (`codec.jl`), + sequences (`sequence.jl`), the conformance dump (`dump.jl`). +- `FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl`: the MTG adapter, loaded with MultiScaleTreeGraph. +- `FspmTF/test/`: the tests, the conformance files included. +- `conformance_run.jl`: the codec's dumps of the conformance files and its written scenes, for + `.github/actions/conformance`. + +Dependencies: JSON.jl and OrderedCollections.jl. See [the documentation](../../doc/julia.md). diff --git a/bindings/julia/conformance_run.jl b/bindings/julia/conformance_run.jl new file mode 100644 index 0000000..c834af8 --- /dev/null +++ b/bindings/julia/conformance_run.jl @@ -0,0 +1,28 @@ +# The Julia codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump +# it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). +# Then .github/actions/conformance compares the dumps with the expected values and validates the written files. +# +# julia --project=bindings/julia/FspmTF bindings/julia/conformance_run.jl CONFORMANCE OUT + +using FspmTF + +function main(conformance::AbstractString, out::AbstractString) + for (folder, _, files) in walkdir(conformance) + for name in sort(files) + endswith(name, ".glb") || continue + relative = relpath(joinpath(folder, name), conformance) + frame = read_frame(joinpath(conformance, relative)) + dump = joinpath(out, "dumps", relative[1:end-length(".glb")] * ".json") + written = joinpath(out, "written", relative) + mkpath(dirname(dump)) + mkpath(dirname(written)) + write_dump(dump, frame) + write_frame(written, frame) # written again from what was read + end + end + index = joinpath(conformance, "growing_sequence", "scene.fspm.json") + isfile(index) && cp(index, joinpath(out, "written", "growing_sequence", "scene.fspm.json"); force = true) + return 0 +end + +main(ARGS[1], ARGS[2]) diff --git a/doc/index.md b/doc/index.md index beb2083..e814dbf 100644 --- a/doc/index.md +++ b/doc/index.md @@ -33,6 +33,7 @@ fspm-TF in five minutes Viewing and checking files Reading and writing in Python A notebook: fspm-TF in Python +Reading and writing in Julia ``` ## Implementing fspm-TF diff --git a/doc/julia.md b/doc/julia.md new file mode 100644 index 0000000..90fb15c --- /dev/null +++ b/doc/julia.md @@ -0,0 +1,92 @@ +# Reading and writing in Julia + +FspmTF.jl, in `bindings/julia/FspmTF/`, reads and writes fspm-TF files in Julia (1.10 or later). It passes the +conformance check like the Python codec. With MultiScaleTreeGraph.jl loaded, it also turns an MTG into a data +structure, and back. This is the MTG library of VirtualPlantLab's PlantGeom and of PlantSimEngine. + +## Installing + +FspmTF.jl is not registered yet. Add it from the repository's subfolder: + +```julia +using Pkg +Pkg.add(url = "https://github.com/openalea/fspm-TF", subdir = "bindings/julia/FspmTF") +``` + +## An MTG, written and read back + +```julia +using FspmTF, MultiScaleTreeGraph + +mtg = read_mtg("plant.mtg") +plants = from_mtg(mtg; id = "plant", + scales = Dict(1 => "Plant", 2 => "Axis", 3 => "Organ"), # entity set names, per scale + variables = Dict("Length" => Variable(unit = "cm", standard_name = "length", + kind = "extensive"))) +write_frame("plant.glb", Frame(data_structures = [plants], time = 0.0, time_unit = "d")) + +frame = read_frame("plant.glb") +same_mtg = to_mtg(frame.data_structures[1]) # the same ids, symbols, links, attributes and order +``` + +`to_mtg` also reads a graph another platform wrote, e.g. a root system simulated by metafspm. Its MTG has one scale +per entity set and its variables as attributes, under a `Scene` root that holds the scalars. + +| fspm-TF | MultiScaleTreeGraph | +|---|---| +| one entity set per scale ≥ 1, `rank` the scale | the nodes of that scale | +| `id`, `label` | `node_id`, `symbol` | +| `complex` | the first ancestor at a smaller scale | +| `parent`, `edge_type` | the parent, at the same scale, by `<` (successor) or `+` (branch) | +| relation `bearer` | the parent otherwise, e.g. an axis borne by an internode | +| `mtg_index` | the node's `index` | +| variables | the attributes, declared by `variables` (MTG.jl has no units) | +| scalars | the attributes of the scale 0 root (`Scene`) | + +Some MTG content has no place in fspm-TF 0.1: missing attributes, dates, the MTG's classes and description, and +the order of children. It is kept in the data structure's `extras["MultiScaleTreeGraph"]`, so that `to_mtg` +rebuilds the MTG exactly. Other readers see the missing values as NaN, `false`, 0 or "". + +## The format's own tables + +```julia +using FspmTF + +frame = read_frame("scene.glb") +frame.time, frame.time_unit, frame.step + +roots = data_structure(frame, "RootPopulation") +segments = entity_set(roots, "SubOrgan") +segments.columns["length"] # one value per segment +segments.variables["length"].unit # its declared unit, e.g. "m" +entity_ids(segments) + +for frame in read_sequence("outputs/frames/scene.fspm.json") # a sequence, frame by frame + println(frame.step) +end +``` + +Writing builds the same objects: + +```julia +segments = EntitySet("SubOrgan"; rank = 6, + columns = Dict("id" => [10, 11], "parent" => [-1, 10], + "edge_type" => ["none", "successor"], "length" => [0.01, 0.012]), + variables = Dict("length" => Variable(unit = "m", standard_name = "length", + kind = "extensive"))) +write_frame("roots.glb", Frame(data_structures = [DataStructure("roots"; entity_sets = [segments])])) +write_sequence("frames", frames) # one GLB per frame, and scene.fspm.json +``` + +A column is a `Vector` (numbers, `Bool`, `String`) or a `Matrix` (one row per entity: a fixed-size vector per +entity). NaN and infinities are written with the specification's stand-ins and read back as they were. + +## Checking it + +```bash +julia --project=bindings/julia/FspmTF -e 'using Pkg; Pkg.test()' +julia --project=bindings/julia/FspmTF bindings/julia/conformance_run.jl conformance out +fspm-tf compare-all out/dumps conformance && fspm-tf validate out/written/*.glb +``` + +The workflow `.github/workflows/julia-codec.yml` runs both, on Julia 1.10 and the latest release. diff --git a/doc/platforms.md b/doc/platforms.md index ae671e0..b570ef5 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -28,7 +28,7 @@ This is a few hundred lines in any language. Existing libraries help: - **C++:** cesium-native reads and writes `EXT_structural_metadata`. - **Java:** JglTF has its JSON classes. -- **Julia:** a small GLB reader is enough; `GLTF.jl` handles the JSON. +- **Julia:** done, `bindings/julia/FspmTF/` ([FspmTF.jl](julia.md)): a hand-written GLB container and JSON.jl. Each codec goes in its own folder, with its usual build files and tests: diff --git a/doc/python.md b/doc/python.md index c4d9190..d0b06d1 100644 --- a/doc/python.md +++ b/doc/python.md @@ -10,7 +10,8 @@ from openalea.metafspm.io.fspm_tf import export_scene, import_frame export_scene("scene.glb", scene) # the scene now data_structures = import_frame("scene.glb") # metafspm's DataStructures, rebuilt bit for bit -scene = Scene(planting, ..., output_dirpath="outputs", frame_period=24) # a frame every 24 steps +recorder = SceneRecorder("outputs", frame_period=24) # openalea.metafspm.scene.recorder: a frame every 24 steps +scene = Scene(planting, ..., recorder=recorder) ``` See [metafspm's documentation](https://github.com/openalea/metafspm) for the options. diff --git a/implementations.md b/implementations.md index 362268a..9fc9d94 100644 --- a/implementations.md +++ b/implementations.md @@ -6,9 +6,12 @@ | openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.1 | | CPlantBox | C++ / Python | planned (phase P4) | | | | | GroIMP | Java | | | | | -| VirtualPlantLab | Julia | | | | | +| FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.1 | +| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.1 | +| VirtualPlantLab (PlantGraphs) | Julia | planned (FspmTF.jl extensions: PlantGeom meshes, PlantGraphs) | | | | Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters -(a platform's model to and from the codec's tables) in the platforms' repositories. An implementation is listed with +(a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional +extension when the platform's library is a plain dependency (Julia's package extensions). An implementation is listed with the version it passes: it reads every conformance file as expected, and writes files that the validator accepts (specification, §9). From f7e1d800152815ac416b43ddda387a7e5dcb4899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 12:10:05 +0100 Subject: [PATCH 03/39] Python codec: the buffer's byteLength counts the binary chunk's padding to 8 bytes (glTF-Validator warned of more than 3 bytes beyond it); the conformance files are unchanged --- bindings/python/src/openalea/fspm_tf/codec.py | 2 ++ bindings/python/test/test_codec.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/fspm_tf/codec.py index 84ef88b..4badc61 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/fspm_tf/codec.py @@ -147,6 +147,8 @@ def add_table(table: Table, class_id: str) -> int: document["meshes"], document["accessors"] = meshes, accessors binary = buffers.binary() if buffers.views: + # the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + binary += b"\0" * (-len(binary) % 8) document["buffers"] = [{"byteLength": len(binary)}] document["bufferViews"] = buffers.views return pack(document, binary) diff --git a/bindings/python/test/test_codec.py b/bindings/python/test/test_codec.py index b7677db..d4d7749 100644 --- a/bindings/python/test/test_codec.py +++ b/bindings/python/test/test_codec.py @@ -59,6 +59,16 @@ def test_glb_layout(): assert "FSPM_scene" not in document.get("extensionsRequired", []) +def test_buffer_length_holds_the_chunk_padding(): + """A binary chunk of 4 bytes of data is padded to 8: the buffer's byteLength counts the padding, so that the + chunk is no longer than the buffer (glTF-Validator: BUFFER_GLB_CHUNK_TOO_BIG beyond 3 bytes).""" + frame = Frame(data_structures=[DataStructure("plants", entity_sets=[ + EntitySet("Plant", columns={"id": np.array([0], dtype=np.int64), "count": np.array([1], dtype=np.int32)}, + rank=1)])]) + document, binary = unpack(to_bytes(frame)) + assert len(binary) % 8 == 0 and document["buffers"][0]["byteLength"] == len(binary) + + def test_column_types(): columns = dict(id=np.arange(3, dtype=np.int64), f64=np.array([0.1, np.nan, -2.5e-300]), infinite=np.array([np.inf, -np.inf, np.nan]), infinite32=np.array([1., np.inf, -np.inf], dtype=np.float32), From f738c5db9cc1825df2281d6ba04c3a258e5a05c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 12:10:05 +0100 Subject: [PATCH 04/39] FspmTF.jl under the MIT licence, as VirtualPlantLab --- bindings/julia/FspmTF/LICENSE | 21 +++++++++++++++++++++ bindings/julia/README.md | 3 ++- doc/julia.md | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 bindings/julia/FspmTF/LICENSE diff --git a/bindings/julia/FspmTF/LICENSE b/bindings/julia/FspmTF/LICENSE new file mode 100644 index 0000000..629f9e7 --- /dev/null +++ b/bindings/julia/FspmTF/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Tristan Gérault and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/bindings/julia/README.md b/bindings/julia/README.md index 3c473da..6ab7077 100644 --- a/bindings/julia/README.md +++ b/bindings/julia/README.md @@ -10,4 +10,5 @@ MultiScaleTreeGraph.jl MTG to and from a `graph` data structure (`from_mtg`, `to - `conformance_run.jl`: the codec's dumps of the conformance files and its written scenes, for `.github/actions/conformance`. -Dependencies: JSON.jl and OrderedCollections.jl. See [the documentation](../../doc/julia.md). +Dependencies: JSON.jl and OrderedCollections.jl. Licence: MIT (`FspmTF/LICENSE`), as VirtualPlantLab; the rest of +fspm-TF is CeCILL-C. See [the documentation](../../doc/julia.md). diff --git a/doc/julia.md b/doc/julia.md index 90fb15c..ceb8b62 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -6,7 +6,7 @@ structure, and back. This is the MTG library of VirtualPlantLab's PlantGeom and ## Installing -FspmTF.jl is not registered yet. Add it from the repository's subfolder: +FspmTF.jl is MIT-licensed, as VirtualPlantLab (the rest of fspm-TF is CeCILL-C). It is not registered yet. Add it from the repository's subfolder: ```julia using Pkg From 7ea08e8d4f0eedaeab9880f0899200a06aec8e81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 12:49:06 +0100 Subject: [PATCH 05/39] FspmTF.jl, MTGs from other platforms' graphs: ids from 0 or sparse numbered 1, 2, ... in id order (MTG.jl sizes its attribute store by the largest id: CPlantBox's 2^32 offsets saturated memory), each id kept in fspm_tf_id; a parent in another complex (a lateral's first segment) kept in fspm_tf_parent and fspm_tf_edge_type, the node under its own complex; from_mtg writes both back (the conformance root system and CPlantBox's plants: MTG and back, the same graph) --- .../ext/FspmTFMultiScaleTreeGraphExt.jl | 70 ++++++++++++++----- bindings/julia/FspmTF/test/runtests.jl | 49 ++++++++++++- doc/julia.md | 12 +++- 3 files changed, 111 insertions(+), 20 deletions(-) diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl index ab91414..4497d57 100644 --- a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -28,6 +28,16 @@ using OrderedCollections: OrderedDict const PLATFORM = "MultiScaleTreeGraph" const METADATA_ATTRIBUTES = (:symbols, :scales, :description) +# MTG.jl's node ids start at 1 and size its attribute store (an id of 2^32 allocates 2^32 rows), fspm-TF's start at 0 +# and may be sparse (CPlantBox's segments: 2^32 + node id). to_mtg then numbers the nodes 1, 2, ... in the order of +# their ids, each keeping its id in this attribute; from_mtg writes it back as the id. +const ID_ATTRIBUTE = :fspm_tf_id +# An entity whose parent (same scale) is in another complex, e.g. the first segment of a lateral root, whose parent +# segment is in the bearing root: MTG.jl takes a node's complex from its ancestors, so the node stays under its complex +# (by '/') and keeps its parent and edge type in these attributes. +const PARENT_ATTRIBUTE = :fspm_tf_parent +const EDGE_TYPE_ATTRIBUTE = :fspm_tf_edge_type +const OWN_ATTRIBUTES = (ID_ATTRIBUTE, PARENT_ATTRIBUTE, EDGE_TYPE_ATTRIBUTE) const LINK_NAMES = Dict(:< => "successor", :+ => "branch", :/ => "decomposition") const LINKS = Dict(name => link for (link, name) in LINK_NAMES) const TEXT_TYPES = Dict("Date" => Date, "DateTime" => DateTime, "Time" => Time) @@ -102,7 +112,8 @@ function attribute_columns(nodes, set_name, variables, skip = ()) isroot(node) && name in METADATA_ATTRIBUTES ? nothing : attribute(node, name; default = nothing) names = Symbol[] for node in nodes, name in attribute_names(node) - name in names || name in skip || (isroot(node) && name in METADATA_ATTRIBUTES) || push!(names, name) + name in names || name in skip || name in OWN_ATTRIBUTES || (isroot(node) && name in METADATA_ATTRIBUTES) || + push!(names, name) end columns, declarations = Columns(), Declarations() missing_rows, text_types = json_object(), json_object() @@ -140,6 +151,7 @@ attributes the scalars. function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,String}(), variables = Dict{String,Variable}(), platform = PLATFORM, model = nothing) variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) + fspm_id(node) = attribute(node, ID_ATTRIBUTE; default = node_id(node)) nodes = preorder(mtg) scene = scale(mtg) == 0 ? mtg : nothing extras = json_object() @@ -162,18 +174,23 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S bearers = OrderedDict{Tuple{String,String},Vector{Tuple{Int64,Int64,String}}}() for (scale_number, scale_nodes) in by_scale set_name = set_names[scale_number] - columns = Columns("id" => Int64[node_id(node) for node in scale_nodes], + columns = Columns("id" => Int64[fspm_id(node) for node in scale_nodes], "label" => String[string(symbol(node)) for node in scale_nodes]) complexes = [complex_node(node) for node in scale_nodes] if scale_number != first(keys(by_scale)) - columns["complex"] = Int64[complex === nothing || complex === scene ? NO_ID : node_id(complex) + columns["complex"] = Int64[complex === nothing || complex === scene ? NO_ID : fspm_id(complex) for complex in complexes] end parents, edge_types = Int64[], String[] for (node, complex) in zip(scale_nodes, complexes) + if PARENT_ATTRIBUTE in attribute_names(node) # a parent in another complex (to_mtg) + push!(parents, attribute(node, PARENT_ATTRIBUTE)) + push!(edge_types, attribute(node, EDGE_TYPE_ATTRIBUTE)) + continue + end tree_parent = parent_node(node) if tree_parent !== nothing && scale(tree_parent) == scale(node) && link(node) in (:<, :+) - push!(parents, node_id(tree_parent)) + push!(parents, fspm_id(tree_parent)) push!(edge_types, LINK_NAMES[link(node)]) continue end @@ -183,10 +200,10 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S tree_parent === complex && link(node) == :/ && continue # its complex, by decomposition tree_parent === scene && link(node) == :/ && continue # the scene's component tree_parent === scene && - throw(ArgumentError("node $(node_id(node)) is linked to the scene by '$(link(node))', not '/'")) + throw(ArgumentError("node $(fspm_id(node)) is linked to the scene by '$(link(node))', not '/'")) pair = (set_name, set_names[scale(tree_parent)]) push!(get!(bearers, pair, Tuple{Int64,Int64,String}[]), - (node_id(node), node_id(tree_parent), LINK_NAMES[link(node)])) + (fspm_id(node), fspm_id(tree_parent), LINK_NAMES[link(node)])) end columns["parent"], columns["edge_type"] = parents, edge_types columns[INDEX_COLUMN] = Int64[index(node) for node in scale_nodes] @@ -218,7 +235,7 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S isempty(metadata) || (extras["metadata"] = metadata) scalars = nothing if scene !== nothing - extras["scene"] = json_object("id" => node_id(scene), "symbol" => string(symbol(scene)), + extras["scene"] = json_object("id" => fspm_id(scene), "symbol" => string(symbol(scene)), "index" => index(scene), "link" => string(link(scene))) columns, declarations, missing_of_scene, text_types_of_scene = attribute_columns([scene], "scalars", variables) @@ -230,8 +247,8 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S end isempty(missing_rows) || (extras["missing"] = missing_rows) isempty(text_types) || (extras["text_types"] = text_types) - order = [node_id(node) for node in nodes] - order == [node_id(node) for node in preorder_by_id(nodes)] || (extras["order"] = order) + order = [fspm_id(node) for node in nodes] + order == [fspm_id(node) for node in preorder_by_id(nodes)] || (extras["order"] = order) return DataStructure(id; kind = "graph", entity_sets = entity_sets, relations = relations, scalars = scalars, platform = platform, model = model, extras = json_object(PLATFORM => extras)) @@ -315,11 +332,24 @@ function FspmTF.to_mtg(data_structure::DataStructure) tree_parent[source] = (target, LINKS[label]) end end + complex_of = Dict{Int64,Int64}() + for set in entity_sets + haskey(set.columns, "complex") || continue + for (id, complex) in zip(set.columns["id"], set.columns["complex"]) + complex == NO_ID || (complex_of[id] = complex) + end + end for set in entity_sets # parents at the same scale, '<' or '+' haskey(set.columns, "parent") || continue edge_types = set.columns["edge_type"] for (id, parent_id, edge_type) in zip(set.columns["id"], set.columns["parent"], edge_types) - parent_id == NO_ID || (tree_parent[id] = (parent_id, edge_type == "successor" ? :< : :+)) + parent_id == NO_ID && continue + if get(complex_of, id, NO_ID) != get(complex_of, parent_id, NO_ID) # in another complex: kept aside + attributes[id][PARENT_ATTRIBUTE] = parent_id + attributes[id][EDGE_TYPE_ATTRIBUTE] = edge_type + else + tree_parent[id] = (parent_id, edge_type == "successor" ? :< : :+) + end end end @@ -329,20 +359,28 @@ function FspmTF.to_mtg(data_structure::DataStructure) scene_attributes = data_structure.scalars === nothing ? Dict{Symbol,Any}() : only(row_attributes(data_structure.scalars, get(missing_rows, "scalars", json_object()), get(text_types, "scalars", json_object()), ())) + # MTG.jl's ids: the entities' own, unless they start below 1 or are sparse; then 1, 2, ... in their order + all_ids = sort(collect(keys(encoding))) + renumbered = !isempty(all_ids) && (first(all_ids) < 1 || last(all_ids) > 2 * length(all_ids) + 1) + number_of = renumbered ? Dict(id => position for (position, id) in enumerate(all_ids)) : nothing + mtg_id(id) = renumbered ? number_of[id] : Int(id) + renumbered && foreach(id -> attributes[id][ID_ATTRIBUTE] = id, all_ids) if scene_entry === nothing && length(roots) == 1 && isempty(scene_attributes) root_id = only(roots) root_encoding = encoding[root_id] - root = Node(root_id, MutableNodeMTG(:/, Symbol(root_encoding.symbol), root_encoding.index, + root = Node(mtg_id(root_id), MutableNodeMTG(:/, Symbol(root_encoding.symbol), root_encoding.index, root_encoding.scale), attributes[root_id]) roots = Int64[] else scene_entry === nothing && - (scene_entry = json_object("id" => maximum(keys(encoding); init = -1) + 1, "symbol" => "Scene", - "index" => 0, "link" => "/")) + (scene_entry = json_object("id" => renumbered ? length(all_ids) + 1 : maximum(all_ids; init = 0) + 1, + "symbol" => "Scene", "index" => 0, "link" => "/")) root = Node(Int(scene_entry["id"]), MutableNodeMTG(Symbol(scene_entry["link"]), Symbol(scene_entry["symbol"]), Int(scene_entry["index"]), 0), scene_attributes) - root_id = node_id(root) + root_id = typemin(Int64) # the scene's key among the entities: not an entity's id end + entity_of = Dict{Int,Int64}(mtg_id(id) => id for id in all_ids) # MTG.jl's id -> the entity's + entity_of[node_id(root)] = root_id children_of = Dict{Int64,Vector{Int64}}() for (id, (parent_id, _)) in tree_parent push!(get!(children_of, parent_id, Int64[]), id) @@ -357,10 +395,10 @@ function FspmTF.to_mtg(data_structure::DataStructure) while !isempty(stack) node = pop!(stack) created = Any[] - for id in get(children_of, node_id(node), Int64[]) + for id in get(children_of, entity_of[node_id(node)], Int64[]) child_encoding = encoding[id] child_link = haskey(tree_parent, id) ? tree_parent[id][2] : :/ - push!(created, Node(Int(id), node, MutableNodeMTG(child_link, Symbol(child_encoding.symbol), + push!(created, Node(mtg_id(id), node, MutableNodeMTG(child_link, Symbol(child_encoding.symbol), child_encoding.index, child_encoding.scale), attributes[id])) end diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl index 2156734..b03aece 100644 --- a/bindings/julia/FspmTF/test/runtests.jl +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -189,15 +189,60 @@ expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] @test attribute(get_node(rebuilt, 6), :length) == 0.2 end + @testset "MultiScaleTreeGraph: ids from 0 and sparse (MTG.jl's: from 1, dense), numbered and written back" begin + plants = DataStructure("plant"; entity_sets = [ + EntitySet("Plant"; columns = Dict("id" => [0]), rank = 1), + EntitySet("Organ"; rank = 5, columns = Dict("id" => [1, 2], "complex" => [0, 0], "parent" => [-1, 1], + "edge_type" => ["none", "branch"], "length" => [1.0, 2.0])), + EntitySet("SubOrgan"; rank = 6, columns = Dict("id" => [2^32 + 5, 2^32 + 9], "complex" => [1, 2], + "parent" => [-1, 2^32 + 5], "edge_type" => ["none", "branch"]))]) + mtg = to_mtg(plants) + nodes = traverse(mtg, node -> node) + @test sort(node_id.(nodes)) == 1:5 # dense, from 1 + @test [attribute(node, :fspm_tf_id) for node in nodes] == [0, 1, 2, 2^32 + 9, 2^32 + 5] + lateral = only(node for node in nodes if attribute(node, :fspm_tf_id) == 2^32 + 9) + @test attribute(parent(lateral), :fspm_tf_id) == 2 && link(lateral) == :/ # under its own organ + @test attribute(lateral, :fspm_tf_parent) == 2^32 + 5 && attribute(lateral, :fspm_tf_edge_type) == "branch" + again = from_mtg(mtg; scales = Dict(1 => "Plant", 5 => "Organ", 6 => "SubOrgan")) + for name in ("Plant", "Organ", "SubOrgan") # the same rows, in the MTG's order + written, read = entity_set(plants, name), entity_set(again, name) + order = sortperm(read.columns["id"]) + for column in ("id", "complex", "parent", "edge_type", "length") + haskey(written.columns, column) || continue + @test read.columns[column][order] == written.columns[column][sortperm(written.columns["id"])] + end + end + @test !haskey(entity_set(again, "SubOrgan").columns, "fspm_tf_id") + end + @testset "MultiScaleTreeGraph: another platform's graph (the conformance root system) as an MTG" begin root_system = read_frame(joinpath(CONFORMANCE, "root_system.glb")).data_structures[1] mtg = to_mtg(root_system) nodes = traverse(mtg, node -> node) @test length(nodes) == sum(row_count(set) for set in root_system.entity_sets) + 1 # + the scene + entity_id(node) = attribute(node, :fspm_tf_id; default = node_id(node)) # ids 10 to 1000: renumbered + node_of = Dict(entity_id(node) => node for node in nodes if !isroot(node)) segments = entity_set(root_system, "SubOrgan") + complex_of = Dict(zip(segments.columns["id"], segments.columns["complex"])) for (id, parent_id) in zip(segments.columns["id"], segments.columns["parent"]) - parent_id == NO_ID || @test node_id(parent(get_node(mtg, Int(id)))) == parent_id + node = node_of[id] + @test entity_id(parent(node)) == complex_of[id] || entity_id(parent(node)) == parent_id + parent_id == NO_ID && continue + if complex_of[parent_id] == complex_of[id] + @test entity_id(parent(node)) == parent_id && link(node) == :< # along its axis + else + @test attribute(node, :fspm_tf_parent) == parent_id && link(node) == :/ # a lateral's first segment + end + end + @test attribute(node_of[segments.columns["id"][1]], :length) == segments.columns["length"][1] + again = from_mtg(mtg; scales = Dict(1 => "Plant", 2 => "Axis", 6 => "SubOrgan")) # and back: the same graph + for set in root_system.entity_sets + read = entity_set(again, set.name) + order, written_order = sortperm(read.columns["id"]), sortperm(set.columns["id"]) + for column in ("id", "complex", "parent", "edge_type", "length") + haskey(set.columns, column) || continue + @test isequal(read.columns[column][order], set.columns[column][written_order]) + end end - @test attribute(get_node(mtg, Int(segments.columns["id"][1])), :length) == segments.columns["length"][1] end end diff --git a/doc/julia.md b/doc/julia.md index ceb8b62..00e2fdb 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -29,8 +29,16 @@ frame = read_frame("plant.glb") same_mtg = to_mtg(frame.data_structures[1]) # the same ids, symbols, links, attributes and order ``` -`to_mtg` also reads a graph another platform wrote, e.g. a root system simulated by metafspm. Its MTG has one scale -per entity set and its variables as attributes, under a `Scene` root that holds the scalars. +`to_mtg` also reads a graph another platform wrote, e.g. a root system simulated by metafspm or a CPlantBox plant. +Its MTG has one scale per entity set and its variables as attributes, under a `Scene` root that holds the scalars. +Two things differ from the other platform's graph, and are undone by `from_mtg`: + +- **Ids.** MTG.jl's node ids start at 1 and size its attribute store. When ids start at 0 or are sparse (CPlantBox's + segments: 2^32 + node id), the nodes are numbered 1, 2, … in id order, and each keeps its id in the attribute + `fspm_tf_id`. +- **A parent in another complex**, such as a lateral root's first segment. MTG.jl takes a node's complex from its + ancestors, so the node stays under its own axis (`/`) and keeps its parent in the attributes `fspm_tf_parent` and + `fspm_tf_edge_type`. | fspm-TF | MultiScaleTreeGraph | |---|---| From f94f227469c03a3ea019859754fc1114b16894cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 12:49:06 +0100 Subject: [PATCH 06/39] The C++ codec (bindings/cpp): the GLB container, the data model, reading and writing frames and sequences, the conformance dump and tool (every conformance file read as expected; what it writes passes both validators with no warning), a CMake package; CPlantBox's adapter (bindings/cpp/cplantbox): organisms as Plant, Organ and SubOrgan with CPlantBox's variables declared and its parameters in extras, PlantVisualiser geometry, the soil grid and segment-to-cell mapping, static root systems from CPlantBox's frames and from other platforms' (by segment ends and complexes); CI on Linux, macOS and Windows, conformance, the adapter on a pinned CPlantBox; doc/cpp.md --- .github/workflows/cpp-codec.yml | 67 ++ CHANGELOG.md | 5 + README.md | 2 +- bindings/cpp/CMakeLists.txt | 75 ++ bindings/cpp/README.md | 20 + bindings/cpp/cmake/fspm_tfConfig.cmake.in | 6 + bindings/cpp/cplantbox/CMakeLists.txt | 27 + .../cplantbox/include/fspm_tf/cplantbox.hpp | 81 ++ bindings/cpp/cplantbox/src/cplantbox.cpp | 467 +++++++++++ .../cpp/cplantbox/test/test_cplantbox.cpp | 211 +++++ bindings/cpp/include/fspm_tf/fspm_tf.hpp | 77 ++ bindings/cpp/include/fspm_tf/model.hpp | 235 ++++++ bindings/cpp/src/bytes.hpp | 56 ++ bindings/cpp/src/codec.cpp | 776 ++++++++++++++++++ bindings/cpp/src/dump.cpp | 168 ++++ bindings/cpp/src/glb.cpp | 72 ++ bindings/cpp/src/model.cpp | 136 +++ bindings/cpp/src/sequence.cpp | 65 ++ bindings/cpp/test/test_fspm_tf.cpp | 238 ++++++ bindings/cpp/tools/conformance_run.cpp | 37 + doc/cpp.md | 88 ++ doc/index.md | 1 + doc/platforms.md | 2 +- implementations.md | 3 +- 24 files changed, 2912 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/cpp-codec.yml create mode 100644 bindings/cpp/CMakeLists.txt create mode 100644 bindings/cpp/README.md create mode 100644 bindings/cpp/cmake/fspm_tfConfig.cmake.in create mode 100644 bindings/cpp/cplantbox/CMakeLists.txt create mode 100644 bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp create mode 100644 bindings/cpp/cplantbox/src/cplantbox.cpp create mode 100644 bindings/cpp/cplantbox/test/test_cplantbox.cpp create mode 100644 bindings/cpp/include/fspm_tf/fspm_tf.hpp create mode 100644 bindings/cpp/include/fspm_tf/model.hpp create mode 100644 bindings/cpp/src/bytes.hpp create mode 100644 bindings/cpp/src/codec.cpp create mode 100644 bindings/cpp/src/dump.cpp create mode 100644 bindings/cpp/src/glb.cpp create mode 100644 bindings/cpp/src/model.cpp create mode 100644 bindings/cpp/src/sequence.cpp create mode 100644 bindings/cpp/test/test_fspm_tf.cpp create mode 100644 bindings/cpp/tools/conformance_run.cpp create mode 100644 doc/cpp.md diff --git a/.github/workflows/cpp-codec.yml b/.github/workflows/cpp-codec.yml new file mode 100644 index 0000000..a0b718e --- /dev/null +++ b/.github/workflows/cpp-codec.yml @@ -0,0 +1,67 @@ +# The C++ codec (bindings/cpp): its tests on Linux, macOS and Windows; the conformance check other codecs run (its +# dumps of the conformance files compared with their expected values, the files it writes validated); and CPlantBox's +# adapter, built on CPlantBox's library (a pinned commit) and tested on CPlantBox plants. OpenAlea's conda CI builds +# the Python package only. +name: C++ codec + +on: + push: + branches: + - main + pull_request: + +env: + CPLANTBOX_COMMIT: 17e1fab617e72fa5e2773dfee723083bdf515053 + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - name: Configure (nlohmann_json fetched) + run: cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + - name: Build + run: cmake --build build --config Release + - name: Tests + run: ctest --test-dir build -C Release --output-on-failure + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Dump and write the conformance scenes with the C++ codec + run: | + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target fspm_tf_conformance_run + ./build/fspm_tf_conformance_run conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + + cplantbox-adapter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: CPlantBox's library (no Python bindings, no PiafMunch) + run: | + git clone --quiet https://github.com/Plant-Root-Soil-Interactions-Modelling/CPlantBox.git cplantbox + git -C cplantbox checkout --quiet "$CPLANTBOX_COMMIT" + cmake -S cplantbox -B cplantbox-build -DCMAKE_BUILD_TYPE=Release -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF \ + -DCPB_BUILD_SHARED=OFF -DCPB_NATIVE_CPU_TUNING=OFF + cmake --build cplantbox-build --target CPlantBox -j 4 + - name: The codec and the adapter + run: | + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release -DFSPM_TF_CPLANTBOX_SOURCE_DIR="$PWD/cplantbox" \ + -DFSPM_TF_CPLANTBOX_LIBRARY="$PWD/cplantbox-build/src/libCPlantBox.a" + cmake --build build -j 4 + ctest --test-dir build --output-on-failure + - name: What the adapter wrote is valid fspm-TF + run: | + python -m pip install numpy jsonschema . + fspm-tf validate build/cplantbox/cplantbox_out/*.glb diff --git a/CHANGELOG.md b/CHANGELOG.md index f9ef5ef..79f0b0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,4 +10,9 @@ - The ParaView reader plugin (`paraview/`). - FspmTF.jl, the Julia codec (`bindings/julia/FspmTF/`): conformant reading and writing, sequences, and a MultiScaleTreeGraph.jl package extension (`from_mtg`, `to_mtg`); its CI, `.github/workflows/julia-codec.yml`. +- The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter + (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph + with segment ends); its CI, `.github/workflows/cpp-codec.yml`. +- FspmTF.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `fspm_tf_id`), and parents + in another complex kept (`fspm_tf_parent`). - The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/README.md b/README.md index 913eebd..1192f0e 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ change there, released as a new version. Everything else implements, checks or d | `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | | `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension | no | +| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | | `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt new file mode 100644 index 0000000..b2f8ada --- /dev/null +++ b/bindings/cpp/CMakeLists.txt @@ -0,0 +1,75 @@ +# fspm_tf, the C++ codec of fspm-TF: a library (fspm_tf::fspm_tf), its tests, the conformance tool, and, when +# CPlantBox is given (FSPM_TF_CPLANTBOX_SOURCE_DIR and FSPM_TF_CPLANTBOX_LIBRARY), CPlantBox's adapter. +# +# cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +cmake_minimum_required(VERSION 3.16) +project(fspm_tf VERSION 0.1.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +option(FSPM_TF_BUILD_TESTS "Build the tests and the conformance tool" ${PROJECT_IS_TOP_LEVEL}) +set(FSPM_TF_CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder (its src/ holds structural/), for the adapter") +set(FSPM_TF_CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox), for the adapter") + +# The one dependency, header-only: nlohmann_json, found installed or fetched +find_package(nlohmann_json 3.11 QUIET) +if(NOT nlohmann_json_FOUND) + include(FetchContent) + FetchContent_Declare(nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz + URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa) + set(JSON_Install ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(nlohmann_json) +endif() + +add_library(fspm_tf + src/model.cpp + src/glb.cpp + src/codec.cpp + src/sequence.cpp + src/dump.cpp) +add_library(fspm_tf::fspm_tf ALIAS fspm_tf) +target_include_directories(fspm_tf PUBLIC + $ + $) +target_link_libraries(fspm_tf PUBLIC nlohmann_json::nlohmann_json) +target_compile_features(fspm_tf PUBLIC cxx_std_17) +set_target_properties(fspm_tf PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(MSVC) + target_compile_options(fspm_tf PRIVATE /W4 /bigobj) +else() + target_compile_options(fspm_tf PRIVATE -Wall -Wextra -Wpedantic) +endif() + +# Installed as a CMake package: find_package(fspm_tf) then target_link_libraries(... fspm_tf::fspm_tf) +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) +install(TARGETS fspm_tf EXPORT fspm_tfTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(EXPORT fspm_tfTargets NAMESPACE fspm_tf:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) +configure_package_config_file(cmake/fspm_tfConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfigVersion.cmake + COMPATIBILITY SameMinorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) + +if(FSPM_TF_BUILD_TESTS) + enable_testing() + set(FSPM_TF_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") + add_executable(fspm_tf_conformance_run tools/conformance_run.cpp) + target_link_libraries(fspm_tf_conformance_run PRIVATE fspm_tf::fspm_tf) + add_executable(fspm_tf_tests test/test_fspm_tf.cpp) + target_link_libraries(fspm_tf_tests PRIVATE fspm_tf::fspm_tf) + add_test(NAME fspm_tf_tests COMMAND fspm_tf_tests "${FSPM_TF_CONFORMANCE}") +endif() + +# CPlantBox's adapter: built when CPlantBox's sources and library are given +if(FSPM_TF_CPLANTBOX_SOURCE_DIR AND FSPM_TF_CPLANTBOX_LIBRARY) + add_subdirectory(cplantbox) +endif() diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md new file mode 100644 index 0000000..477ad47 --- /dev/null +++ b/bindings/cpp/README.md @@ -0,0 +1,20 @@ +# fspm_tf (C++) + +The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.1). In `cplantbox/` is +CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, +and turns a graph into a static CPlantBox root system. + +- `include/fspm_tf/`: the data model (`model.hpp`) and the codec (`fspm_tf.hpp`). +- `src/`: the GLB container, reading and writing, sequences, the conformance dump. +- `tools/conformance_run.cpp`: the codec's dumps of the conformance files and its written scenes, for + `.github/actions/conformance`. +- `test/`: the tests, the conformance files included. +- `cplantbox/`: the adapter (`fspm_tf::cplantbox`), built when `FSPM_TF_CPLANTBOX_SOURCE_DIR` and + `FSPM_TF_CPLANTBOX_LIBRARY` are given. + +```bash +cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +``` + +Its one dependency is nlohmann_json (header-only, MIT), found installed or fetched. It installs as a CMake package: +`find_package(fspm_tf)`, then `target_link_libraries(... fspm_tf::fspm_tf)`. See [the documentation](../../doc/cpp.md). diff --git a/bindings/cpp/cmake/fspm_tfConfig.cmake.in b/bindings/cpp/cmake/fspm_tfConfig.cmake.in new file mode 100644 index 0000000..6051350 --- /dev/null +++ b/bindings/cpp/cmake/fspm_tfConfig.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(nlohmann_json 3.11) +include("${CMAKE_CURRENT_LIST_DIR}/fspm_tfTargets.cmake") +check_required_components(fspm_tf) diff --git a/bindings/cpp/cplantbox/CMakeLists.txt b/bindings/cpp/cplantbox/CMakeLists.txt new file mode 100644 index 0000000..c5d730e --- /dev/null +++ b/bindings/cpp/cplantbox/CMakeLists.txt @@ -0,0 +1,27 @@ +# CPlantBox's adapter (fspm_tf::cplantbox), on CPlantBox's built library and its sources' headers: +# +# cmake -S bindings/cpp -B build -DFSPM_TF_CPLANTBOX_SOURCE_DIR=/path/to/CPlantBox \ +# -DFSPM_TF_CPLANTBOX_LIBRARY=/path/to/build/src/libCPlantBox.a +# +# It is meant to move into CPlantBox once its team agrees (devplan_bindings.md, P4). +set(CPLANTBOX_SOURCE "${FSPM_TF_CPLANTBOX_SOURCE_DIR}/src") +add_library(CPlantBox::CPlantBox UNKNOWN IMPORTED) +set_target_properties(CPlantBox::CPlantBox PROPERTIES + IMPORTED_LOCATION "${FSPM_TF_CPLANTBOX_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CPLANTBOX_SOURCE}/structural;${CPLANTBOX_SOURCE}/functional;${CPLANTBOX_SOURCE}/visualisation;${CPLANTBOX_SOURCE}/external/tinyxml2;${CPLANTBOX_SOURCE}/external/eigen") + +add_library(fspm_tf_cplantbox src/cplantbox.cpp) +add_library(fspm_tf::cplantbox ALIAS fspm_tf_cplantbox) +target_include_directories(fspm_tf_cplantbox PUBLIC $) +target_link_libraries(fspm_tf_cplantbox PUBLIC fspm_tf::fspm_tf CPlantBox::CPlantBox) +if(MSVC) + target_compile_definitions(fspm_tf_cplantbox PRIVATE _USE_MATH_DEFINES NOMINMAX) +endif() + +if(FSPM_TF_BUILD_TESTS) + add_executable(fspm_tf_cplantbox_tests test/test_cplantbox.cpp) + target_link_libraries(fspm_tf_cplantbox_tests PRIVATE fspm_tf::cplantbox) + add_test(NAME fspm_tf_cplantbox_tests + COMMAND fspm_tf_cplantbox_tests "${FSPM_TF_CONFORMANCE}" "${FSPM_TF_CPLANTBOX_SOURCE_DIR}/modelparameter/structural" + "${CMAKE_CURRENT_BINARY_DIR}/cplantbox_out") +endif() diff --git a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp new file mode 100644 index 0000000..5d34508 --- /dev/null +++ b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp @@ -0,0 +1,81 @@ +// CPlantBox's adapter: an organism (a Plant, a RootSystem, a MappedPlant) as an fspm-TF graph, its leaves' and +// stems' geometry as a mesh, its soil grid and segment-to-cell mapping; and back, a graph (CPlantBox's, or another +// platform's) as a static CPlantBox root system. +// +// | fspm-TF | CPlantBox | +// |----------------------------------|---------------------------------------------------------------------------| +// | Plant (rank 1), one row, id 0 | the organism | +// | Organ (rank 5), id 1 + organ id | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | +// | | plant, parent its parent organ (branch) | +// | SubOrgan (rank 6), id | its segments: complex its organ, parent the segment ending at its | +// | 2^32 + distal node id | proximal node (successor along an organ, branch at a lateral's base) | +// | positions (proximal_end_*, ...) | node coordinates, cm converted to metres | +// | other variables | CPlantBox's names and units (cm, d), declared with their standard names | +// | extras["cplantbox"] | the id offsets, the organism's parameters (its XML) | +// +// CPlantBox has two scales below the plant: organs and segments. A frame with more scales (e.g. metafspm's Plant, +// Axis, SubOrgan) is read through the entity set holding the segment ends (proximal_end_*, distal_end_*) and the +// coarser entity set its complex points to, whatever their names and ranks (see to_root_system). +#pragma once + +#include +#include +#include +#include +#include + +#include "fspm_tf/fspm_tf.hpp" + +namespace CPlantBox { +class Organism; +class MappedPlant; +class MappedSegments; +class Plant; +} // namespace CPlantBox + +namespace fspm_tf::cplantbox { + +/// The name of CPlantBox's entry in a data structure's extras, and of its column in the vocabulary. +inline constexpr const char *PLATFORM = "cplantbox"; +/// Organ ids are CPlantBox's organ ids plus this offset (the plant is 0). +inline constexpr std::int64_t ORGAN_ID_OFFSET = 1; +/// Segment ids are their distal node's id plus this offset: stable while the plant grows, distinct from organ ids. +inline constexpr std::int64_t SEGMENT_ID_OFFSET = std::int64_t(1) << 32; + +struct Options { + std::string id = "plant"; ///< the data structure's id + std::optional model; ///< e.g. the parameter file's name + bool parameters_in_extras = true; ///< the organism's parameters (XML) in extras, to rebuild it +}; + +/// The organism as a graph data structure: its plant, organs and segments, with their variables. +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options = {}); + +/// The organism at its simulation time (days) as a frame of one data structure. +Frame to_frame(const CPlantBox::Organism &organism, const Options &options = {}); + +/// Adds the plant's geometry (CPlantBox's PlantVisualiser: leaf blades and stems, triangles) to its data structure, as +/// a mesh whose vertices carry their segment, organ and plant rows. resolution: PlantVisualiser's geometry +/// resolution (points around a stem), leaf_resolution: points along a leaf. +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, + int resolution = 8, int leaf_resolution = 20); + +/// The rectangular soil grid of a MappedSegments (set by setRectangularGrid), as a regular_grid data structure of +/// `cells` (cell ids: C-order index of (x, y, z), z pointing up, metres). +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id = "soil", + const std::string &cells = "Voxel"); + +/// The segments' soil cells (MappedSegments::seg2cell) as a mapping from the plant's SubOrgan to the soil's cells, +/// weight 1. The grid must be set without cutting segments (cut = false), so that segments keep CPlantBox's numbering. +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name = "roots_in_soil"); + +/// A graph as a static CPlantBox root system (StaticRoot organs on a seed, as CPlantBox's StaticPlant builds from +/// RSML): its segments' ends as the organs' nodes, a lateral at its parent's node, each organ's radius the mean of its +/// segments'. The segments: the entity set holding the segment ends (proximal_end_* and distal_end_*, by standard +/// name or by CPlantBox's names); the organs: the entity set their complex points to. Units are converted from those +/// declared (m, cm, mm). Organs that are not roots (CPlantBox's organ_type, when written by CPlantBox) are skipped: +/// CPlantBox has static roots only. +std::shared_ptr to_root_system(const DataStructure &data_structure); + +} // namespace fspm_tf::cplantbox diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp new file mode 100644 index 0000000..d1b3821 --- /dev/null +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -0,0 +1,467 @@ +#include "fspm_tf/cplantbox.hpp" + +#include +#include +#include +#include +#include +#include + +#include "MappedOrganism.h" +#include "Organ.h" +#include "Organism.h" +#include "Plant.h" +#include "PlantVisualiser.h" +#include "Root.h" +#include "Seed.h" +#include "organparameter.h" +#include "rootparameter.h" +#include "tinyxml2.h" + +namespace fspm_tf::cplantbox { + +namespace { + +constexpr double METRES_PER_CENTIMETRE = 0.01; + +Variable declared(const std::string &unit, const std::string &kind, const std::string &description, + std::optional standard_name = std::nullopt, const std::string &role = "state") { + Variable variable; + variable.unit = unit; + variable.kind = kind; + variable.description = description; + variable.standard_name = std::move(standard_name); + variable.role = role; + return variable; +} + +/// The vocabulary's label of a CPlantBox organ type: a root or a stem is a whole axis, a leaf an organ. +std::string organ_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_seed: return "Seed"; + case CPlantBox::Organism::ot_root: return "RootAxis"; + case CPlantBox::Organism::ot_stem: return "ShootAxis"; + case CPlantBox::Organism::ot_leaf: return "Leaf"; + default: return "Organ"; + } +} + +std::string segment_label(int organ_type) { + switch (organ_type) { + case CPlantBox::Organism::ot_root: return "RootSegment"; + case CPlantBox::Organism::ot_stem: return "StemElement"; + case CPlantBox::Organism::ot_leaf: return "LeafElement"; + default: return "SubOrgan"; + } +} + +/// The organism's organs, seeds included, parents before their laterals. +std::vector> all_organs(const CPlantBox::Organism &organism) { + std::vector> organs; + std::set seen; + std::function &)> visit = [&](const std::shared_ptr &organ) { + if (!organ || seen.count(organ.get())) return; + seen.insert(organ.get()); + organs.push_back(organ); + for (int child = 0; child < organ->getNumberOfChildren(); ++child) visit(organ->getChild(child)); + }; + for (const auto &organ : organism.getOrgans(-1, true)) { // from each organ's root: its seed, or itself + auto base = organ; + while (base->getParent()) base = base->getParent(); + visit(base); + } + return organs; +} + +/// The organism's parameters as CPlantBox's XML (as Organism::writeParameters writes them, skipping the empty entries +/// a MappedPlant can leave in its parameter map, on which writeParameters fails). +std::string parameters_xml(const CPlantBox::Organism &organism) { + tinyxml2::XMLDocument document; + tinyxml2::XMLElement *parameters = document.NewElement("plant"); + for (int organ_type = 0; organ_type <= CPlantBox::Organism::ot_leaf; ++organ_type) + for (const auto ¶meter : organism.getOrganRandomParameter(organ_type)) + if (parameter && (organ_type == CPlantBox::Organism::ot_seed || parameter->subType > 0)) + parameters->InsertEndChild(parameter->writeXML(document, false)); + document.InsertEndChild(parameters); + tinyxml2::XMLPrinter printer; + document.Print(&printer); + return printer.CStr(); +} + +} // namespace + +DataStructure to_data_structure(const CPlantBox::Organism &organism, const Options &options) { + const auto organs = all_organs(organism); + + // Plant: the organism + EntitySet plant("Plant", 1); + plant.set("id", std::vector{0}); + + // Organ: one row per organ + EntitySet organ_set("Organ", 5); + std::vector organ_ids, complexes, parents; + std::vector labels, edge_types; + std::vector organ_types, sub_types, orders, parent_node_indices, node_counts; + std::vector ages, lengths, radii, creation_times; + std::vector alive, active; + for (const auto &organ : organs) { + const auto parent = organ->getParent(); + organ_ids.push_back(ORGAN_ID_OFFSET + organ->getId()); + complexes.push_back(0); + parents.push_back(parent ? ORGAN_ID_OFFSET + parent->getId() : NO_ID); + edge_types.push_back(parent ? "branch" : "none"); + labels.push_back(organ_label(organ->organType())); + organ_types.push_back(organ->organType()); + sub_types.push_back(static_cast(organ->getParameter("subType"))); + orders.push_back(static_cast(organ->getParameter("order"))); + parent_node_indices.push_back(organ->getParentNI()); + node_counts.push_back(organ->getNumberOfNodes()); + ages.push_back(organ->getAge()); + lengths.push_back(organ->getLength(true)); + radii.push_back(organ->getParameter("radius")); + creation_times.push_back(organ->getNumberOfNodes() > 0 ? organ->getNodeCT(0) : std::nan("")); + alive.push_back(organ->isAlive()); + active.push_back(organ->isActive()); + } + organ_set.set("id", organ_ids).set("label", Column(labels)).set("complex", complexes).set("parent", parents) + .set("edge_type", Column(edge_types)).set("organType", organ_types).set("subType", sub_types) + .set("order", orders).set("age", ages).set("length", lengths).set("radius", radii) + .set("creationTime", creation_times).set("parentNI", parent_node_indices).set("numberOfNodes", node_counts) + .set("alive", alive).set("active", active); + organ_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type: 1 seed, 2 root, 3 stem, 4 leaf", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type: the organ's parameter set", {}, "setting")) + .declare("order", declared("1", "descriptor", "branching order (0: a base organ)", "branching_order")) + .declare("age", declared("d", "intensive", "the organ's age", "age")) + .declare("length", declared("cm", "extensive", "the organ's realized length", "length")) + .declare("radius", declared("cm", "intensive", "the organ's radius", "radius", "parameter")) + .declare("creationTime", declared("d", "descriptor", "the creation time of the organ's first node")) + .declare("parentNI", declared("1", "descriptor", "the node index, in its parent organ, where it emerges")) + .declare("numberOfNodes", declared("1", "descriptor", "the number of its nodes")) + .declare("alive", declared("1", "descriptor", "whether it is alive")) + .declare("active", declared("1", "descriptor", "whether it still grows")); + + // SubOrgan: one row per segment, its id its distal node's + std::map segment_ending_at; // node id -> segment id + for (const auto &organ : organs) + for (int node = 1; node < organ->getNumberOfNodes(); ++node) + segment_ending_at[organ->getNodeId(node)] = SEGMENT_ID_OFFSET + organ->getNodeId(node); + EntitySet segment_set("SubOrgan", 6); + std::vector segment_ids, segment_complexes, segment_parents; + std::vector segment_labels, segment_edge_types; + std::vector segment_organ_types, segment_sub_types; + std::vector x1, y1, z1, x2, y2, z2, segment_radii, segment_lengths, segment_creation_times; + for (std::size_t index = 0; index < organs.size(); ++index) { + const auto &organ = organs[index]; + const double radius = radii[index]; + for (int node = 1; node < organ->getNumberOfNodes(); ++node) { + const auto proximal = organ->getNode(node - 1), distal = organ->getNode(node); + segment_ids.push_back(SEGMENT_ID_OFFSET + organ->getNodeId(node)); + segment_complexes.push_back(organ_ids[index]); + const auto parent = segment_ending_at.find(organ->getNodeId(node - 1)); + segment_parents.push_back(parent == segment_ending_at.end() ? NO_ID : parent->second); + segment_edge_types.push_back(parent == segment_ending_at.end() ? "none" : node == 1 ? "branch" : "successor"); + segment_labels.push_back(segment_label(organ->organType())); + segment_organ_types.push_back(organ->organType()); + segment_sub_types.push_back(sub_types[index]); + x1.push_back(proximal.x * METRES_PER_CENTIMETRE); + y1.push_back(proximal.y * METRES_PER_CENTIMETRE); + z1.push_back(proximal.z * METRES_PER_CENTIMETRE); + x2.push_back(distal.x * METRES_PER_CENTIMETRE); + y2.push_back(distal.y * METRES_PER_CENTIMETRE); + z2.push_back(distal.z * METRES_PER_CENTIMETRE); + segment_radii.push_back(radius); + segment_lengths.push_back(distal.minus(proximal).length()); + segment_creation_times.push_back(organ->getNodeCT(node)); + } + } + segment_set.set("id", segment_ids).set("label", Column(segment_labels)).set("complex", segment_complexes) + .set("parent", segment_parents).set("edge_type", Column(segment_edge_types)) + .set("organType", segment_organ_types).set("subType", segment_sub_types) + .set("x1", x1).set("y1", y1).set("z1", z1).set("x2", x2).set("y2", y2).set("z2", z2) + .set("radius", segment_radii).set("length", segment_lengths).set("creationTime", segment_creation_times); + segment_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type of its organ", {}, "setting")) + .declare("subType", declared("1", "descriptor", "CPlantBox's sub type of its organ", {}, "setting")) + .declare("x1", declared("m", "descriptor", "proximal node, x", "proximal_end_x")) + .declare("y1", declared("m", "descriptor", "proximal node, y", "proximal_end_y")) + .declare("z1", declared("m", "descriptor", "proximal node, z", "proximal_end_z")) + .declare("x2", declared("m", "descriptor", "distal node, x", "distal_end_x")) + .declare("y2", declared("m", "descriptor", "distal node, y", "distal_end_y")) + .declare("z2", declared("m", "descriptor", "distal node, z", "distal_end_z")) + .declare("radius", declared("cm", "intensive", "the segment's radius (its organ's)", "radius", "parameter")) + .declare("length", declared("cm", "extensive", "the segment's length", "length", "derived")) + .declare("creationTime", declared("d", "descriptor", "the creation time of its distal node")); + + DataStructure data_structure(options.id, "graph"); + data_structure.platform = "CPlantBox"; + data_structure.model = options.model; + data_structure.entity_sets = {plant, organ_set, segment_set}; + json extras = {{"organ_id_offset", ORGAN_ID_OFFSET}, {"segment_id_offset", SEGMENT_ID_OFFSET}, + {"length_unit", "cm"}, {"time_unit", "d"}}; + if (options.parameters_in_extras) extras["parameters"] = parameters_xml(organism); + data_structure.extras = {{PLATFORM, extras}}; + return data_structure; +} + +Frame to_frame(const CPlantBox::Organism &organism, const Options &options) { + Frame frame; + frame.data_structures.push_back(to_data_structure(organism, options)); + frame.time = organism.getSimTime(); + frame.time_unit = "d"; + frame.writer = {{"platform", "CPlantBox"}, {"adapter", "fspm_tf::cplantbox"}}; + frame.vocabulary_version = "0.1"; + return frame; +} + +void add_geometry(DataStructure &data_structure, const std::shared_ptr &plant, int resolution, + int leaf_resolution) { + CPlantBox::PlantVisualiser visualiser(plant); + visualiser.SetGeometryResolution(resolution); + visualiser.SetLeafResolution(leaf_resolution); + visualiser.ComputeGeometry(); + const auto points = visualiser.GetGeometry(); // x, y, z per vertex, cm + const auto indices = visualiser.GetGeometryIndices(); + const auto node_ids = visualiser.GetGeometryNodeIds(); // the node each vertex belongs to + const auto &segments = data_structure.entity_set("SubOrgan"); + const auto &organs = data_structure.entity_set("Organ"); + std::map segment_row, organ_row; + const auto &segment_ids = segments.column("id").as(); + const auto &segment_complexes = segments.column("complex").as(); + for (std::size_t row = 0; row < segment_ids.size(); ++row) segment_row[segment_ids[row]] = std::int64_t(row); + const auto &organ_ids = organs.column("id").as(); + for (std::size_t row = 0; row < organ_ids.size(); ++row) organ_row[organ_ids[row]] = std::int64_t(row); + Mesh mesh; + mesh.name = "plant"; + mesh.positions.reserve(points.size()); + for (const double value : points) mesh.positions.push_back(value * METRES_PER_CENTIMETRE); + std::vector segment_rows, organ_rows, plant_rows; + for (std::size_t vertex = 0; vertex < mesh.vertex_count(); ++vertex) { + const auto node = vertex < node_ids.size() ? node_ids[vertex] : -1; + const auto found = segment_row.find(SEGMENT_ID_OFFSET + node); + if (found == segment_row.end()) { // a base node: no segment ends there + segment_rows.push_back(NO_ID); + organ_rows.push_back(NO_ID); + } else { + segment_rows.push_back(found->second); + organ_rows.push_back(organ_row.at(segment_complexes[static_cast(found->second)])); + } + plant_rows.push_back(0); + } + mesh.feature_rows = {{"SubOrgan", segment_rows}, {"Organ", organ_rows}, {"Plant", plant_rows}}; + mesh.indices = std::vector(indices.begin(), indices.end()); + data_structure.meshes.push_back(std::move(mesh)); +} + +DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::string &id, const std::string &cells) { + const auto &minimum = segments.minBound, &maximum = segments.maxBound, &resolution = segments.resolution; + const std::vector shape = {std::int64_t(resolution.x), std::int64_t(resolution.y), + std::int64_t(resolution.z)}; + if (shape[0] <= 0 || shape[1] <= 0 || shape[2] <= 0) throw Error("no rectangular grid set (setRectangularGrid)"); + Grid grid; + grid.shape = shape; + grid.origin = {minimum.x * METRES_PER_CENTIMETRE, minimum.y * METRES_PER_CENTIMETRE, minimum.z * METRES_PER_CENTIMETRE}; + grid.spacing = {(maximum.x - minimum.x) / resolution.x * METRES_PER_CENTIMETRE, + (maximum.y - minimum.y) / resolution.y * METRES_PER_CENTIMETRE, + (maximum.z - minimum.z) / resolution.z * METRES_PER_CENTIMETRE}; + grid.periodic = {false, false, false}; + grid.z_up = true; + grid.cells = cells; + EntitySet voxels(cells); + std::vector ids(static_cast(shape[0] * shape[1] * shape[2])); + std::iota(ids.begin(), ids.end(), 0); + voxels.set("id", ids); + DataStructure soil(id, "regular_grid"); + soil.platform = "CPlantBox"; + soil.grid = grid; + soil.entity_sets.push_back(voxels); + return soil; +} + +Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, + const DataStructure &soil, const std::string &name) { + const auto &resolution = segments.resolution; + const auto nx = std::int64_t(resolution.x), ny = std::int64_t(resolution.y), nz = std::int64_t(resolution.z); + const auto &segment_set = plant.entity_set("SubOrgan"); + const auto &ids = segment_set.column("id").as(); + const std::set known(ids.begin(), ids.end()); + std::vector sources, targets; + for (const auto &[segment, cell] : segments.seg2cell) { + if (cell < 0) continue; // above the soil + const auto id = SEGMENT_ID_OFFSET + segments.segments.at(static_cast(segment)).y; + if (!known.count(id)) + throw Error("segment " + std::to_string(segment) + " of the mapped segments is not in '" + plant.id + + "': set the grid without cutting segments"); + // CPlantBox's cell index has x fastest, then y, then z from the bottom: fspm-TF's is C order of (x, y, z) + const std::int64_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + sources.push_back(id); + targets.push_back((i * ny + j) * nz + k); + } + Mapping mapping(name, {plant.id, "SubOrgan"}, {soil.id, soil.grid ? soil.grid->cells : "Voxel"}, "containment"); + mapping.set("source", sources).set("target", targets).set("weight", std::vector(sources.size(), 1.0)); + return mapping; +} + +// ── A graph as a static root system ────────────────────────────────────── + +namespace { + +/// The factor converting a length unit to centimetres. +double to_centimetres(const std::string &unit) { + if (unit == "m") return 100.; + if (unit == "cm") return 1.; + if (unit == "mm") return 0.1; + if (unit == "um") return 1e-4; + throw Error("length unit '" + unit + "' not converted to cm (m, cm, mm, um)"); +} + +/// The column of `set` holding the standard name (or, failing that, named `fallback`); nullptr if none. +const std::pair *find_variable(const EntitySet &set, const std::string &standard_name, + const std::string &fallback) { + for (const auto &item : set.columns) { + auto declared = set.variables.find(item.first); + if (declared != set.variables.end() && declared->second.standard_name == standard_name) return &item; + } + for (const auto &item : set.columns) + if (item.first == fallback) return &item; + return nullptr; +} + +std::vector as_doubles(const Column &column) { + return std::visit([](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) + return std::vector(values.begin(), values.end()); + else + throw Error("a numeric column expected"); + return std::vector{}; + }, column.values); +} + +std::vector centimetres(const EntitySet &set, const std::pair &item) { + auto values = as_doubles(item.second); + auto declared = set.variables.find(item.first); + const double factor = to_centimetres(declared == set.variables.end() ? "m" : declared->second.unit); + for (auto &value : values) value *= factor; + return values; +} + +} // namespace + +std::shared_ptr to_root_system(const DataStructure &data_structure) { + // The segments: the entity set with the segment ends; the organs: the one their complex points to + const EntitySet *segments = nullptr; + const char *ends[6][2] = {{"proximal_end_x", "x1"}, {"proximal_end_y", "y1"}, {"proximal_end_z", "z1"}, + {"distal_end_x", "x2"}, {"distal_end_y", "y2"}, {"distal_end_z", "z2"}}; + for (const auto &set : data_structure.entity_sets) { + bool all = set.has_column("id") && set.has_column("complex"); + for (const auto &end : ends) all = all && find_variable(set, end[0], end[1]); + if (all && (!segments || (set.rank && segments->rank && *set.rank > *segments->rank))) segments = &set; + } + if (!segments) throw Error("'" + data_structure.id + "': no entity set with segment ends and complexes"); + std::vector> coordinates; + for (const auto &end : ends) coordinates.push_back(centimetres(*segments, *find_variable(*segments, end[0], end[1]))); + const auto &segment_ids = segments->column("id").as(); + const auto &segment_complexes = segments->column("complex").as(); + std::vector segment_parents(segment_ids.size(), NO_ID); + if (segments->has_column("parent")) segment_parents = segments->column("parent").as(); + std::vector segment_radii(segment_ids.size(), 0.05); + if (auto radius = find_variable(*segments, "radius", "radius")) segment_radii = centimetres(*segments, *radius); + + const EntitySet *organs = nullptr; + const std::set complex_ids(segment_complexes.begin(), segment_complexes.end()); + for (const auto &set : data_structure.entity_sets) { + if (&set == segments || !set.has_column("id")) continue; + const auto &ids = set.column("id").as(); + if (std::all_of(complex_ids.begin(), complex_ids.end(), + [&](std::int64_t id) { return std::find(ids.begin(), ids.end(), id) != ids.end(); })) { + organs = &set; + break; + } + } + std::map organ_types; // CPlantBox's own frames: roots only + if (organs && organs->has_column("organType")) { + const auto &ids = organs->column("id").as(); + const auto types = as_doubles(organs->column("organType")); + for (std::size_t row = 0; row < ids.size(); ++row) organ_types[ids[row]] = int(types[row]); + } + + // Each organ's segments, in order along it (from the one whose parent is outside the organ) + std::map row_of; + for (std::size_t row = 0; row < segment_ids.size(); ++row) row_of[segment_ids[row]] = row; + std::map> rows_of_organ; + std::map next_in_organ; + std::vector organ_order; // organs, in the order of their first segment + for (std::size_t row = 0; row < segment_ids.size(); ++row) { + const auto organ = segment_complexes[row]; + if (organ_types.count(organ) && organ_types[organ] != CPlantBox::Organism::ot_root) continue; + if (!rows_of_organ.count(organ)) organ_order.push_back(organ); + rows_of_organ[organ].push_back(row); + const auto parent = row_of.find(segment_parents[row]); + if (parent != row_of.end() && segment_complexes[parent->second] == organ) next_in_organ[parent->second] = row; + } + + auto plant = std::make_shared(); + plant->initializeReader(); + auto seed = std::make_shared(plant); + plant->addOrgan(seed); + std::map> organ_of; + std::map, int>> node_of_segment; // row -> organ, node + // Parents first: an organ's first segment's parent belongs to an organ already built + std::vector pending = organ_order, deferred; + std::size_t progress = 1; + while (!pending.empty() && progress) { + progress = 0; + deferred.clear(); + for (const auto organ : pending) { + const auto &rows = rows_of_organ[organ]; + std::size_t first = rows.front(); + for (const auto row : rows) { + const auto parent = row_of.find(segment_parents[row]); + if (parent == row_of.end() || segment_complexes[parent->second] != organ) { + first = row; + break; + } + } + const auto parent_segment = row_of.find(segment_parents[first]); + std::shared_ptr parent_organ = seed; + int parent_node = 0; + if (parent_segment != row_of.end()) { + const auto found = node_of_segment.find(parent_segment->second); + if (found == node_of_segment.end()) { // its parent organ is not built yet + deferred.push_back(organ); + continue; + } + parent_organ = found->second.first; + parent_node = found->second.second; + } + std::vector chain = {first}; + while (next_in_organ.count(chain.back())) chain.push_back(next_in_organ[chain.back()]); + double length = 0., radius = 0.; + for (const auto row : chain) { + length += std::sqrt(std::pow(coordinates[3][row] - coordinates[0][row], 2) + + std::pow(coordinates[4][row] - coordinates[1][row], 2) + + std::pow(coordinates[5][row] - coordinates[2][row], 2)); + radius += segment_radii[row] / double(chain.size()); + } + auto parameter = std::make_shared(0, 0., length, std::vector{}, + 1e6, radius, 0., 1e6, false); + auto root = std::make_shared(plant->getOrganIndex(), parameter, length, parent_node); + root->setOrganism(plant); + if (parent_segment != row_of.end()) + root->addNode(parent_organ->getNode(parent_node), parent_organ->getNodeId(parent_node), 0.); + else + root->addNode(CPlantBox::Vector3d(coordinates[0][first], coordinates[1][first], coordinates[2][first]), 0.); + for (const auto row : chain) { + root->addNode(CPlantBox::Vector3d(coordinates[3][row], coordinates[4][row], coordinates[5][row]), 0.); + node_of_segment[row] = {root, root->getNumberOfNodes() - 1}; + } + parent_organ->addChild(root); + organ_of[organ] = root; + ++progress; + } + pending = deferred; + } + if (!pending.empty()) throw Error("'" + data_structure.id + "': organs whose parent segments are not found"); + return plant; +} + +} // namespace fspm_tf::cplantbox diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp new file mode 100644 index 0000000..23b5082 --- /dev/null +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -0,0 +1,211 @@ +// CPlantBox's adapter: a simulated plant exported (organs, segments, variables, geometry, soil and mapping), written +// and read back; a static root system rebuilt from its own frame and from another platform's (the conformance root +// system, three scales). +// +// fspm_tf_cplantbox_tests CONFORMANCE MODELPARAMETER OUT +#include +#include +#include +#include +#include +#include +#include + +#include "fspm_tf/cplantbox.hpp" + +#include "MappedOrganism.h" +#include "Plant.h" + +namespace filesystem = std::filesystem; +using namespace fspm_tf; + +namespace { + +int failures = 0, checks = 0; + +void check(bool condition, const std::string &what) { + ++checks; + if (!condition) { + ++failures; + std::cerr << "FAILED: " << what << "\n"; + } +} + +std::shared_ptr grown_plant(const std::string ¶meters, double days) { + auto plant = std::make_shared(1); + plant->readParameters(parameters, "plant", true, false); + plant->initialize(false); + plant->simulate(days, false); + return plant; +} + +/// The column of an entity set as doubles. +std::vector values(const EntitySet &set, const std::string &name) { + return std::visit([](const auto &column) { + using T = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v && !std::is_same_v) return std::vector(column.begin(), column.end()); + return std::vector{}; + }, set.column(name).values); +} + +/// The topology is valid: ids unique, complexes and parents found, edge types consistent. +void check_topology(const DataStructure &data_structure, const std::string &what) { + std::set all; + std::size_t total = 0; + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + all.insert(ids.begin(), ids.end()); + total += ids.size(); + } + check(all.size() == total, what + ": ids unique across entity sets"); + for (const auto &set : data_structure.entity_sets) { + const auto ids = set.ids(); + const std::set own(ids.begin(), ids.end()); + if (set.has_column("parent")) { + const auto &parents = set.column("parent").as(); + const auto &edges = set.column("edge_type").as(); + bool good = true; + for (std::size_t row = 0; row < parents.size(); ++row) + good = good && (parents[row] == NO_ID ? edges[row] == "none" : own.count(parents[row]) && edges[row] != "none"); + check(good, what + ": parents of " + set.name + " in their set, with their edge types"); + } + if (set.has_column("complex")) { + bool good = true; + for (const auto complex : set.column("complex").as()) good = good && (complex == NO_ID || all.count(complex)); + check(good, what + ": complexes of " + set.name + " found"); + } + } +} + +void test_root_system(const filesystem::path ¶meters, const filesystem::path &out) { + const auto plant = grown_plant((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), 30.); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + const Frame frame = cplantbox::to_frame(*plant, options); + const auto &data_structure = frame.data_structures[0]; + const auto &segments = data_structure.entity_set("SubOrgan"); + check(segments.row_count() == std::size_t(plant->getNumberOfSegments()), "one row per CPlantBox segment"); + check(data_structure.entity_set("Organ").row_count() >= plant->getOrgans().size(), "every organ (and the seed)"); + check_topology(data_structure, "root system"); + double total = 0.; + for (const auto length : values(segments, "length")) total += length; + double organs_total = 0.; + for (const auto length : plant->getParameter("length")) organs_total += length; + check(std::abs(total - organs_total) < 1e-9 * organs_total, "segment lengths sum to the organs' lengths"); + check(frame.time == 30., "the frame's time: the simulation time, days"); + + write_frame((out / "anagallis.glb").string(), frame); // written, read back the same + const auto read_back = read_frame((out / "anagallis.glb").string()); + check(compare(frame_to_json(read_back), frame_to_json(frame)).empty(), "the root system read back as written"); + + // Rebuilt as a static root system (CPlantBox has static roots only: the stem is left out), and exported again: + // the same root segments (ends to the nanometre) + const auto rebuilt = cplantbox::to_root_system(read_back.data_structures[0]); + const auto organ_types = values(segments, "organType"); + const auto root_segments = std::count(organ_types.begin(), organ_types.end(), double(CPlantBox::Organism::ot_root)); + check(rebuilt->getNumberOfSegments() == root_segments, + "the static root system has every root segment (" + std::to_string(rebuilt->getNumberOfSegments()) + " of " + + std::to_string(root_segments) + ")"); + const auto again = cplantbox::to_data_structure(*rebuilt); + const auto &rebuilt_segments = again.entity_set("SubOrgan"); + double root_total = 0., rebuilt_total = 0.; + const auto lengths = values(segments, "length"); + for (std::size_t row = 0; row < lengths.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) root_total += lengths[row]; + for (const auto length : values(rebuilt_segments, "length")) rebuilt_total += length; + check(std::abs(rebuilt_total - root_total) < 1e-9 * root_total, "the static root system has the same root length"); + std::multiset ends, rebuilt_ends; + for (const char *name : {"x2", "y2", "z2"}) { + const auto coordinates = values(segments, name); + for (std::size_t row = 0; row < coordinates.size(); ++row) + if (organ_types[row] == CPlantBox::Organism::ot_root) ends.insert(std::llround(coordinates[row] * 1e9)); + for (const auto value : values(rebuilt_segments, name)) rebuilt_ends.insert(std::llround(value * 1e9)); + } + check(ends == rebuilt_ends, "the static root system's segment ends are the same"); +} + +void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "fspm2023.xml").string(), "plant", true, false); + plant->initialize(false); + plant->simulate(28., false); + auto data_structure = cplantbox::to_data_structure(*plant); + cplantbox::add_geometry(data_structure, plant); + check(data_structure.meshes.size() == 1 && data_structure.meshes[0].vertex_count() > 0, "a mesh of the shoot"); + const auto &rows = data_structure.meshes[0].feature_rows[0].second; + std::size_t with_segment = 0; + for (const auto row : rows) with_segment += row != NO_ID; + check(with_segment > rows.size() / 2, "most vertices carry their segment"); + check_topology(data_structure, "shoot and roots"); + + plant->setRectangularGrid(CPlantBox::Vector3d(-10, -10, -40), CPlantBox::Vector3d(10, 10, 0), + CPlantBox::Vector3d(4, 4, 8), false); + const auto soil = cplantbox::soil_grid(*plant); + const auto mapping = cplantbox::segments_in_soil(*plant, data_structure, soil); + check(soil.entity_set("Voxel").row_count() == 4 * 4 * 8, "the soil grid's cells"); + check(mapping.row_count() > 0, "segments mapped to soil cells"); + bool inside = true; // each mapped segment's midpoint is in its cell + const auto &segments = data_structure.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = segments.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + const auto x1 = values(segments, "x1"), y1 = values(segments, "y1"), z1 = values(segments, "z1"); + const auto x2 = values(segments, "x2"), y2 = values(segments, "y2"), z2 = values(segments, "z2"); + const auto &grid = *soil.grid; + const auto &sources = mapping.column("source").as(); + const auto &targets = mapping.column("target").as(); + for (std::size_t item = 0; item < sources.size(); ++item) { + const auto row = row_of.at(sources[item]); + const double middle[3] = {(x1[row] + x2[row]) / 2, (y1[row] + y2[row]) / 2, (z1[row] + z2[row]) / 2}; + std::int64_t index[3]; + for (int axis = 0; axis < 3; ++axis) + index[axis] = std::int64_t(std::floor((middle[axis] - grid.origin[axis]) / grid.spacing[axis])); + inside = inside && targets[item] == (index[0] * grid.shape[1] + index[1]) * grid.shape[2] + index[2]; + } + check(inside, "each mapped segment's midpoint lies in its cell (C order of x, y, z)"); + + Frame frame; + frame.time = plant->getSimTime(); + frame.data_structures = {data_structure, soil}; + frame.mappings = {mapping}; + write_frame((out / "fspm2023.glb").string(), frame); + const auto read_back = read_frame((out / "fspm2023.glb").string()); + check(compare(frame_to_json(read_back), frame_to_json(frame)).empty(), "the plant, soil and mapping read back"); +} + +void test_another_platform(const filesystem::path &conformance) { + // The conformance root system: Plant, Axis and SubOrgan scales (metafspm's layout), segments in metres + const auto frame = read_frame((conformance / "root_system.glb").string()); + const auto &data_structure = frame.data_structures[0]; + const auto rebuilt = cplantbox::to_root_system(data_structure); + const auto &segments = data_structure.entity_set("SubOrgan"); + check(rebuilt->getNumberOfSegments() == int(segments.row_count()), "another platform's segments, one each"); + check(int(rebuilt->getOrgans().size()) == int(data_structure.entity_set("Axis").row_count()), + "another platform's axes as CPlantBox organs"); + double total = 0., rebuilt_total = 0.; + for (const auto length : values(segments, "length")) total += length * 100.; // m -> cm + for (const auto length : rebuilt->getParameter("length")) rebuilt_total += length; + check(std::abs(total - rebuilt_total) < 1e-6 * total, "another platform's root length, in cm"); +} + +} // namespace + +int main(int argument_count, char **arguments) { + if (argument_count != 4) { + std::cerr << "usage: fspm_tf_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; + return 2; + } + const filesystem::path conformance = arguments[1], parameters = arguments[2], out = arguments[3]; + filesystem::create_directories(out); + for (auto test : {std::function([&] { test_root_system(parameters, out); }), + std::function([&] { test_plant_with_geometry_and_soil(parameters, out); }), + std::function([&] { test_another_platform(conformance); })}) { + try { + test(); + } catch (const std::exception &error) { + check(false, std::string("exception: ") + error.what()); + } + } + std::cout << checks - failures << " of " << checks << " checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/bindings/cpp/include/fspm_tf/fspm_tf.hpp b/bindings/cpp/include/fspm_tf/fspm_tf.hpp new file mode 100644 index 0000000..93eb769 --- /dev/null +++ b/bindings/cpp/include/fspm_tf/fspm_tf.hpp @@ -0,0 +1,77 @@ +// Reading and writing fspm-TF files in C++: whole functional-structural plant model scenes (multiscale plant graphs, +// their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 +// binary (GLB) file per time step. The specification is spec/specification.md in the fspm-TF repository. +// +// read_frame(path) -> Frame write_frame(path, frame) +// from_bytes(data) -> Frame to_bytes(frame) -> bytes +// read_index(path), read_sequence(path) write_sequence(directory, frames) +// frame_to_json(frame) the layout of the conformance files' expected values +#pragma once + +#include +#include +#include +#include + +#include "fspm_tf/model.hpp" + +namespace fspm_tf { + +// ── The GLB container ───────────────────────────────────────────────────── + +/// The bytes of a GLB file from its JSON document and its binary buffer (both padded to 8 bytes). +std::vector pack(const json &document, const std::vector &binary); +/// The JSON document (keys in their order) and the binary buffer of a GLB file's bytes. +std::pair> unpack(const std::vector &data); + +// ── Frames ──────────────────────────────────────────────────────────────── + +/// The GLB file of a frame. +std::vector to_bytes(const Frame &frame); +/// The frame of a GLB file's bytes. +Frame from_bytes(const std::vector &data); +/// Writes a frame to the GLB file `path`. +void write_frame(const std::string &path, const Frame &frame); +/// The frame of the GLB file `path`. +Frame read_frame(const std::string &path); + +// ── Sequences ───────────────────────────────────────────────────────────── + +/// The name of a sequence's index file. +inline constexpr const char *INDEX_FILE = "scene.fspm.json"; + +/// One frame of a sequence's index. +struct IndexEntry { + double time = 0.0; + std::int64_t step = 0; + std::string file; + std::string path; ///< the file's absolute path (when read) +}; + +/// A sequence's index: its frames, in time order. +struct Index { + std::string version = SPECIFICATION_VERSION; + std::string time_unit = "d"; + std::vector frames; +}; + +/// Writes the index of a sequence. +void write_index(const std::string &path, const Index &index); +/// The index of a sequence, each frame with the absolute path of its file. +Index read_index(const std::string &path); +/// Writes frames to `directory`, one GLB each (name_000000.glb, ...), and their index; the index's path. +std::string write_sequence(const std::string &directory, const std::vector &frames, + const std::string &name = "frame"); +/// The frames of the sequence whose index is `path`. +std::vector read_sequence(const std::string &path); + +// ── The conformance files' layout ───────────────────────────────────────── + +/// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): +/// every table's columns as lists, NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. +json frame_to_json(const Frame &frame); +/// The differences between two JSON values, as `fspm-tf compare` finds them: objects by key, arrays in order, +/// numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). +std::vector compare(const json &actual, const json &expected, const std::string &path = "$"); + +} // namespace fspm_tf diff --git a/bindings/cpp/include/fspm_tf/model.hpp b/bindings/cpp/include/fspm_tf/model.hpp new file mode 100644 index 0000000..b5617f8 --- /dev/null +++ b/bindings/cpp/include/fspm_tf/model.hpp @@ -0,0 +1,235 @@ +// The fspm-TF data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +// +// Frame one time step of a scene +// DataStructure one plant population's graph, or one environment model's grid +// EntitySet the entities of one scale (or a grid's cells): a table of columns +// Relation an edge table between entity sets (cell adjacency, a branch's bearer, ...) +// Table the data structure's scalars: a table of one row +// Grid a regular or adaptive grid's layout +// Mesh triangles, each vertex carrying the row of its entity in one or several entity sets +// Mapping source entity <-> target entity weights between two data structures +// +// A column holds one value per row (numbers, booleans, text), or a fixed-size vector per row (`width` values per +// row, row after row). +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace fspm_tf { + +using json = nlohmann::ordered_json; + +/// The version of the specification this library reads and writes. +inline constexpr const char *SPECIFICATION_VERSION = "0.1"; +/// The id of no entity: the complex or parent of an entity that has none, the feature row of a vertex with none. +inline constexpr std::int64_t NO_ID = -1; +/// The values of edge_type: the root of its scale, MTG '<', MTG '+'. +inline const std::vector EDGE_TYPES = {"none", "successor", "branch"}; + +/// What the specification forbids, or a file that is not fspm-TF. +struct Error : std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/// The values of a column: one vector per element type. Booleans are std::vector; text, labels and edge types +/// std::vector. +using ColumnValues = std::variant, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector, + std::vector, std::vector, std::vector>; + +/// A column: its values, `width` per row (1: one value per row; >= 2: a fixed-size vector per row). +struct Column { + ColumnValues values; + std::size_t width = 1; + + Column() : values(std::vector{}) {} + template Column(std::vector column_values, std::size_t width = 1) + : values(std::move(column_values)), width(width) {} + Column(std::initializer_list text) : values(std::vector(text.begin(), text.end())) {} + + /// The number of values (rows × width). + std::size_t size() const; + /// The number of rows. + std::size_t rows() const { return width == 0 ? 0 : size() / width; } + /// The values as a vector of T (throws Error when the column holds another type). + template const std::vector &as() const { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template std::vector &as() { + if (auto found = std::get_if>(&values)) return *found; + throw Error("a column of another type than the one asked for"); + } + template bool holds() const { return std::holds_alternative>(values); } + /// The column's type name in the conformance files' expected values: numpy's (float64, int64, bool...), or + /// "string". + std::string dtype() const; +}; + +/// The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind +/// (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance. +struct Variable { + std::string unit = "1"; + std::optional standard_name; + std::optional kind; + std::string role = "state"; + std::optional description; + std::optional minimum; + std::optional maximum; + std::vector references; + std::vector doi; + + bool operator==(const Variable &other) const; +}; + +enum class TableKind { scalars, entity_set, relation, mapping }; + +/// Columns of equal length, in their order, and the declarations of its variables. +struct Table { + std::string name; + std::vector> columns; + std::map variables; + TableKind table_kind = TableKind::scalars; + + Table() = default; + explicit Table(std::string name, TableKind kind = TableKind::scalars) : name(std::move(name)), table_kind(kind) {} + + /// The number of rows (0 without columns). + std::size_t row_count() const; + bool has_column(const std::string &column_name) const; + const Column &column(const std::string &column_name) const; + Column &column(const std::string &column_name); + /// Sets the column `column_name`, replacing it or appending it; returns the table, for chaining. + Table &set(const std::string &column_name, Column values); + /// Declares the variable `column_name`. + Table &declare(const std::string &column_name, Variable variable); + /// The columns whose meaning the specification fixes, for this kind of table. + const std::vector &reserved() const; + bool is_reserved(const std::string &column_name) const; + /// The names of the variable columns (the columns but the reserved ones). + std::vector variable_names() const; + /// Throws Error when columns have different numbers of rows. + void check_lengths() const; +}; + +/// The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (no rank). +/// Reserved columns: id (int64, stable across frames, never reused), label (text), complex (the id of the containing +/// entity at a coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), +/// edge_type ("none", "successor", "branch"); an adaptive grid's cells: level and index. +struct EntitySet : Table { + std::optional rank; + + EntitySet() : Table("", TableKind::entity_set) {} + explicit EntitySet(std::string name, std::optional rank = std::nullopt) + : Table(std::move(name), TableKind::entity_set), rank(rank) {} + /// The ids of its entities: its id column, or a grid's row numbers (from 0) when it has none. + std::vector ids() const; +}; + +/// An edge table: source and target entity ids (in the entity sets source_set and target_set), an optional id per +/// edge, an optional relation label per edge, and edge variables. +struct Relation : Table { + std::string source_set; + std::string target_set; + + Relation() : Table("", TableKind::relation) {} + Relation(std::string name, std::string source_set, std::string target_set) + : Table(std::move(name), TableKind::relation), source_set(std::move(source_set)), + target_set(std::move(target_set)) {} +}; + +/// Weights between the entities of two data structures: source and target ids, weight. source and target: (data +/// structure id, entity set name). +struct Mapping : Table { + std::pair source; + std::pair target; + std::optional method; + + Mapping() : Table("", TableKind::mapping) {} + Mapping(std::string name, std::pair source, std::pair target, + std::optional method = std::nullopt) + : Table(std::move(name), TableKind::mapping), source(std::move(source)), target(std::move(target)), + method(std::move(method)) {} +}; + +/// A grid's layout. Regular: shape cells of size spacing from origin, along axes, periodic per axis, z pointing up or +/// down (z_up). Adaptive: the same for its base level, refined up to max_level, its leaf cells (an entity set with +/// level and index columns) explicit. cells: the name of the cells' entity set. +struct Grid { + std::string kind = "regular"; + std::vector shape; + std::vector origin; + std::vector spacing; + std::vector axes = {"x", "y", "z"}; + std::vector periodic; + bool z_up = true; + std::int64_t max_level = 0; + std::string cells = "Cell"; +}; + +/// Triangles: positions (3 per vertex, in the data structure's frame), three consecutive vertices per triangle, or +/// indices (3 per triangle, from 0) into them; feature_rows: per entity set, the row (from 0) of each vertex's +/// entity in that set, NO_ID for none; display: per variable, its float copy per vertex, from the entity set +/// display_sources[variable]. +struct Mesh { + std::string name; + std::vector positions; + std::vector>> feature_rows; + std::optional> indices; + std::vector>> display; + std::map display_sources; + + std::size_t vertex_count() const { return positions.size() / 3; } +}; + +/// One plant population's graph, or one environment model's grid. origin: its frame in the scene, a 4×4 matrix row +/// by row. extras: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); +/// other readers keep them without interpreting them. +struct DataStructure { + std::string id; + std::string kind = "graph"; + std::vector entity_sets; + std::vector relations; + std::optional scalars; + std::optional grid; + std::vector meshes; + std::array origin = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + std::optional platform; + std::optional model; + json extras = json::object(); + + DataStructure() = default; + explicit DataStructure(std::string id, std::string kind = "graph") : id(std::move(id)), kind(std::move(kind)) {} + const EntitySet &entity_set(const std::string &name) const; + EntitySet &entity_set(const std::string &name); +}; + +/// A scene at one time step (time in time_unit, step), or a parameter set. unknown: the content of a read file this +/// library does not interpret (other root extensions, the root's extras), written back as it was. +struct Frame { + std::vector data_structures; + std::vector mappings; + double time = 0.0; + std::string time_unit = "d"; + std::int64_t step = 0; + json writer = json::object(); + std::optional vocabulary_version; + json unknown = json::object(); + + const DataStructure &data_structure(const std::string &id) const; + DataStructure &data_structure(const std::string &id); +}; + +} // namespace fspm_tf diff --git a/bindings/cpp/src/bytes.hpp b/bindings/cpp/src/bytes.hpp new file mode 100644 index 0000000..b7e0abb --- /dev/null +++ b/bindings/cpp/src/bytes.hpp @@ -0,0 +1,56 @@ +// Little-endian values in byte buffers, whatever the machine's byte order. +#pragma once + +#include +#include +#include +#include +#include + +#include "fspm_tf/model.hpp" + +namespace fspm_tf::detail { + +inline bool machine_is_little_endian() { + const std::uint16_t probe = 1; + std::uint8_t first; + std::memcpy(&first, &probe, 1); + return first == 1; +} + +/// Appends `count` values of type T, little-endian. +template void append_values(std::vector &bytes, const T *values, std::size_t count) { + const std::size_t start = bytes.size(); + bytes.resize(start + count * sizeof(T)); + if (count == 0) return; + std::memcpy(bytes.data() + start, values, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (std::size_t item = 0; item < count; ++item) + std::reverse(bytes.begin() + start + item * sizeof(T), bytes.begin() + start + (item + 1) * sizeof(T)); +} + +/// The `count` values of type T stored little-endian in `bytes` (of `available` bytes) from byte `offset`. +template +std::vector read_values(const std::uint8_t *bytes, std::size_t available, std::size_t count, + std::size_t offset = 0) { + if (offset + count * sizeof(T) > available) + throw Error("a buffer view holds " + std::to_string(available) + " bytes, " + + std::to_string(offset + count * sizeof(T)) + " needed"); + std::vector values(count); + if (count == 0) return values; + std::memcpy(values.data(), bytes + offset, count * sizeof(T)); + if (!machine_is_little_endian() && sizeof(T) > 1) + for (auto &value : values) { + auto raw = reinterpret_cast(&value); + std::reverse(raw, raw + sizeof(T)); + } + return values; +} + +inline std::uint32_t read_word(const std::vector &data, std::size_t offset) { + return read_values(data.data(), data.size(), 1, offset)[0]; +} + +inline std::size_t padding(std::size_t length) { return (8 - length % 8) % 8; } + +} // namespace fspm_tf::detail diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp new file mode 100644 index 0000000..2e767a4 --- /dev/null +++ b/bindings/cpp/src/codec.cpp @@ -0,0 +1,776 @@ +// Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), +// meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data +// structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, +// standard names, kinds and roles. +#include "fspm_tf/fspm_tf.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bytes.hpp" + +namespace fspm_tf { + +namespace { + +const std::string EXTENSION = "FSPM_scene"; +const std::string METADATA = "EXT_structural_metadata"; +const std::string FEATURES = "EXT_mesh_features"; +const std::string EDGE_TYPE_ENUM = "FSPM_edge_type"; +const std::vector ENUM_COLUMNS = {"label", "edge_type", "relation"}; +const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index"}; +constexpr int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; +constexpr int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; +constexpr std::int64_t MAX_FLOAT_FEATURE = std::int64_t(1) << 24; // feature rows are float vertex attributes +// The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about +// x, column-major). Readers of the tables use the scene's frame, not this node. +const std::vector Z_UP_TO_Y_UP = {1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1}; + +bool contains(const std::vector &names, const std::string &name) { + return std::find(names.begin(), names.end(), name) != names.end(); +} + +void check_identifier(const std::string &name, const std::string &what) { + static const std::regex identifier("^[a-zA-Z_][a-zA-Z0-9_]*$"); + if (!std::regex_match(name, identifier)) + throw Error(what + " '" + name + "' must be an identifier (letters, digits, '_'; not starting with a digit)"); +} + +template struct Component; +template <> struct Component { static constexpr const char *name = "INT8"; }; +template <> struct Component { static constexpr const char *name = "UINT8"; }; +template <> struct Component { static constexpr const char *name = "INT16"; }; +template <> struct Component { static constexpr const char *name = "UINT16"; }; +template <> struct Component { static constexpr const char *name = "INT32"; }; +template <> struct Component { static constexpr const char *name = "UINT32"; }; +template <> struct Component { static constexpr const char *name = "INT64"; }; +template <> struct Component { static constexpr const char *name = "UINT64"; }; +template <> struct Component { static constexpr const char *name = "FLOAT32"; }; +template <> struct Component { static constexpr const char *name = "FLOAT64"; }; + +template constexpr bool is_number_v = std::is_arithmetic_v && !std::is_same_v; + +// EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; +inf +// and -inf as the largest finite value and the next one after the most negative, declared in the property's extras +// (FSPM_infinity). +template T no_data_float() { return std::numeric_limits::lowest(); } +template T positive_stand_in() { return std::numeric_limits::max(); } +template T negative_stand_in() { + return std::nextafter(std::numeric_limits::lowest(), T(0)); +} + +json get(const json &object, const std::string &key, json fallback = nullptr) { + if (object.is_object()) { + auto found = object.find(key); + if (found != object.end()) return *found; + } + return fallback; +} + +/// Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. +struct BufferBuilder { + std::vector data; + json views = json::array(); + + std::int64_t add_bytes(const std::vector &bytes, int target = 0) { + data.resize(data.size() + detail::padding(data.size()), 0); + json view = {{"buffer", 0}, {"byteOffset", data.size()}, {"byteLength", bytes.size()}}; + if (target != 0) view["target"] = target; + data.insert(data.end(), bytes.begin(), bytes.end()); + views.push_back(view); + return static_cast(views.size()) - 1; + } + template std::int64_t add(const std::vector &values, int target = 0) { + std::vector bytes; + detail::append_values(bytes, values.data(), values.size()); + return add_bytes(bytes, target); + } +}; + +json declaration_json(const Variable &variable) { + json declaration = {{"unit", variable.unit}, {"role", variable.role}}; + if (variable.standard_name) declaration["standardName"] = *variable.standard_name; + if (variable.kind) declaration["kind"] = *variable.kind; + if (variable.minimum) declaration["minimum"] = *variable.minimum; + if (variable.maximum) declaration["maximum"] = *variable.maximum; + if (!variable.references.empty()) declaration["references"] = variable.references; + if (!variable.doi.empty()) declaration["doi"] = variable.doi; + return declaration; +} + +std::pair> enum_codes(const std::string &name, const std::vector &values, + const std::string &class_id, json &schema, + std::string &value_type) { + if (name == "edge_type") { + std::vector codes; + for (const auto &value : values) { + auto found = std::find(EDGE_TYPES.begin(), EDGE_TYPES.end(), value); + if (found == EDGE_TYPES.end()) + throw Error("edge_type of '" + class_id + "': unknown value '" + value + + "' (allowed: none, successor, branch)"); + codes.push_back(static_cast(found - EDGE_TYPES.begin())); + } + value_type = "UINT8"; + return {EDGE_TYPE_ENUM, codes}; + } + std::set sorted(values.begin(), values.end()); + std::vector names(sorted.begin(), sorted.end()); + const std::string enum_name = class_id + "__" + name; + value_type = names.size() <= 256 ? "UINT8" : "UINT16"; + json enum_values = json::array(); + std::map code_of; + for (std::size_t code = 0; code < names.size(); ++code) { + enum_values.push_back({{"name", names[code]}, {"value", code}}); + code_of[names[code]] = static_cast(code); + } + schema["enums"][enum_name] = {{"valueType", value_type}, {"values", enum_values}}; + std::vector bytes; + for (const auto &value : values) { + const std::uint16_t code = code_of[value]; + if (value_type == "UINT8") { + const auto narrow = static_cast(code); + detail::append_values(bytes, &narrow, 1); + } else { + detail::append_values(bytes, &code, 1); + } + } + return {enum_name, bytes}; +} + +/// The class properties and property table values of a table's columns. +std::pair encode_table(const Table &table, const std::string &class_id, BufferBuilder &buffers, + json &schema) { + table.check_lengths(); + json properties = json::object(), stored = json::object(); + for (const auto &[name, column] : table.columns) { + check_identifier(name, "column of '" + table.name + "'"); + json property = json::object(); + auto declared = table.variables.find(name); + if (declared != table.variables.end() && declared->second.description) + property["description"] = *declared->second.description; + const bool reserved = table.is_reserved(name); + const std::string where = "column '" + name + "' of '" + table.name + "'"; + if (reserved && contains(ENUM_COLUMNS, name)) { + if (!column.holds()) throw Error(where + ": labels are text"); + std::string value_type; + auto [enum_name, codes] = enum_codes(name, column.as(), class_id, schema, value_type); + property["type"] = "ENUM"; + property["enumType"] = enum_name; + stored[name] = {{"values", buffers.add_bytes(codes)}}; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": text vectors are not supported"); + property["type"] = "STRING"; + std::vector text; + std::vector offsets = {0}; + for (const auto &value : column.as()) { + text.insert(text.end(), value.begin(), value.end()); + offsets.push_back(text.size()); + } + json item = {{"values", buffers.add_bytes(text)}}; + if (offsets.back() < (std::uint64_t(1) << 32)) { + item["stringOffsets"] = buffers.add(std::vector(offsets.begin(), offsets.end())); + item["stringOffsetType"] = "UINT32"; + } else { + item["stringOffsets"] = buffers.add(offsets); + item["stringOffsetType"] = "UINT64"; + } + stored[name] = item; + } else if (column.holds()) { + if (column.width != 1) throw Error(where + ": boolean vectors are not supported"); + property["type"] = "BOOLEAN"; + const auto &values = column.as(); + std::vector packed((values.size() + 7) / 8, 0); + for (std::size_t row = 0; row < values.size(); ++row) + if (values[row]) packed[row / 8] |= static_cast(1u << (row % 8)); // bit order: little + stored[name] = {{"values", buffers.add_bytes(packed)}}; + } else { + std::visit([&](const auto &values) { + using T = typename std::decay_t::value_type; + if constexpr (is_number_v) { + if constexpr (std::is_integral_v && !std::is_same_v) { + if (reserved && contains(ID_COLUMNS, name)) { // ids are INT64, whatever the platform's + property["type"] = "SCALAR"; + property["componentType"] = "INT64"; + if (column.width >= 2) { + property["array"] = true; + property["count"] = column.width; + } + if (name == "complex" || name == "parent") property["noData"] = NO_ID; + stored[name] = {{"values", buffers.add(std::vector(values.begin(), values.end()))}}; + return; + } + } + property["type"] = "SCALAR"; + property["componentType"] = Component::name; + if (column.width != 1) { + if (column.width < 2) throw Error(where + ": vectors have at least 2 components"); + property["array"] = true; + property["count"] = column.width; + } + std::vector data = values; + if constexpr (std::is_floating_point_v) { + const bool infinite = std::any_of(data.begin(), data.end(), [](T value) { return std::isinf(value); }); + if (infinite) { + const T positive = positive_stand_in(), negative = negative_stand_in(); + if (std::any_of(data.begin(), data.end(), + [&](T value) { return value == positive || value == negative; })) + throw Error(where + " holds both infinities and the values standing for them"); + property["extras"] = {{"FSPM_infinity", {{"positive", static_cast(positive)}, + {"negative", static_cast(negative)}}}}; + for (auto &value : data) + if (std::isinf(value)) value = value > 0 ? positive : negative; + } + if (std::any_of(data.begin(), data.end(), [](T value) { return std::isnan(value); })) { + const T no_data = no_data_float(); + if (std::any_of(data.begin(), data.end(), [&](T value) { return value == no_data; })) + throw Error(where + " holds both NaN and the value standing for NaN"); + property["noData"] = static_cast(no_data); + for (auto &value : data) + if (std::isnan(value)) value = no_data; + } + } else if (reserved && (name == "complex" || name == "parent")) { + property["noData"] = NO_ID; + } + stored[name] = {{"values", buffers.add(data)}}; + } + }, column.values); + } + if (name == "id" && table.table_kind == TableKind::entity_set) property["required"] = true; + properties[name] = property; + } + return {properties, stored}; +} + +json grid_json(const Grid &grid) { + json entry = {{"kind", grid.kind}, {"shape", grid.shape}, {"origin", grid.origin}, {"spacing", grid.spacing}, + {"axes", grid.axes}, {"periodic", grid.periodic}, {"zUp", grid.z_up}, {"cells", grid.cells}}; + if (grid.kind == "adaptive") entry["maxLevel"] = grid.max_level; + return entry; +} + +std::string display_attribute(const std::string &name) { + std::string attribute = "_" + name; + std::transform(attribute.begin(), attribute.end(), attribute.begin(), + [](unsigned char character) { return static_cast(std::toupper(character)); }); + return attribute; +} + +json encode_mesh(const Mesh &mesh, const DataStructure &data_structure, const json &entry, BufferBuilder &buffers, + json &accessors) { + const std::size_t vertices = mesh.vertex_count(); + if (mesh.positions.size() != vertices * 3) throw Error("mesh '" + mesh.name + "': 3 coordinates per vertex"); + auto accessor = [&](auto values, const char *type, int component_type, int target, std::size_t count, + bool bounds = false) { + json item = {{"bufferView", buffers.add(values, target)}, {"componentType", component_type}, + {"count", count}, {"type", type}}; + if (bounds && count > 0) { + std::vector minimum(3, std::numeric_limits::infinity()), maximum(3, -minimum[0]); + for (std::size_t vertex = 0; vertex < count; ++vertex) + for (int axis = 0; axis < 3; ++axis) { + minimum[axis] = std::min(minimum[axis], static_cast(values[vertex * 3 + axis])); + maximum[axis] = std::max(maximum[axis], static_cast(values[vertex * 3 + axis])); + } + item["min"] = minimum; + item["max"] = maximum; + } + accessors.push_back(item); + return static_cast(accessors.size()) - 1; + }; + std::vector positions(mesh.positions.begin(), mesh.positions.end()); + json attributes = {{"POSITION", accessor(positions, "VEC3", FLOAT, ARRAY_BUFFER, vertices, true)}}; + std::map tables; + for (const auto &set_entry : entry["entitySets"]) + tables[set_entry["name"].get()] = set_entry["propertyTable"].get(); + json feature_ids = json::array(); + for (std::size_t set_index = 0; set_index < mesh.feature_rows.size(); ++set_index) { + const auto &[set_name, rows] = mesh.feature_rows[set_index]; + if (!tables.count(set_name)) + throw Error("mesh '" + mesh.name + "': no entity set '" + set_name + "' in '" + data_structure.id + "'"); + if (rows.size() != vertices) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' are not one per vertex"); + const auto count = static_cast(data_structure.entity_set(set_name).row_count()); + if (count >= MAX_FLOAT_FEATURE) + throw Error("mesh '" + mesh.name + "': too many entities in '" + set_name + "', feature IDs stop at 2^24"); + std::vector stored; + std::set present; + bool has_null = false; + for (const auto row : rows) { + if (row < NO_ID || row >= count) + throw Error("mesh '" + mesh.name + "': feature rows of '" + set_name + "' outside its rows"); + has_null = has_null || row == NO_ID; + if (row != NO_ID) present.insert(row); + stored.push_back(static_cast(row == NO_ID ? count : row)); // null: one past the last row + } + attributes["_FEATURE_ID_" + std::to_string(set_index)] = accessor(stored, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + json feature = {{"featureCount", present.size()}, {"attribute", set_index}, + {"propertyTable", tables[set_name]}, {"label", set_name}}; + if (has_null) feature["nullFeatureId"] = count; + feature_ids.push_back(feature); + } + for (const auto &[name, values] : mesh.display) { + if (values.size() != vertices) + throw Error("mesh '" + mesh.name + "': display copy '" + name + "' is not one value per vertex"); + attributes[display_attribute(name)] = accessor(values, "SCALAR", FLOAT, ARRAY_BUFFER, vertices); + } + json primitive = {{"attributes", attributes}, {"mode", 4}}; + if (mesh.indices) { + std::vector indices(mesh.indices->begin(), mesh.indices->end()); + primitive["indices"] = accessor(indices, "SCALAR", UNSIGNED_INT, ELEMENT_ARRAY_BUFFER, indices.size()); + } + if (!feature_ids.empty()) primitive["extensions"] = {{FEATURES, {{"featureIds", feature_ids}}}}; + return {{"name", mesh.name}, {"primitives", json::array({primitive})}}; +} + +bool is_identity(const std::array &origin) { + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) + if (origin[row * 4 + column] != (row == column ? 1.0 : 0.0)) return false; + return true; +} + +std::vector column_major(const std::array &origin) { + std::vector values(16); + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) values[column * 4 + row] = origin[row * 4 + column]; + return values; +} + +} // namespace + +// ── Writing ────────────────────────────────────────────────────────────── + +std::vector to_bytes(const Frame &frame) { + BufferBuilder buffers; + json edge_values = json::array(); + for (std::size_t value = 0; value < EDGE_TYPES.size(); ++value) + edge_values.push_back({{"name", EDGE_TYPES[value]}, {"value", value}}); + json schema = {{"id", "fspm_scene"}, {"classes", json::object()}, + {"enums", {{EDGE_TYPE_ENUM, {{"valueType", "UINT8"}, {"values", edge_values}}}}}}; + json tables = json::array(), variables = json::object(); + json document = {{"asset", {{"version", "2.0"}, {"generator", std::string("fspm_tf (C++) ") + SPECIFICATION_VERSION}}}, + {"scene", 0}, {"scenes", json::array({{{"nodes", json::array({0})}}})}, + {"nodes", json::array({{{"name", "fspm_scene"}, {"matrix", Z_UP_TO_Y_UP}, + {"children", json::array()}}})}}; + json accessors = json::array(), meshes = json::array(); + + auto add_table = [&](const Table &table, const std::string &class_id) { + check_identifier(class_id, "class"); + auto [properties, stored] = encode_table(table, class_id, buffers, schema); + schema["classes"][class_id] = {{"name", table.name}, {"properties", properties}}; + tables.push_back({{"name", table.name}, {"class", class_id}, {"count", table.row_count()}, + {"properties", stored}}); + json declared = json::object(); + for (const auto &name : table.variable_names()) { + auto found = table.variables.find(name); + if (found != table.variables.end()) declared[name] = declaration_json(found->second); + } + if (!declared.empty()) variables[class_id] = declared; + return static_cast(tables.size()) - 1; + }; + + json data_structures = json::array(); + for (const auto &data_structure : frame.data_structures) { + check_identifier(data_structure.id, "data structure id"); + json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}}; + if (data_structure.platform) entry["platform"] = *data_structure.platform; + if (data_structure.model) entry["model"] = *data_structure.model; + entry["origin"] = column_major(data_structure.origin); // column-major, as glTF matrices + json sets = json::array(); + for (const auto &set : data_structure.entity_sets) { + json item = {{"name", set.name}, {"rank", nullptr}}; + if (set.rank) item["rank"] = *set.rank; + item["propertyTable"] = add_table(set, data_structure.id + "__" + set.name); + sets.push_back(item); + } + entry["entitySets"] = sets; + if (!data_structure.relations.empty()) { + json relations = json::array(); + for (const auto &relation : data_structure.relations) + relations.push_back({{"name", relation.name}, {"sourceSet", relation.source_set}, + {"targetSet", relation.target_set}, + {"propertyTable", add_table(relation, data_structure.id + "__" + relation.name)}}); + entry["relations"] = relations; + } + if (data_structure.scalars) entry["scalars"] = add_table(*data_structure.scalars, data_structure.id + "__scalars"); + if (data_structure.grid) entry["grid"] = grid_json(*data_structure.grid); + if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + json node = {{"name", data_structure.id}}; + if (!is_identity(data_structure.origin)) node["matrix"] = entry["origin"]; + document["nodes"].push_back(node); + const auto node_index = document["nodes"].size() - 1; + document["nodes"][0]["children"].push_back(node_index); + entry["node"] = node_index; + if (!data_structure.meshes.empty()) { + document["nodes"][node_index]["children"] = json::array(); + entry["meshes"] = json::array(); + for (const auto &mesh : data_structure.meshes) { + meshes.push_back(encode_mesh(mesh, data_structure, entry, buffers, accessors)); + document["nodes"].push_back({{"name", mesh.name}, {"mesh", meshes.size() - 1}}); + document["nodes"][node_index]["children"].push_back(document["nodes"].size() - 1); + json mesh_entry = {{"name", mesh.name}, {"mesh", meshes.size() - 1}}; + if (!mesh.display.empty()) { + json display = json::array(); + for (const auto &item : mesh.display) { + auto source = mesh.display_sources.find(item.first); + display.push_back({{"attribute", display_attribute(item.first)}, {"variable", item.first}, + {"entitySet", source == mesh.display_sources.end() ? json(nullptr) + : json(source->second)}}); + } + mesh_entry["display"] = display; + } + entry["meshes"].push_back(mesh_entry); + } + } + data_structures.push_back(entry); + } + + json mappings = json::array(); + for (const auto &mapping : frame.mappings) { + check_identifier(mapping.name, "mapping name"); + json entry = {{"name", mapping.name}, + {"source", {{"dataStructure", mapping.source.first}, {"entitySet", mapping.source.second}}}, + {"target", {{"dataStructure", mapping.target.first}, {"entitySet", mapping.target.second}}}}; + if (mapping.method) entry["method"] = *mapping.method; + entry["propertyTable"] = add_table(mapping, "mapping__" + mapping.name); + mappings.push_back(entry); + } + + json scene = {{"version", SPECIFICATION_VERSION}, + {"frame", {{"time", frame.time}, {"timeUnit", frame.time_unit}, {"step", frame.step}}}}; + if (frame.vocabulary_version) scene["vocabulary"] = *frame.vocabulary_version; + if (!frame.writer.empty()) scene["writer"] = frame.writer; + scene["dataStructures"] = data_structures; + if (!mappings.empty()) scene["mappings"] = mappings; + if (!variables.empty()) scene["variables"] = variables; + + const json other_extensions = get(frame.unknown, "extensions", json::object()); + json extensions = {{METADATA, {{"schema", schema}, {"propertyTables", tables}}}, {EXTENSION, scene}}; + json used = json::array({METADATA, EXTENSION}); + if (!meshes.empty()) used.push_back(FEATURES); + for (const auto &item : other_extensions.items()) { + extensions[item.key()] = item.value(); + used.push_back(item.key()); + } + document["extensionsUsed"] = used; + document["extensions"] = extensions; + if (frame.unknown.is_object() && frame.unknown.contains("extras")) document["extras"] = frame.unknown["extras"]; + if (!meshes.empty()) { + document["meshes"] = meshes; + document["accessors"] = accessors; + } + if (!buffers.views.empty()) { + // the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + buffers.data.resize(buffers.data.size() + detail::padding(buffers.data.size()), 0); + document["buffers"] = json::array({{{"byteLength", buffers.data.size()}}}); + document["bufferViews"] = buffers.views; + } + return pack(document, buffers.data); +} + +void write_frame(const std::string &path, const Frame &frame) { + const auto bytes = to_bytes(frame); + std::ofstream file(path, std::ios::binary); + if (!file) throw Error("cannot write '" + path + "'"); + file.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} + +// ── Reading ────────────────────────────────────────────────────────────── + +namespace { + +struct ViewBytes { + const std::uint8_t *data; + std::size_t size; +}; + +ViewBytes view_bytes(const json &document, const std::vector &binary, std::int64_t index) { + const json &view = document.at("bufferViews").at(static_cast(index)); + if (get(view, "buffer", 0).get() != 0) + throw Error("fspm-TF files keep every buffer view in the GLB binary chunk"); + const auto start = get(view, "byteOffset", 0).get(); + const auto length = view.at("byteLength").get(); + if (start + length > binary.size()) throw Error("a buffer view runs past the binary chunk"); + return {binary.data() + start, length}; +} + +template std::vector read_view(const ViewBytes &bytes, std::size_t count, std::size_t offset = 0) { + return detail::read_values(bytes.data, bytes.size, count, offset); +} + +template +Column decode_scalar(const ViewBytes &raw, const json &property, std::size_t count, std::size_t width) { + std::vector values = read_view(raw, count * width); + if constexpr (std::is_floating_point_v) { + if (property.contains("noData")) { + const T no_data = static_cast(property["noData"].get()); + for (auto &value : values) + if (value == no_data) value = std::numeric_limits::quiet_NaN(); + } + const json infinity = get(get(property, "extras", json::object()), "FSPM_infinity"); + if (!infinity.is_null()) { + const T positive = static_cast(infinity["positive"].get()); + const T negative = static_cast(infinity["negative"].get()); + for (auto &value : values) { + if (value == positive) value = std::numeric_limits::infinity(); + else if (value == negative) value = -std::numeric_limits::infinity(); + } + } + } + return Column(std::move(values), width); +} + +Column decode_scalar_of(const std::string &component, const ViewBytes &raw, const json &property, std::size_t count, + std::size_t width) { + if (component == "INT8") return decode_scalar(raw, property, count, width); + if (component == "UINT8") return decode_scalar(raw, property, count, width); + if (component == "INT16") return decode_scalar(raw, property, count, width); + if (component == "UINT16") return decode_scalar(raw, property, count, width); + if (component == "INT32") return decode_scalar(raw, property, count, width); + if (component == "UINT32") return decode_scalar(raw, property, count, width); + if (component == "INT64") return decode_scalar(raw, property, count, width); + if (component == "UINT64") return decode_scalar(raw, property, count, width); + if (component == "FLOAT32") return decode_scalar(raw, property, count, width); + if (component == "FLOAT64") return decode_scalar(raw, property, count, width); + throw Error("component type " + component + " is not part of fspm-TF 0.1"); +} + +std::vector read_unsigned(const std::string &type, const ViewBytes &raw, std::size_t count) { + std::vector values; + if (type == "UINT8") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT16") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT32") for (auto value : read_view(raw, count)) values.push_back(value); + else if (type == "UINT64") values = read_view(raw, count); + else throw Error("offsets or enum values of type " + type + " are not part of fspm-TF 0.1"); + return values; +} + +void decode_table(const json &document, const std::vector &binary, const json &description, + const json &schema, const json &variables, Table &table) { + const std::string class_id = description.at("class").get(); + table.name = get(description, "name", class_id).get(); + const json &properties = schema.at("classes").at(class_id).at("properties"); + const auto count = description.at("count").get(); + for (const auto &item : description.at("properties").items()) { + const std::string &name = item.key(); + const json &stored = item.value(); + const json &property = properties.at(name); + const auto raw = view_bytes(document, binary, stored.at("values").get()); + const std::string type = property.at("type").get(); + Column column; + if (type == "ENUM") { + const json &enumeration = schema.at("enums").at(property.at("enumType").get()); + const auto codes = read_unsigned(get(enumeration, "valueType", "UINT16").get(), raw, count); + std::map name_of; + for (const auto &value : enumeration.at("values")) + name_of[value.at("value").get()] = value.at("name").get(); + std::vector labels; + for (auto code : codes) labels.push_back(name_of.at(code)); + column = Column(std::move(labels)); + } else if (type == "STRING") { + const auto offsets = + read_unsigned(get(stored, "stringOffsetType", "UINT32").get(), + view_bytes(document, binary, stored.at("stringOffsets").get()), count + 1); + std::vector text; + for (std::size_t row = 0; row < count; ++row) { + if (offsets[row + 1] > raw.size || offsets[row] > offsets[row + 1]) + throw Error("'" + class_id + "." + name + "': string offsets outside its values"); + text.emplace_back(reinterpret_cast(raw.data + offsets[row]), offsets[row + 1] - offsets[row]); + } + column = Column(std::move(text)); + } else if (type == "BOOLEAN") { + if (raw.size * 8 < count) throw Error("'" + class_id + "." + name + "': too few bits"); + std::vector values(count); + for (std::size_t row = 0; row < count; ++row) values[row] = (raw.data[row / 8] >> (row % 8)) & 1u; + column = Column(std::move(values)); + } else if (type == "SCALAR") { + const bool vector = get(property, "array", false).get(); + if (vector && !property.contains("count")) + throw Error("'" + class_id + "." + name + "': variable-length arrays are not part of fspm-TF 0.1"); + const std::size_t width = vector ? property["count"].get() : 1; + column = decode_scalar_of(property.at("componentType").get(), raw, property, count, width); + } else { + throw Error("'" + class_id + "." + name + "': property type " + type + " is not part of fspm-TF 0.1"); + } + table.columns.emplace_back(name, std::move(column)); + const json declared = get(get(variables, class_id, json::object()), name); + if (!declared.is_null() || property.contains("description")) { + Variable variable; + variable.unit = get(declared, "unit", "1").get(); + variable.role = get(declared, "role", "state").get(); + if (!get(declared, "standardName").is_null()) variable.standard_name = declared["standardName"].get(); + if (!get(declared, "kind").is_null()) variable.kind = declared["kind"].get(); + if (property.contains("description")) variable.description = property["description"].get(); + if (!get(declared, "minimum").is_null()) variable.minimum = declared["minimum"].get(); + if (!get(declared, "maximum").is_null()) variable.maximum = declared["maximum"].get(); + for (const auto &reference : get(declared, "references", json::array())) + variable.references.push_back(reference.get()); + for (const auto &doi : get(declared, "doi", json::array())) variable.doi.push_back(doi.get()); + table.variables[name] = variable; + } + } +} + +template +std::vector accessor_values(const json &document, const std::vector &binary, std::int64_t index, + std::size_t &width) { + const json &accessor = document.at("accessors").at(static_cast(index)); + const std::string type = accessor.at("type").get(); + width = type == "SCALAR" ? 1 : type == "VEC2" ? 2 : type == "VEC3" ? 3 : type == "VEC4" ? 4 : 0; + if (width == 0) throw Error("accessor type " + type + " is not used by fspm-TF"); + const auto view_index = accessor.at("bufferView").get(); + const auto raw = view_bytes(document, binary, view_index); + const json &view = document["bufferViews"][static_cast(view_index)]; + const auto count = accessor.at("count").get(); + const auto start = get(accessor, "byteOffset", 0).get(); + const auto component = accessor.at("componentType").get(); + auto read = [&](auto tag) { + using C = decltype(tag); + const std::size_t item_size = sizeof(C) * width; + const auto stride = get(view, "byteStride", item_size).template get(); + std::vector values; + values.reserve(count * width); + for (std::size_t item = 0; item < count; ++item) + for (auto value : read_view(raw, width, start + item * stride)) values.push_back(static_cast(value)); + return values; + }; + if (component == FLOAT) return read(float{}); + if (component == UNSIGNED_INT) return read(std::uint32_t{}); + if (component == UNSIGNED_SHORT) return read(std::uint16_t{}); + if (component == UNSIGNED_BYTE) return read(std::uint8_t{}); + throw Error("accessor component type " + std::to_string(component) + " is not used by fspm-TF"); +} + +Mesh decode_mesh(const json &document, const std::vector &binary, const json &mesh_entry, + const std::map &tables) { + const json &primitive = document.at("meshes").at(mesh_entry.at("mesh").get()).at("primitives").at(0); + const json &attributes = primitive.at("attributes"); + Mesh mesh; + mesh.name = mesh_entry.at("name").get(); + std::size_t width = 0; + const auto positions = accessor_values(document, binary, attributes.at("POSITION").get(), width); + mesh.positions.assign(positions.begin(), positions.end()); + const json features = get(get(get(primitive, "extensions", json::object()), FEATURES, json::object()), "featureIds", + json::array()); + for (const auto &feature : features) { + const std::string attribute = "_FEATURE_ID_" + std::to_string(feature.at("attribute").get()); + auto rows = accessor_values(document, binary, attributes.at(attribute).get(), width); + if (feature.contains("nullFeatureId")) { + const auto null_row = feature["nullFeatureId"].get(); + for (auto &row : rows) + if (row == null_row) row = NO_ID; + } + mesh.feature_rows.emplace_back(tables.at(feature.at("propertyTable").get()), std::move(rows)); + } + for (const auto &item : get(mesh_entry, "display", json::array())) { + const std::string variable = item.at("variable").get(); + mesh.display.emplace_back(variable, accessor_values( + document, binary, attributes.at(item.at("attribute").get()).get(), width)); + if (!get(item, "entitySet").is_null()) mesh.display_sources[variable] = item["entitySet"].get(); + } + if (primitive.contains("indices")) + mesh.indices = accessor_values(document, binary, primitive["indices"].get(), width); + return mesh; +} + +} // namespace + +Frame from_bytes(const std::vector &data) { + auto [document, binary] = unpack(data); + const json extensions = get(document, "extensions", json::object()); + if (!extensions.contains(EXTENSION)) throw Error("not an fspm-TF file: no " + EXTENSION + " extension"); + const json &scene = extensions[EXTENSION]; + const json metadata = get(extensions, METADATA, {{"schema", json::object()}, {"propertyTables", json::array()}}); + const json variables = get(scene, "variables", json::object()); + auto table = [&](std::int64_t index, Table &target) { + decode_table(document, binary, metadata.at("propertyTables").at(static_cast(index)), + metadata.at("schema"), variables, target); + }; + + Frame frame; + const json frame_entry = get(scene, "frame", json::object()); + frame.time = get(frame_entry, "time", 0.0).get(); + frame.time_unit = get(frame_entry, "timeUnit", "d").get(); + frame.step = get(frame_entry, "step", 0).get(); + frame.writer = get(scene, "writer", json::object()); + if (!get(scene, "vocabulary").is_null()) frame.vocabulary_version = scene["vocabulary"].get(); + for (const auto &entry : get(scene, "dataStructures", json::array())) { + DataStructure data_structure(entry.at("id").get(), get(entry, "kind", "graph").get()); + if (!get(entry, "platform").is_null()) data_structure.platform = entry["platform"].get(); + if (!get(entry, "model").is_null()) data_structure.model = entry["model"].get(); + data_structure.extras = get(entry, "extras", json::object()); + if (entry.contains("origin")) { + const auto values = entry["origin"].get>(); + if (values.size() != 16) throw Error("data structure '" + data_structure.id + "': an origin of 16 numbers"); + for (int row = 0; row < 4; ++row) + for (int column = 0; column < 4; ++column) + data_structure.origin[row * 4 + column] = values[column * 4 + row]; + } + std::map tables; + for (const auto &item : get(entry, "entitySets", json::array())) { + EntitySet set; + if (!get(item, "rank").is_null()) set.rank = item["rank"].get(); + table(item.at("propertyTable").get(), set); + tables[item["propertyTable"].get()] = item.at("name").get(); + data_structure.entity_sets.push_back(std::move(set)); + } + for (const auto &item : get(entry, "relations", json::array())) { + Relation relation("", item.at("sourceSet").get(), item.at("targetSet").get()); + table(item.at("propertyTable").get(), relation); + data_structure.relations.push_back(std::move(relation)); + } + if (entry.contains("scalars")) { + Table scalars; + table(entry["scalars"].get(), scalars); + data_structure.scalars = std::move(scalars); + } + if (entry.contains("grid")) { + const json &item = entry["grid"]; + Grid grid; + grid.kind = item.at("kind").get(); + grid.shape = item.at("shape").get>(); + grid.origin = item.at("origin").get>(); + grid.spacing = item.at("spacing").get>(); + grid.axes = item.at("axes").get>(); + grid.periodic = item.at("periodic").get>(); + grid.z_up = get(item, "zUp", true).get(); + grid.max_level = get(item, "maxLevel", 0).get(); + grid.cells = item.at("cells").get(); + data_structure.grid = std::move(grid); + } + for (const auto &mesh_entry : get(entry, "meshes", json::array())) + data_structure.meshes.push_back(decode_mesh(document, binary, mesh_entry, tables)); + frame.data_structures.push_back(std::move(data_structure)); + } + for (const auto &entry : get(scene, "mappings", json::array())) { + Mapping mapping("", {entry.at("source").at("dataStructure").get(), + entry.at("source").at("entitySet").get()}, + {entry.at("target").at("dataStructure").get(), + entry.at("target").at("entitySet").get()}); + if (!get(entry, "method").is_null()) mapping.method = entry["method"].get(); + table(entry.at("propertyTable").get(), mapping); + frame.mappings.push_back(std::move(mapping)); + } + json other_extensions = json::object(); + for (const auto &item : extensions.items()) + if (item.key() != METADATA && item.key() != EXTENSION && item.key() != FEATURES) + other_extensions[item.key()] = item.value(); + if (!other_extensions.empty()) frame.unknown["extensions"] = other_extensions; + if (document.contains("extras")) frame.unknown["extras"] = document["extras"]; + return frame; +} + +Frame read_frame(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error("cannot read '" + path + "'"); + std::vector data((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + return from_bytes(data); +} + +} // namespace fspm_tf diff --git a/bindings/cpp/src/dump.cpp b/bindings/cpp/src/dump.cpp new file mode 100644 index 0000000..fd40c8d --- /dev/null +++ b/bindings/cpp/src/dump.cpp @@ -0,0 +1,168 @@ +// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md), and +// the comparison `fspm-tf compare` makes. +#include "fspm_tf/fspm_tf.hpp" + +#include +#include + +namespace fspm_tf { + +namespace { + +template json value_json(T value) { + if constexpr (std::is_floating_point_v) { + if (std::isnan(value)) return nullptr; + if (std::isinf(value)) return value > 0 ? "Infinity" : "-Infinity"; + return static_cast(value); + } else { + return value; + } +} + +json column_json(const Column &column) { + return std::visit([&](const auto &values) { + json items = json::array(); + if (column.width == 1) { + for (std::size_t item = 0; item < values.size(); ++item) items.push_back(value_json(values[item])); + return items; + } + for (std::size_t row = 0; row < column.rows(); ++row) { + json vector = json::array(); + for (std::size_t component = 0; component < column.width; ++component) + vector.push_back(value_json(values[row * column.width + component])); + items.push_back(vector); + } + return items; + }, column.values); +} + +json table_json(const Table &table) { + json columns = json::object(); + for (const auto &[name, column] : table.columns) + columns[name] = {{"dtype", column.dtype()}, {"values", column_json(column)}}; + json entry = {{"name", table.name}, {"count", table.row_count()}, {"columns", columns}}; + json declarations = json::object(); + for (const auto &[name, variable] : table.variables) { + json declared = {{"unit", variable.unit}, {"role", variable.role}}; + if (variable.standard_name) declared["standard_name"] = *variable.standard_name; + if (variable.kind) declared["kind"] = *variable.kind; + if (variable.description) declared["description"] = *variable.description; + if (variable.minimum) declared["minimum"] = *variable.minimum; + if (variable.maximum) declared["maximum"] = *variable.maximum; + if (!variable.references.empty()) declared["references"] = variable.references; + if (!variable.doi.empty()) declared["doi"] = variable.doi; + declarations[name] = declared; + } + if (!declarations.empty()) entry["variables"] = declarations; + return entry; +} + +bool is_number(const json &value) { return value.is_number(); } + +} // namespace + +json frame_to_json(const Frame &frame) { + json data_structures = json::array(); + for (const auto &data_structure : frame.data_structures) { + json origin = json::array(); + for (int row = 0; row < 4; ++row) + origin.push_back({data_structure.origin[row * 4], data_structure.origin[row * 4 + 1], + data_structure.origin[row * 4 + 2], data_structure.origin[row * 4 + 3]}); + json sets = json::array(); + for (const auto &set : data_structure.entity_sets) { + json item = table_json(set); + item["rank"] = set.rank ? json(*set.rank) : json(nullptr); + sets.push_back(item); + } + json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}, {"origin", origin}, + {"entity_sets", sets}}; + if (data_structure.platform) entry["platform"] = *data_structure.platform; + if (data_structure.model) entry["model"] = *data_structure.model; + if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + if (!data_structure.relations.empty()) { + json relations = json::array(); + for (const auto &relation : data_structure.relations) { + json item = table_json(relation); + item["source_set"] = relation.source_set; + item["target_set"] = relation.target_set; + relations.push_back(item); + } + entry["relations"] = relations; + } + if (data_structure.scalars) entry["scalars"] = table_json(*data_structure.scalars); + if (data_structure.grid) { + const Grid &grid = *data_structure.grid; + entry["grid"] = {{"kind", grid.kind}, {"shape", grid.shape}, {"origin", grid.origin}, + {"spacing", grid.spacing}, {"axes", grid.axes}, {"periodic", grid.periodic}, + {"z_up", grid.z_up}, {"max_level", grid.max_level}, {"cells", grid.cells}}; + } + if (!data_structure.meshes.empty()) { + json meshes = json::array(); + for (const auto &mesh : data_structure.meshes) { + json positions = json::array(); + for (std::size_t vertex = 0; vertex < mesh.vertex_count(); ++vertex) + positions.push_back({static_cast(static_cast(mesh.positions[vertex * 3])), + static_cast(static_cast(mesh.positions[vertex * 3 + 1])), + static_cast(static_cast(mesh.positions[vertex * 3 + 2]))}); + json feature_rows = json::object(); + for (const auto &[name, rows] : mesh.feature_rows) feature_rows[name] = rows; + json item = {{"name", mesh.name}, {"positions_float32", positions}, {"feature_rows", feature_rows}}; + if (mesh.indices) item["indices"] = *mesh.indices; + if (!mesh.display.empty()) { + json display = json::object(); + for (const auto &[name, values] : mesh.display) display[name] = column_json(Column(values)); + item["display"] = display; + } + meshes.push_back(item); + } + entry["meshes"] = meshes; + } + data_structures.push_back(entry); + } + json mappings = json::array(); + for (const auto &mapping : frame.mappings) { + json item = table_json(mapping); + item["source"] = {mapping.source.first, mapping.source.second}; + item["target"] = {mapping.target.first, mapping.target.second}; + item["method"] = mapping.method ? json(*mapping.method) : json(nullptr); + mappings.push_back(item); + } + return {{"time", frame.time}, {"time_unit", frame.time_unit}, {"step", frame.step}, + {"data_structures", data_structures}, {"mappings", mappings}}; +} + +std::vector compare(const json &actual, const json &expected, const std::string &path) { + std::vector found; + if (expected.is_object()) { + if (!actual.is_object()) return {path + ": an object expected"}; + for (const auto &item : expected.items()) + if (!actual.contains(item.key())) found.push_back(path + ": missing key '" + item.key() + "'"); + for (const auto &item : actual.items()) + if (!expected.contains(item.key())) found.push_back(path + ": unexpected key '" + item.key() + "'"); + for (const auto &item : expected.items()) + if (actual.contains(item.key())) { + auto more = compare(actual[item.key()], item.value(), path + "." + item.key()); + found.insert(found.end(), more.begin(), more.end()); + } + return found; + } + if (expected.is_array()) { + if (!actual.is_array()) return {path + ": an array expected"}; + if (actual.size() != expected.size()) + return {path + ": " + std::to_string(actual.size()) + " items, " + std::to_string(expected.size()) + + " expected"}; + for (std::size_t item = 0; item < expected.size(); ++item) { + auto more = compare(actual[item], expected[item], path + "[" + std::to_string(item) + "]"); + found.insert(found.end(), more.begin(), more.end()); + } + return found; + } + if (is_number(expected)) { + if (is_number(actual) && actual.get() == expected.get()) return {}; + return {path + ": " + actual.dump() + ", " + expected.dump() + " expected"}; + } + if (actual != expected) return {path + ": " + actual.dump() + ", " + expected.dump() + " expected"}; + return {}; +} + +} // namespace fspm_tf diff --git a/bindings/cpp/src/glb.cpp b/bindings/cpp/src/glb.cpp new file mode 100644 index 0000000..410d3bc --- /dev/null +++ b/bindings/cpp/src/glb.cpp @@ -0,0 +1,72 @@ +// The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary +// chunk padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. +#include "fspm_tf/fspm_tf.hpp" + +#include "bytes.hpp" + +namespace fspm_tf { + +namespace { +constexpr std::uint32_t GLB_MAGIC = 0x46546C67; // "glTF" +constexpr std::uint32_t GLB_VERSION = 2; +constexpr std::uint32_t JSON_CHUNK = 0x4E4F534A; // "JSON" +constexpr std::uint32_t BINARY_CHUNK = 0x004E4942; // "BIN\0" + +void append_word(std::vector &bytes, std::uint32_t value) { detail::append_values(bytes, &value, 1); } +} // namespace + +std::vector pack(const json &document, const std::vector &binary) { + std::string text = document.dump(-1, ' ', false, json::error_handler_t::strict); + text.append(detail::padding(text.size()), ' '); + std::vector padded = binary; + padded.resize(binary.size() + detail::padding(binary.size()), 0); + const std::size_t total = 12 + 8 + text.size() + (padded.empty() ? 0 : 8 + padded.size()); + if (total > 0xFFFFFFFFull) throw Error("a GLB file is at most 4 GiB"); + std::vector bytes; + bytes.reserve(total); + append_word(bytes, GLB_MAGIC); + append_word(bytes, GLB_VERSION); + append_word(bytes, static_cast(total)); + append_word(bytes, static_cast(text.size())); + append_word(bytes, JSON_CHUNK); + bytes.insert(bytes.end(), text.begin(), text.end()); + if (!padded.empty()) { + append_word(bytes, static_cast(padded.size())); + append_word(bytes, BINARY_CHUNK); + bytes.insert(bytes.end(), padded.begin(), padded.end()); + } + return bytes; +} + +std::pair> unpack(const std::vector &data) { + if (data.size() < 20) throw Error("not a GLB file: too short"); + if (detail::read_word(data, 0) != GLB_MAGIC) throw Error("not a GLB file: wrong magic"); + const auto version = detail::read_word(data, 4); + if (version != GLB_VERSION) throw Error("GLB version " + std::to_string(version) + ", 2 expected"); + const auto total = detail::read_word(data, 8); + if (total != data.size()) + throw Error("GLB header length " + std::to_string(total) + ", the file holds " + std::to_string(data.size()) + + " bytes"); + std::size_t offset = 12; + bool found_document = false; + json document; + std::vector binary; + while (offset < total) { + const std::size_t chunk_length = detail::read_word(data, offset); + const auto chunk_type = detail::read_word(data, offset + 4); + if (offset + 8 + chunk_length > total) throw Error("a GLB chunk runs past the end of the file"); + const auto begin = data.begin() + static_cast(offset + 8); + const auto end = begin + static_cast(chunk_length); + if (chunk_type == JSON_CHUNK) { + document = json::parse(begin, end); + found_document = true; + } else if (chunk_type == BINARY_CHUNK) { + binary.assign(begin, end); + } + offset += 8 + chunk_length; + } + if (!found_document) throw Error("GLB file without a JSON chunk"); + return {std::move(document), std::move(binary)}; +} + +} // namespace fspm_tf diff --git a/bindings/cpp/src/model.cpp b/bindings/cpp/src/model.cpp new file mode 100644 index 0000000..b267685 --- /dev/null +++ b/bindings/cpp/src/model.cpp @@ -0,0 +1,136 @@ +#include "fspm_tf/model.hpp" + +#include +#include + +namespace fspm_tf { + +std::size_t Column::size() const { + return std::visit([](const auto &column_values) { return column_values.size(); }, values); +} + +namespace { + +template struct TypeName; +template <> struct TypeName { static constexpr const char *value = "int8"; }; +template <> struct TypeName { static constexpr const char *value = "uint8"; }; +template <> struct TypeName { static constexpr const char *value = "int16"; }; +template <> struct TypeName { static constexpr const char *value = "uint16"; }; +template <> struct TypeName { static constexpr const char *value = "int32"; }; +template <> struct TypeName { static constexpr const char *value = "uint32"; }; +template <> struct TypeName { static constexpr const char *value = "int64"; }; +template <> struct TypeName { static constexpr const char *value = "uint64"; }; +template <> struct TypeName { static constexpr const char *value = "float32"; }; +template <> struct TypeName { static constexpr const char *value = "float64"; }; +template <> struct TypeName { static constexpr const char *value = "bool"; }; +template <> struct TypeName { static constexpr const char *value = "string"; }; + +const std::vector NO_RESERVED = {}; +const std::vector ENTITY_SET_RESERVED = {"id", "label", "complex", "parent", "edge_type", "level", "index"}; +const std::vector RELATION_RESERVED = {"id", "source", "target", "relation"}; +const std::vector MAPPING_RESERVED = {"source", "target", "weight"}; + +} // namespace + +std::string Column::dtype() const { + return std::visit([](const auto &column_values) -> std::string { + using T = typename std::decay_t::value_type; + return TypeName::value; + }, values); +} + +bool Variable::operator==(const Variable &other) const { + return unit == other.unit && standard_name == other.standard_name && kind == other.kind && role == other.role && + description == other.description && minimum == other.minimum && maximum == other.maximum && + references == other.references && doi == other.doi; +} + +std::size_t Table::row_count() const { return columns.empty() ? 0 : columns.front().second.rows(); } + +bool Table::has_column(const std::string &column_name) const { + return std::any_of(columns.begin(), columns.end(), [&](const auto &item) { return item.first == column_name; }); +} + +const Column &Table::column(const std::string &column_name) const { + for (const auto &item : columns) + if (item.first == column_name) return item.second; + throw Error("table '" + name + "' has no column '" + column_name + "'"); +} + +Column &Table::column(const std::string &column_name) { + return const_cast(static_cast(*this).column(column_name)); +} + +Table &Table::set(const std::string &column_name, Column values) { + for (auto &item : columns) + if (item.first == column_name) { + item.second = std::move(values); + return *this; + } + columns.emplace_back(column_name, std::move(values)); + return *this; +} + +Table &Table::declare(const std::string &column_name, Variable variable) { + variables[column_name] = std::move(variable); + return *this; +} + +const std::vector &Table::reserved() const { + switch (table_kind) { + case TableKind::entity_set: return ENTITY_SET_RESERVED; + case TableKind::relation: return RELATION_RESERVED; + case TableKind::mapping: return MAPPING_RESERVED; + default: return NO_RESERVED; + } +} + +bool Table::is_reserved(const std::string &column_name) const { + const auto &names = reserved(); + return std::find(names.begin(), names.end(), column_name) != names.end(); +} + +std::vector Table::variable_names() const { + std::vector names; + for (const auto &item : columns) + if (!is_reserved(item.first)) names.push_back(item.first); + return names; +} + +void Table::check_lengths() const { + for (const auto &item : columns) { + if (item.second.width == 0 || item.second.size() % item.second.width != 0) + throw Error("column '" + item.first + "' of '" + name + "': its values are not whole rows"); + if (item.second.rows() != row_count()) + throw Error("table '" + name + "': columns of different lengths"); + } +} + +std::vector EntitySet::ids() const { + if (has_column("id")) return column("id").as(); + std::vector rows(row_count()); + std::iota(rows.begin(), rows.end(), 0); + return rows; +} + +const EntitySet &DataStructure::entity_set(const std::string &name) const { + for (const auto &candidate : entity_sets) + if (candidate.name == name) return candidate; + throw Error("data structure '" + id + "' has no entity set '" + name + "'"); +} + +EntitySet &DataStructure::entity_set(const std::string &name) { + return const_cast(static_cast(*this).entity_set(name)); +} + +const DataStructure &Frame::data_structure(const std::string &id) const { + for (const auto &candidate : data_structures) + if (candidate.id == id) return candidate; + throw Error("the frame has no data structure '" + id + "'"); +} + +DataStructure &Frame::data_structure(const std::string &id) { + return const_cast(static_cast(*this).data_structure(id)); +} + +} // namespace fspm_tf diff --git a/bindings/cpp/src/sequence.cpp b/bindings/cpp/src/sequence.cpp new file mode 100644 index 0000000..6537fa7 --- /dev/null +++ b/bindings/cpp/src/sequence.cpp @@ -0,0 +1,65 @@ +// Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. +#include "fspm_tf/fspm_tf.hpp" + +#include +#include +#include +#include +#include + +namespace fspm_tf { + +void write_index(const std::string &path, const Index &index) { + json frames = json::array(); + for (const auto &entry : index.frames) + frames.push_back({{"time", entry.time}, {"step", entry.step}, {"file", entry.file}}); + const json document = {{"fspmIndex", index.version}, {"timeUnit", index.time_unit}, {"frames", frames}}; + std::ofstream file(path, std::ios::binary); + if (!file) throw Error("cannot write '" + path + "'"); + file << document.dump(1) << "\n"; +} + +Index read_index(const std::string &path) { + std::ifstream file(path, std::ios::binary); + if (!file) throw Error("cannot read '" + path + "'"); + const json document = json::parse(file); + Index index; + index.version = document.value("fspmIndex", std::string(SPECIFICATION_VERSION)); + index.time_unit = document.value("timeUnit", std::string("d")); + const auto directory = std::filesystem::absolute(std::filesystem::path(path)).parent_path(); + for (const auto &item : document.value("frames", json::array())) { + IndexEntry entry; + entry.time = item.at("time").get(); + entry.step = item.at("step").get(); + entry.file = item.at("file").get(); + entry.path = (directory / entry.file).string(); + index.frames.push_back(entry); + } + return index; +} + +std::string write_sequence(const std::string &directory, const std::vector &frames, const std::string &name) { + std::filesystem::create_directories(directory); + Index index; + std::set time_units; + for (std::size_t number = 0; number < frames.size(); ++number) { + std::ostringstream file; + file << name << "_" << std::setw(6) << std::setfill('0') << number << ".glb"; + write_frame((std::filesystem::path(directory) / file.str()).string(), frames[number]); + index.frames.push_back({frames[number].time, frames[number].step, file.str(), ""}); + time_units.insert(frames[number].time_unit); + } + if (time_units.size() > 1) throw Error("the frames' times are in several units"); + if (!time_units.empty()) index.time_unit = *time_units.begin(); + const auto path = (std::filesystem::path(directory) / INDEX_FILE).string(); + write_index(path, index); + return path; +} + +std::vector read_sequence(const std::string &path) { + std::vector frames; + for (const auto &entry : read_index(path).frames) frames.push_back(read_frame(entry.path)); + return frames; +} + +} // namespace fspm_tf diff --git a/bindings/cpp/test/test_fspm_tf.cpp b/bindings/cpp/test/test_fspm_tf.cpp new file mode 100644 index 0000000..7b4482a --- /dev/null +++ b/bindings/cpp/test/test_fspm_tf.cpp @@ -0,0 +1,238 @@ +// The C++ codec's tests: the conformance files read as their expected values, written and read back the same, every +// column type, meshes, sequences, and what the specification forbids refused. +// +// fspm_tf_tests CONFORMANCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fspm_tf/fspm_tf.hpp" + +namespace filesystem = std::filesystem; +using namespace fspm_tf; + +namespace { + +int failures = 0, checks = 0; + +void check(bool condition, const std::string &what) { + ++checks; + if (!condition) { + ++failures; + std::cerr << "FAILED: " << what << "\n"; + } +} + +template void check_throws(Function function, const std::string &what) { + bool thrown = false; + try { + function(); + } catch (const Error &) { + thrown = true; + } + check(thrown, what + " (an Error expected)"); +} + +std::vector conformance_files(const filesystem::path &conformance) { + std::vector files; + for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) + if (entry.is_regular_file() && entry.path().extension() == ".glb") files.push_back(entry.path()); + std::sort(files.begin(), files.end()); + return files; +} + +json expected_of(const filesystem::path &glb) { + auto path = glb; + path.replace_extension(".expected.json"); + std::ifstream file(path, std::ios::binary); + return json::parse(file); +} + +std::vector file_bytes(const filesystem::path &path) { + std::ifstream file(path, std::ios::binary); + return {std::istreambuf_iterator(file), std::istreambuf_iterator()}; +} + +void report(const std::vector &differences, const std::string &what) { + for (std::size_t item = 0; item < std::min(differences.size(), 5); ++item) + std::cerr << " " << differences[item] << "\n"; + check(differences.empty(), what); +} + +void test_conformance(const filesystem::path &conformance) { + const auto files = conformance_files(conformance); + check(files.size() == 9, "nine conformance files"); + for (const auto &path : files) { + const auto name = filesystem::relative(path, conformance).string(); + const auto data = file_bytes(path); + auto [document, binary] = unpack(data); // the container, repacked + auto [again, binary_again] = unpack(pack(document, binary)); + check(binary_again == binary && again == document && binary.size() % 8 == 0, "GLB repacked: " + name); + const auto frame = read_frame(path.string()); // reading + report(compare(json::parse(frame_to_json(frame).dump()), expected_of(path)), "read as expected: " + name); + const auto written = from_bytes(to_bytes(frame)); // writing + report(compare(json::parse(frame_to_json(written).dump()), expected_of(path)), "written and read: " + name); + } +} + +void test_columns() { + EntitySet organs("Organ", 1); + organs.set("id", std::vector{0, 1, 2}) + .set("label", {"Leaf", "Internode", "Leaf"}) + .set("parent", std::vector{-1, 0, 1}) + .set("edge_type", {"none", "successor", "branch"}) + .set("with_nan", std::vector{1.0, std::nan(""), -2.5}) + .set("with_infinities", std::vector{std::numeric_limits::infinity(), 1.f, + -std::numeric_limits::infinity()}) + .set("flag", std::vector{true, false, true}) + .set("name", {"\xc3\xa9", "", "a b"}) + .set("position", Column(std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9}, 3)) + .set("column_int8", std::vector{1, 2, 3}) + .set("column_uint8", std::vector{1, 2, 3}) + .set("column_int16", std::vector{1, 2, 3}) + .set("column_uint16", std::vector{1, 2, 3}) + .set("column_int32", std::vector{1, 2, 3}) + .set("column_uint32", std::vector{1, 2, 3}) + .set("column_uint64", std::vector{1, 2, 3}); + Variable length; + length.unit = "m"; + length.standard_name = "length"; + length.kind = "extensive"; + length.description = "a length"; + length.minimum = 0; + length.references = {"a reference"}; + organs.declare("with_nan", length); + DataStructure plants("plants"); + plants.entity_sets.push_back(organs); + plants.origin = {1, 0, 0, 0.5, 0, 1, 0, 0, 0, 0, 1, -0.1, 0, 0, 0, 1}; + plants.extras = {{"platform", {1, 2}}}; + Frame frame; + frame.data_structures.push_back(plants); + frame.time = 2.5; + frame.time_unit = "h"; + frame.step = 3; + + const auto read_back = from_bytes(to_bytes(frame)); + const auto &set = read_back.data_structure("plants").entity_set("Organ"); + check(set.column("id").holds(), "ids are int64"); + const auto &with_nan = set.column("with_nan").as(); + check(with_nan[0] == 1.0 && std::isnan(with_nan[1]) && with_nan[2] == -2.5, "NaN read back"); + const auto &infinities = set.column("with_infinities").as(); + check(std::isinf(infinities[0]) && infinities[0] > 0 && infinities[1] == 1.f && std::isinf(infinities[2]) && + infinities[2] < 0, "infinities read back"); + check(set.column("flag").as() == std::vector{true, false, true}, "booleans read back"); + check(set.column("name").as() == std::vector{"\xc3\xa9", "", "a b"}, "text read back"); + check(set.column("position").width == 3 && + set.column("position").as() == std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9}, "vectors read back"); + check(set.column("label").as() == std::vector{"Leaf", "Internode", "Leaf"}, "labels"); + check(set.column("column_uint16").as() == std::vector{1, 2, 3}, "uint16 read back"); + check(set.column("column_uint64").as() == std::vector{1, 2, 3}, "uint64 read back"); + check(set.variables.at("with_nan") == length, "declaration read back"); + check(read_back.data_structure("plants").origin == plants.origin, "origin read back"); + check(read_back.data_structure("plants").extras == plants.extras, "extras read back"); + check(read_back.time == 2.5 && read_back.time_unit == "h" && read_back.step == 3, "time read back"); +} + +void test_mesh() { + EntitySet segments("SubOrgan", 6); + segments.set("id", std::vector{10, 11}); + Mesh mesh; + mesh.name = "roots"; + mesh.positions = {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}; + mesh.feature_rows = {{"SubOrgan", {0, 1, 1, -1}}}; + mesh.indices = std::vector{0, 1, 2, 0, 2, 3}; + mesh.display = {{"radius", {1.f, 2.f, 2.f, 0.f}}}; + mesh.display_sources = {{"radius", "SubOrgan"}}; + DataStructure roots("roots"); + roots.entity_sets.push_back(segments); + roots.meshes.push_back(mesh); + Frame frame; + frame.data_structures.push_back(roots); + const auto read_back = from_bytes(to_bytes(frame)).data_structures[0].meshes[0]; + check(read_back.positions == mesh.positions && read_back.feature_rows == mesh.feature_rows && + read_back.indices == mesh.indices && read_back.display == mesh.display && + read_back.display_sources == mesh.display_sources, "mesh read back"); +} + +void test_refused() { + auto frame_with = [](EntitySet set) { + DataStructure plants("plants"); + plants.entity_sets.push_back(std::move(set)); + Frame frame; + frame.data_structures.push_back(plants); + return frame; + }; + EntitySet bad_column("Organ"); + bad_column.set("id", std::vector{0}).set("1st", std::vector{1.0}); + check_throws([&] { to_bytes(frame_with(bad_column)); }, "a column name that is not an identifier"); + Frame bad_id; + bad_id.data_structures.emplace_back("bad id"); + check_throws([&] { to_bytes(bad_id); }, "a data structure id that is not an identifier"); + EntitySet bad_edge("Organ"); + bad_edge.set("id", std::vector{0}).set("parent", std::vector{-1}).set("edge_type", {"root"}); + check_throws([&] { to_bytes(frame_with(bad_edge)); }, "an unknown edge type"); + EntitySet both("Organ"); + both.set("id", std::vector{0, 1}) + .set("x", std::vector{std::nan(""), std::numeric_limits::lowest()}); + check_throws([&] { to_bytes(frame_with(both)); }, "NaN and its stand-in in one column"); + EntitySet lengths("Organ"); + lengths.set("id", std::vector{0, 1}).set("x", std::vector{1.0}); + check_throws([&] { to_bytes(frame_with(lengths)); }, "columns of different lengths"); + check_throws([] { unpack({1, 2, 3}); }, "not a GLB file"); +} + +void test_sequence(const filesystem::path &conformance) { + const auto folder = filesystem::temp_directory_path() / "fspm_tf_cpp_sequence"; + filesystem::remove_all(folder); + std::vector frames; + for (int step = 0; step < 3; ++step) { + EntitySet plant("Plant", 1); + plant.set("id", std::vector{0}).set("biomass", std::vector{double(step)}); + DataStructure plants("plants"); + plants.entity_sets.push_back(plant); + Frame frame; + frame.data_structures.push_back(plants); + frame.time = step / 24.0; + frame.step = step; + frames.push_back(frame); + } + const auto index = write_sequence(folder.string(), frames); + const auto entries = read_index(index).frames; + check(entries.size() == 3 && entries[2].step == 2, "index written and read"); + const auto read_back = read_sequence(index); + check(read_back.size() == 3 && read_back[2].data_structures[0].entity_sets[0].column("biomass").as()[0] == 2.0, + "sequence read back"); + const auto growing = read_index((conformance / "growing_sequence" / INDEX_FILE).string()); + check(std::all_of(growing.frames.begin(), growing.frames.end(), + [](const IndexEntry &entry) { return filesystem::exists(entry.path); }), + "the conformance sequence's files found"); + filesystem::remove_all(folder); +} + +} // namespace + +int main(int argument_count, char **arguments) { + if (argument_count != 2) { + std::cerr << "usage: fspm_tf_tests CONFORMANCE\n"; + return 2; + } + const filesystem::path conformance = arguments[1]; + const std::vector>> tests = { + {"conformance", [&] { test_conformance(conformance); }}, {"columns", test_columns}, {"mesh", test_mesh}, + {"refused", test_refused}, {"sequence", [&] { test_sequence(conformance); }}}; + for (const auto &[name, test] : tests) { + try { + test(); + } catch (const std::exception &error) { + check(false, name + ": " + error.what()); + } + } + std::cout << checks - failures << " of " << checks << " checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/bindings/cpp/tools/conformance_run.cpp b/bindings/cpp/tools/conformance_run.cpp new file mode 100644 index 0000000..662040e --- /dev/null +++ b/bindings/cpp/tools/conformance_run.cpp @@ -0,0 +1,37 @@ +// The C++ codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump +// it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). +// Then .github/actions/conformance compares the dumps with the expected values and validates the written files. +// +// fspm_tf_conformance_run CONFORMANCE OUT +#include +#include +#include + +#include "fspm_tf/fspm_tf.hpp" + +namespace filesystem = std::filesystem; + +int main(int argument_count, char **arguments) { + if (argument_count != 3) { + std::cerr << "usage: fspm_tf_conformance_run CONFORMANCE OUT\n"; + return 2; + } + const filesystem::path conformance = arguments[1], out = arguments[2]; + for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) { + if (!entry.is_regular_file() || entry.path().extension() != ".glb") continue; + const auto relative = filesystem::relative(entry.path(), conformance); + const auto frame = fspm_tf::read_frame(entry.path().string()); + auto dump = out / "dumps" / relative; + dump.replace_extension(".json"); + const auto written = out / "written" / relative; + filesystem::create_directories(dump.parent_path()); + filesystem::create_directories(written.parent_path()); + std::ofstream(dump, std::ios::binary) << fspm_tf::frame_to_json(frame).dump(1) << "\n"; + fspm_tf::write_frame(written.string(), frame); // written again from what was read + } + const auto index = conformance / "growing_sequence" / fspm_tf::INDEX_FILE; + if (filesystem::exists(index)) + filesystem::copy_file(index, out / "written" / "growing_sequence" / fspm_tf::INDEX_FILE, + filesystem::copy_options::overwrite_existing); + return 0; +} diff --git a/doc/cpp.md b/doc/cpp.md new file mode 100644 index 0000000..6fa1332 --- /dev/null +++ b/doc/cpp.md @@ -0,0 +1,88 @@ +# Reading and writing in C++, and CPlantBox + +`bindings/cpp/` holds the C++ codec (C++17). It reads and writes fspm-TF files and passes the conformance check. In +`bindings/cpp/cplantbox/` is CPlantBox's adapter. Both have tests, run in `.github/workflows/cpp-codec.yml` on +Linux, macOS and Windows. + +## Building + +```bash +cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +cmake --install build --prefix /some/prefix # then: find_package(fspm_tf), fspm_tf::fspm_tf +``` + +The one dependency, nlohmann_json, is used when installed and fetched otherwise. + +## The format's own tables + +```cpp +#include + +auto frame = fspm_tf::read_frame("scene.glb"); +const auto &segments = frame.data_structure("RootPopulation").entity_set("SubOrgan"); +const auto &length = segments.column("length").as(); // one value per segment +const auto unit = segments.variables.at("length").unit; // its declared unit, e.g. "m" + +fspm_tf::EntitySet organs("Organ", 5); +organs.set("id", std::vector{1, 2}) + .set("parent", std::vector{-1, 1}) + .set("edge_type", {"none", "branch"}) + .set("length", std::vector{0.12, 0.05}); +fspm_tf::Variable declaration; +declaration.unit = "m"; +declaration.standard_name = "length"; +organs.declare("length", declaration); +fspm_tf::DataStructure plants("plants"); +plants.entity_sets.push_back(organs); +fspm_tf::Frame written; +written.data_structures.push_back(plants); +fspm_tf::write_frame("plants.glb", written); +``` + +A column holds a vector of numbers (any integer or float type), `bool` or `std::string`. With a `width` of 2 or +more, it holds a fixed-size vector per row. NaN and infinities are written with the specification's stand-ins and +read back as they were. + +## CPlantBox + +Build the adapter against CPlantBox's sources and its library: + +```bash +cmake -S CPlantBox -B cpb -DCPB_BUILD_PYTHON=OFF -DPIAFMUNCH=OFF -DCPB_BUILD_SHARED=OFF && cmake --build cpb --target CPlantBox +cmake -S bindings/cpp -B build -DFSPM_TF_CPLANTBOX_SOURCE_DIR=$PWD/CPlantBox \ + -DFSPM_TF_CPLANTBOX_LIBRARY=$PWD/cpb/src/libCPlantBox.a +``` + +```cpp +#include + +auto plant = std::make_shared(1); +plant->readParameters("fspm2023.xml"); +plant->initialize(); +plant->simulate(28., false); + +auto frame = fspm_tf::cplantbox::to_frame(*plant); // Plant, Organ, SubOrgan; time in days +fspm_tf::cplantbox::add_geometry(frame.data_structures[0], plant); // leaves and stems, triangles +plant->setRectangularGrid({-10, -10, -40}, {10, 10, 0}, {4, 4, 8}, false); +auto soil = fspm_tf::cplantbox::soil_grid(*plant); +frame.mappings.push_back(fspm_tf::cplantbox::segments_in_soil(*plant, frame.data_structures[0], soil)); +frame.data_structures.push_back(soil); +fspm_tf::write_frame("plant.glb", frame); + +// A frame (CPlantBox's, or another platform's) as a static CPlantBox root system +auto roots = fspm_tf::cplantbox::to_root_system(fspm_tf::read_frame("roots.glb").data_structures[0]); +``` + +| fspm-TF | CPlantBox | +|---|---| +| Plant (rank 1), id 0 | the organism | +| Organ (rank 5), id 1 + organ id | a seed, a root or a stem (labelled `RootAxis`, `ShootAxis`: a whole axis), a leaf | +| SubOrgan (rank 6), id 2^32 + distal node id | a segment; its parent is the segment ending at its proximal node | +| `x1` … `z2` (`proximal_end_*`, `distal_end_*`) | the segment's nodes, cm converted to metres | +| other variables | CPlantBox's names and units (`length` in cm, `age` in d, `order`, `subType`, …), with their standard names | +| `extras["cplantbox"]` | the id offsets, the organism's parameters as CPlantBox's XML | + +CPlantBox has two scales below the plant. `to_root_system` reads frames with more scales by what they hold: the +segments are the entity set with segment ends, and the organs are the set their `complex` points to. For example, +metafspm's Plant, Axis and SubOrgan become CPlantBox root organs. CPlantBox has static roots only, so stems and +leaves are not rebuilt. diff --git a/doc/index.md b/doc/index.md index e814dbf..e0d8852 100644 --- a/doc/index.md +++ b/doc/index.md @@ -34,6 +34,7 @@ Viewing and checking files Reading and writing in Python A notebook: fspm-TF in Python Reading and writing in Julia +Reading and writing in C++, and CPlantBox ``` ## Implementing fspm-TF diff --git a/doc/platforms.md b/doc/platforms.md index b570ef5..bc4e9b4 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -26,7 +26,7 @@ What it has to handle: This is a few hundred lines in any language. Existing libraries help: -- **C++:** cesium-native reads and writes `EXT_structural_metadata`. +- **C++:** done, `bindings/cpp/` ([C++ and CPlantBox](cpp.md)): hand-written, with nlohmann_json. - **Java:** JglTF has its JSON classes. - **Julia:** done, `bindings/julia/FspmTF/` ([FspmTF.jl](julia.md)): a hand-written GLB container and JSON.jl. diff --git a/implementations.md b/implementations.md index 9fc9d94..7e3b9d8 100644 --- a/implementations.md +++ b/implementations.md @@ -4,7 +4,8 @@ |---|---|---|---|---|---| | openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.1 | | openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.1 | -| CPlantBox | C++ / Python | planned (phase P4) | | | | +| fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.1 | +| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.1 | | GroIMP | Java | | | | | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.1 | | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.1 | From 2b7a6a10d638ec4d7fd5de215ec113c3c626482f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 13:01:17 +0100 Subject: [PATCH 07/39] FspmTF.jl and VirtualPlantLab's own graphs: package extensions for PlantGraphs (from_graph, to_graph: one entity set, the nodes' types as labels and their fields as variables, the graph's data as scalars, the turtle's movements left out, edge types guessed; nodes rebuilt from their rows) and PlantGeomTurtle (add_geometry!: feed! replayed, each triangle with its node, the mesh equal to PlantGeomPrimitives.Mesh(graph)); their tests in their own project (PlantGraphs needs Julia 1.12) and CI job; the attribute-to-column conversion shared by the adapters (src/values.jl) --- .github/workflows/julia-codec.yml | 14 ++ CHANGELOG.md | 2 + bindings/julia/FspmTF/Project.toml | 8 + .../ext/FspmTFMultiScaleTreeGraphExt.jl | 37 +--- .../FspmTF/ext/FspmTFPlantGeomTurtleExt.jl | 65 ++++++ .../julia/FspmTF/ext/FspmTFPlantGraphsExt.jl | 192 ++++++++++++++++++ bindings/julia/FspmTF/src/FspmTF.jl | 33 ++- bindings/julia/FspmTF/src/values.jl | 36 ++++ .../FspmTF/test/plantgraphs/Project.toml | 12 ++ .../julia/FspmTF/test/plantgraphs/runtests.jl | 79 +++++++ bindings/julia/README.md | 7 +- doc/julia.md | 34 +++- implementations.md | 2 +- 13 files changed, 479 insertions(+), 42 deletions(-) create mode 100644 bindings/julia/FspmTF/ext/FspmTFPlantGeomTurtleExt.jl create mode 100644 bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl create mode 100644 bindings/julia/FspmTF/src/values.jl create mode 100644 bindings/julia/FspmTF/test/plantgraphs/Project.toml create mode 100644 bindings/julia/FspmTF/test/plantgraphs/runtests.jl diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml index ef928cd..c8a3f1a 100644 --- a/.github/workflows/julia-codec.yml +++ b/.github/workflows/julia-codec.yml @@ -37,6 +37,20 @@ jobs: with: project: bindings/julia/FspmTF + # VirtualPlantLab's PlantGraphs needs Julia 1.12 or later: its extensions are tested in their own project + virtualplantlab: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - name: PlantGraphs and PlantGeomTurtle extensions + run: | + julia --project=bindings/julia/FspmTF/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/FspmTF"); Pkg.instantiate()' + julia --project=bindings/julia/FspmTF/test/plantgraphs bindings/julia/FspmTF/test/plantgraphs/runtests.jl + conformance: runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f0b0e..c25317e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ - The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph with segment ends); its CI, `.github/workflows/cpp-codec.yml`. +- FspmTF.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry + (PlantGeomTurtle: `add_geometry!`), as package extensions. - FspmTF.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `fspm_tf_id`), and parents in another complex kept (`fspm_tf_parent`). - The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/bindings/julia/FspmTF/Project.toml b/bindings/julia/FspmTF/Project.toml index 7a621ce..5fde8d9 100644 --- a/bindings/julia/FspmTF/Project.toml +++ b/bindings/julia/FspmTF/Project.toml @@ -10,14 +10,22 @@ OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" [weakdeps] MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" +PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" +PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" [extensions] FspmTFMultiScaleTreeGraphExt = "MultiScaleTreeGraph" +FspmTFPlantGeomTurtleExt = ["PlantGraphs", "PlantGeomTurtle", "PlantGeomPrimitives"] +FspmTFPlantGraphsExt = "PlantGraphs" [compat] Dates = "1" JSON = "1" MultiScaleTreeGraph = "0.16" +PlantGeomPrimitives = "1" +PlantGeomTurtle = "1" +PlantGraphs = "1" OrderedCollections = "1, 2" julia = "1.10" diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl index 4497d57..0c4a38c 100644 --- a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -40,7 +40,6 @@ const EDGE_TYPE_ATTRIBUTE = :fspm_tf_edge_type const OWN_ATTRIBUTES = (ID_ATTRIBUTE, PARENT_ATTRIBUTE, EDGE_TYPE_ATTRIBUTE) const LINK_NAMES = Dict(:< => "successor", :+ => "branch", :/ => "decomposition") const LINKS = Dict(name => link for (link, name) in LINK_NAMES) -const TEXT_TYPES = Dict("Date" => Date, "DateTime" => DateTime, "Time" => Time) const INDEX_COLUMN = "mtg_index" const INDEX_VARIABLE = Variable(unit = "1", kind = "descriptor", role = "state", description = "the node's index in its MTG code (1 in Internode1)") @@ -72,38 +71,6 @@ default_set_name(scale_number, symbols) = # ── Attributes as columns ─────────────────────────────────────────────── -const TextType = Union{Date,DateTime,Time} - -"""A column of `values` (`nothing` where a node has no such attribute): the column, its missing rows (0-based), -the type written as text (or nothing); nothing for values fspm-TF cannot hold.""" -function attribute_column(values::Vector) - present = [value for value in values if value !== nothing] - missing_rows = [row - 1 for (row, value) in enumerate(values) if value === nothing] - text_type = nothing - if all(value -> value isa Bool, present) - column = Bool[value === nothing ? false : value for value in values] - elseif all(value -> value isa Integer && !(value isa Bool), present) - column = Int64[value === nothing ? 0 : value for value in values] - elseif all(value -> value isa Real && !(value isa Bool), present) - column = Float64[value === nothing ? NaN : value for value in values] - elseif all(value -> value isa Union{AbstractString,Symbol}, present) - column = String[value === nothing ? "" : string(value) for value in values] - elseif !isempty(present) && all(value -> value isa TextType, present) && length(Set(typeof.(present))) == 1 - text_type = string(nameof(typeof(first(present)))) - column = String[value === nothing ? "" : string(value) for value in values] - elseif all(value -> value isa AbstractVector{<:Real}, present) && length(Set(length.(present))) == 1 && - length(first(present)) >= 2 - width = length(first(present)) - integer = all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) - column = integer ? zeros(Int64, length(values), width) : fill(NaN, length(values), width) - for (row, value) in enumerate(values) - value === nothing || (column[row, :] .= value) - end - else - return nothing - end - return column, missing_rows, text_type -end """Columns of `nodes`' attributes, by name in order of first appearance; the missing rows and text types of each. `skip`: attributes not written; the MTG's root's metadata (MTG.jl's `symbols`, `scales`, `description`) neither.""" @@ -119,7 +86,7 @@ function attribute_columns(nodes, set_name, variables, skip = ()) missing_rows, text_types = json_object(), json_object() for name in names values = Any[attribute_of(node, name) for node in nodes] - converted = attribute_column(values) + converted = FspmTF.column_of_values(values) if converted === nothing @warn "attribute '$name' of '$set_name' not written: fspm-TF holds numbers, booleans, text, dates " * "and fixed-size numeric vectors" @@ -272,7 +239,7 @@ attribute_value(column::AbstractMatrix, row) = collect(column[row, :]) function restore(value, text_type) text_type === nothing && return value - return TEXT_TYPES[text_type](value) + return FspmTF.TEXT_TYPES[text_type](value) end """The attributes of each row of `table`: Dict(column => value), without the rows' missing ones.""" diff --git a/bindings/julia/FspmTF/ext/FspmTFPlantGeomTurtleExt.jl b/bindings/julia/FspmTF/ext/FspmTFPlantGeomTurtleExt.jl new file mode 100644 index 0000000..6f734e7 --- /dev/null +++ b/bindings/julia/FspmTF/ext/FspmTFPlantGeomTurtleExt.jl @@ -0,0 +1,65 @@ +""" +A VirtualPlantLab graph's geometry (PlantGeomTurtle: each node's `feed!` adds triangles to a turtle's mesh) as a mesh of +its data structure, each vertex carrying the row of the node that made it: the node itself when it is an entity, else +(a turtle's movement) its nearest kept ancestor. +""" +module FspmTFPlantGeomTurtleExt + +using FspmTF +using FspmTF: NO_ID +import PlantGeomPrimitives as PGP +import PlantGeomTurtle as PGT +import PlantGraphs as PG + +static_of(graph::PG.Graph) = PG.static_graph(graph) +static_of(graph::PG.StaticGraph) = graph +graph_data(graph::PG.Graph) = PG.data(graph) +graph_data(::PG.StaticGraph) = nothing + +""" + add_geometry!(data_structure, graph; entity_set="Organ", turtle=Turtle(Float64), name="plant") -> Mesh + +Runs the turtle over the graph as `feed!(turtle, graph)` does (the same order, the same branching states), records +which node made each triangle, and adds the mesh to the data structure (written by `from_graph` from the same graph). +The positions are the turtle's (VPL: metres, z up). The triangles' colours and materials are not written (fspm-TF has +no slot for them). +""" +function FspmTF.add_geometry!(data_structure::DataStructure, graph::Union{PG.Graph,PG.StaticGraph}; + entity_set::AbstractString = "Organ", turtle = PGT.Turtle(Float64), + name::AbstractString = "plant") + static = static_of(graph) + set = FspmTF.entity_set(data_structure, entity_set) + row_of = Dict(id => row - 1 for (row, id) in enumerate(set.columns["id"])) + mesh = PGP.Mesh(turtle) + vertex_rows = Int64[] + # As PlantGeomTurtle's feed!(turtle, graph): depth first, the turtle's state saved before each branch + stack = Tuple{PG.GraphNode,Int64}[(PG.getroot(static), NO_ID)] # node, its nearest entity's row + while !isempty(stack) + node, ancestor_row = pop!(stack) + before = length(PGP.vertices(mesh)) + PGT.feed!(turtle, PG.data(node), graph_data(graph)) + own = PG.self_id(node) >= 0 ? get(row_of, PG.self_id(node), nothing) : nothing # SET nodes: id -1 + row = own === nothing ? ancestor_row : own + append!(vertex_rows, fill(row, length(PGP.vertices(mesh)) - before)) + children = collect(PG.children(node, static)) + if length(children) == 1 + push!(stack, (only(children), row)) + elseif length(children) > 1 + for child in children + push!(stack, (PG.GraphNode(PGT.SET(to = PGT.pos(turtle), head = PGT.head(turtle), + up = PGT.up(turtle), arm = PGT.arm(turtle))), row)) + push!(stack, (child, row)) + end + end + end + vertices = PGP.vertices(mesh) + positions = Matrix{Float64}(undef, length(vertices), 3) + for (index, vertex) in enumerate(vertices) + positions[index, :] .= (vertex[1], vertex[2], vertex[3]) + end + added = Mesh(name; positions = positions, feature_rows = Dict(entity_set => vertex_rows)) + push!(data_structure.meshes, added) + return added +end + +end diff --git a/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl new file mode 100644 index 0000000..a9e9ffc --- /dev/null +++ b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl @@ -0,0 +1,192 @@ +""" +A VirtualPlantLab graph (PlantGraphs.jl: a `Graph` or a `StaticGraph`) as an fspm-TF `graph` data structure, and back: + +| fspm-TF | PlantGraphs | +|---|---| +| one entity set (default `Organ`, rank 5) | the graph's nodes, but the turtle's movements (PlantGeomTurtle: `T`, `RU`, `F`, `SET`, …) | +| `id`, `label` | the node's id, the name of its type (`Internode`, `Leaf`, …) | +| `parent` | its nearest kept ancestor | +| `edge_type` | guessed: PlantGraphs stores none; a node's last kept child follows it (`successor`), the others branch | +| variables | the fields of each node's type (missing in other types: see below) | +| scalars | the graph's data (`data(graph)`), its fields | + +PlantGraphs has one scale. What fspm-TF 0.1 cannot say is kept in `extras["VirtualPlantLab"]`: the rows where a +node's type has no such field, the types written as text (dates), each type's field names (to rebuild nodes with +`to_graph`), and that edge types were guessed. +""" +module FspmTFPlantGraphsExt + +using FspmTF +using FspmTF: Columns, Declarations, json_object, check_identifier, column_of_values +using OrderedCollections: OrderedDict +import PlantGraphs as PG + +const PLATFORM = "VirtualPlantLab" +const IDENTIFIER = r"^[a-zA-Z_][a-zA-Z0-9_]*$" + +"""Whether a node is kept: the turtle's movements and state nodes (PlantGeomTurtle's types) are not.""" +default_keep(node_data) = nameof(parentmodule(typeof(node_data))) != :PlantGeomTurtle + +label_of(node_data) = string(nameof(typeof(node_data))) + +field_values(object) = object === nothing ? Pair{Symbol,Any}[] : + [name => getfield(object, name) for name in fieldnames(typeof(object))] + +"""Columns of the objects' fields, by name in order of first appearance; their missing rows and text types.""" +function field_columns(objects, set_name, variables) + names = Symbol[] + values_of = [Dict{Symbol,Any}(field_values(object)) for object in objects] + for object in objects, (name, _) in field_values(object) + name in names || push!(names, name) + end + columns, declarations, missing_rows, text_types = Columns(), Declarations(), json_object(), json_object() + for name in names + column_name = string(name) + if !occursin(IDENTIFIER, column_name) || column_name in FspmTF.ENTITY_SET_RESERVED + @warn "field '$name' of '$set_name' not written: not an identifier, or a reserved column name" + continue + end + converted = column_of_values(Any[get(values, name, nothing) for values in values_of]) + if converted === nothing + @warn "field '$name' of '$set_name' not written: fspm-TF holds numbers, booleans, text, dates and " * + "fixed-size numeric vectors" + continue + end + column, missing_of_column, text_type = converted + columns[column_name] = column + isempty(missing_of_column) || (missing_rows[column_name] = missing_of_column) + text_type === nothing || (text_types[column_name] = text_type) + haskey(variables, column_name) && (declarations[column_name] = variables[column_name]) + end + return columns, declarations, missing_rows, text_types +end + +static_of(graph::PG.Graph) = PG.static_graph(graph) +static_of(graph::PG.StaticGraph) = graph +graph_data(graph::PG.Graph) = PG.data(graph) +graph_data(::PG.StaticGraph) = nothing + +"""The graph's kept nodes in depth-first order (children in their order), each with its nearest kept ancestor's id.""" +function kept_nodes(static, keep) + rows = Tuple{PG.GraphNode,Int}[] # node, its kept parent's id (NO_ID: none) + stack = Tuple{PG.GraphNode,Int}[(PG.getroot(static), NO_ID)] + while !isempty(stack) + node, kept_parent = pop!(stack) + parent_of_children = kept_parent + if keep(PG.data(node)) + push!(rows, (node, kept_parent)) + parent_of_children = PG.self_id(node) + end + for child in reverse(collect(PG.children(node, static))) + push!(stack, (child, parent_of_children)) + end + end + return rows +end + +""" + from_graph(graph; id="plant", entity_set="Organ", rank=5, variables=Dict(), keep=default_keep, + platform="VirtualPlantLab", model=nothing) -> DataStructure + +The PlantGraphs graph as a `graph` data structure of one entity set. `variables`: the fields' declarations (name => +`Variable`). `keep`: which nodes (by their data) are entities; by default all but the turtle's movements. +""" +function FspmTF.from_graph(graph::Union{PG.Graph,PG.StaticGraph}; id::AbstractString = "plant", + entity_set::AbstractString = "Organ", rank = 5, variables = Dict{String,Variable}(), + keep = default_keep, platform = PLATFORM, model = nothing) + check_identifier(entity_set, "entity set name") + variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) + static = static_of(graph) + rows = kept_nodes(static, keep) + ids = Int64[PG.self_id(node) for (node, _) in rows] + parents = Int64[parent for (_, parent) in rows] + last_child = Dict{Int64,Int64}() # each parent's last kept child + for (child, parent) in zip(ids, parents) + parent == NO_ID || (last_child[parent] = child) + end + edge_types = String[parent == NO_ID ? "none" : last_child[parent] == child ? "successor" : "branch" + for (child, parent) in zip(ids, parents)] + objects = [PG.data(node) for (node, _) in rows] + columns = Columns("id" => ids, "label" => String[label_of(object) for object in objects], "parent" => parents, + "edge_type" => edge_types) + fields, declarations, missing_rows, text_types = field_columns(objects, entity_set, variables) + merge!(columns, fields) + extras = json_object("edge_types" => "guessed: PlantGraphs stores none; a node's last child follows it, the others branch", + "types" => json_object(label_of(object) => string.(fieldnames(typeof(object))) + for object in unique(object -> typeof(object), objects))) + isempty(missing_rows) || (extras["missing"] = json_object(entity_set => missing_rows)) + isempty(text_types) || (extras["text_types"] = json_object(entity_set => text_types)) + + scalars = nothing + data = graph_data(graph) + if data !== nothing && !isempty(fieldnames(typeof(data))) + scalar_columns, scalar_declarations, missing_scalars, text_scalars = field_columns([data], "scalars", variables) + if !isempty(scalar_columns) + scalars = Table("scalars"; columns = scalar_columns, variables = scalar_declarations) + extras["data_type"] = string(nameof(typeof(data))) + isempty(text_scalars) || (extras["text_types"] = merge(get(extras, "text_types", json_object()), + json_object("scalars" => text_scalars))) + end + end + set = EntitySet(entity_set; columns = columns, variables = declarations, rank = rank) + return DataStructure(id; kind = "graph", entity_sets = [set], scalars = scalars, platform = platform, + model = model, extras = json_object(PLATFORM => extras)) +end + +# ── Data structure → graph ────────────────────────────────────────────── + +row_value(column::AbstractVector, row) = column[row] +row_value(column::AbstractMatrix, row) = collect(column[row, :]) + +""" + to_graph(data_structure; types, entity_set=nothing) -> (StaticGraph, ids) + +The data structure's entities as a PlantGraphs `StaticGraph`, each built as `types[label](fields...)` from its row, +its type's fields in their order (`types`: label => node type, e.g. `Dict("Internode" => Internode)`). `entity_set`: +the set read (by default the finest one with parents). `ids`: the entity ids => their ids in the graph (PlantGraphs +gives new ids). Wrap the graph as VPL does: `Graph(axiom = static_graph, rules = ..., data = ...)`. +""" +function FspmTF.to_graph(data_structure::DataStructure; types::AbstractDict, entity_set = nothing) + candidates = [set for set in data_structure.entity_sets if haskey(set.columns, "parent")] + set = entity_set === nothing ? + (isempty(candidates) ? throw(ArgumentError("'$(data_structure.id)': no entity set with parents")) : + last(sort(candidates; by = set -> something(set.rank, 0)))) : + FspmTF.entity_set(data_structure, entity_set) + extras = get(data_structure.extras, PLATFORM, json_object()) + missing_rows = Dict(name => Set(rows) for (name, rows) in pairs(get(get(extras, "missing", json_object()), set.name, json_object()))) + text_types = get(get(extras, "text_types", json_object()), set.name, json_object()) + labels = set.columns["label"] + objects = Vector{Any}(undef, row_count(set)) + for row in 1:row_count(set) + haskey(types, labels[row]) || throw(ArgumentError("no node type given for label '$(labels[row])'")) + node_type = types[labels[row]] + arguments = map(fieldnames(node_type)) do field + name = string(field) + haskey(set.columns, name) && !((row - 1) in get(missing_rows, name, Set{Int}())) || + throw(ArgumentError("'$(labels[row])' row $row: no value for field '$name'")) + value = row_value(set.columns[name], row) + haskey(text_types, name) ? FspmTF.TEXT_TYPES[text_types[name]](value) : value + end + objects[row] = node_type(arguments...) + end + ids, parents = set.columns["id"], set.columns["parent"] + children_of = Dict{Int64,Vector{Int}}() + roots = Int[] + for row in 1:row_count(set) + parents[row] == NO_ID ? push!(roots, row) : push!(get!(children_of, parents[row], Int[]), row) + end + length(roots) == 1 || throw(ArgumentError("'$(set.name)': $(length(roots)) roots, a PlantGraphs graph has one")) + static = PG.StaticGraph(PG.GraphNode(objects[only(roots)])) + graph_ids = Dict{Int64,Int}(ids[only(roots)] => PG.self_id(PG.getroot(static))) + stack = [only(roots)] + while !isempty(stack) + row = pop!(stack) + for child in get(children_of, ids[row], Int[]) + graph_ids[ids[child]] = append!(static, graph_ids[ids[row]], PG.GraphNode(objects[child])) + push!(stack, child) + end + end + return static, graph_ids +end + +end diff --git a/bindings/julia/FspmTF/src/FspmTF.jl b/bindings/julia/FspmTF/src/FspmTF.jl index 13b8249..2620bb7 100644 --- a/bindings/julia/FspmTF/src/FspmTF.jl +++ b/bindings/julia/FspmTF/src/FspmTF.jl @@ -10,10 +10,13 @@ binary (GLB) file per time step. The specification is `spec/specification.md` in read_index(path), read_sequence(path) write_sequence(directory, frames) frame_to_dict(frame), write_dump(path, frame) the layout of the conformance files' expected values -With MultiScaleTreeGraph loaded, `from_mtg` and `to_mtg` convert an MTG to and from a data structure. +With MultiScaleTreeGraph loaded, `from_mtg` and `to_mtg` convert an MTG to and from a data structure. With +VirtualPlantLab's PlantGraphs loaded, `from_graph` and `to_graph` convert its graphs; with PlantGeomTurtle too, +`add_geometry!` adds a graph's turtle geometry as a mesh. """ module FspmTF +using Dates using JSON using OrderedCollections: OrderedDict @@ -25,11 +28,12 @@ include("model.jl") include("codec.jl") include("sequence.jl") include("dump.jl") +include("values.jl") export Frame, DataStructure, EntitySet, Relation, Table, Mapping, Grid, Mesh, Variable, NO_ID, EDGE_TYPES export read_frame, write_frame, from_bytes, to_bytes, read_index, read_sequence, write_sequence, write_index export frame_to_dict, write_dump, row_count, variable_names, entity_ids, entity_set, data_structure -export from_mtg, to_mtg +export from_mtg, to_mtg, from_graph, to_graph, add_geometry! """ from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) @@ -48,4 +52,29 @@ package extension `FspmTFMultiScaleTreeGraphExt`). """ function to_mtg end +""" + from_graph(graph; id="plant", entity_set="Organ", rank=5, variables=Dict(), keep, platform="VirtualPlantLab", + model=nothing) -> DataStructure + +A VirtualPlantLab graph (PlantGraphs: `Graph` or `StaticGraph`) as a `graph` data structure of one entity set. +Defined when PlantGraphs is loaded (the package extension `FspmTFPlantGraphsExt`). +""" +function from_graph end + +""" + to_graph(data_structure; types, entity_set=nothing) -> (StaticGraph, ids) + +A data structure's entities as a PlantGraphs `StaticGraph`, each node built from its row by `types[label]`. Defined +when PlantGraphs is loaded (the package extension `FspmTFPlantGraphsExt`). +""" +function to_graph end + +""" + add_geometry!(data_structure, graph; entity_set="Organ", turtle=Turtle(Float64), name="plant") -> Mesh + +Adds a VirtualPlantLab graph's turtle geometry to its data structure as a mesh, each vertex carrying its node's row. +Defined when PlantGraphs, PlantGeomTurtle and PlantGeomPrimitives are loaded (`FspmTFPlantGeomTurtleExt`). +""" +function add_geometry! end + end diff --git a/bindings/julia/FspmTF/src/values.jl b/bindings/julia/FspmTF/src/values.jl new file mode 100644 index 0000000..02f84f3 --- /dev/null +++ b/bindings/julia/FspmTF/src/values.jl @@ -0,0 +1,36 @@ +# Values of a platform's objects (attributes, fields) as columns: what the adapters share. + +const TextType = Union{Date,DateTime,Time} +"""The types written as ISO 8601 text, by name (an adapter records the name to read them back).""" +const TEXT_TYPES = Dict("Date" => Date, "DateTime" => DateTime, "Time" => Time) + +"""A column of `values` (`nothing` where a node has no such attribute): the column, its missing rows (0-based), +the type written as text (or nothing); nothing for values fspm-TF cannot hold.""" +function column_of_values(values::AbstractVector) + present = [value for value in values if value !== nothing] + missing_rows = [row - 1 for (row, value) in enumerate(values) if value === nothing] + text_type = nothing + if all(value -> value isa Bool, present) + column = Bool[value === nothing ? false : value for value in values] + elseif all(value -> value isa Integer && !(value isa Bool), present) + column = Int64[value === nothing ? 0 : value for value in values] + elseif all(value -> value isa Real && !(value isa Bool), present) + column = Float64[value === nothing ? NaN : value for value in values] + elseif all(value -> value isa Union{AbstractString,Symbol}, present) + column = String[value === nothing ? "" : string(value) for value in values] + elseif !isempty(present) && all(value -> value isa TextType, present) && length(Set(typeof.(present))) == 1 + text_type = string(nameof(typeof(first(present)))) + column = String[value === nothing ? "" : string(value) for value in values] + elseif all(value -> value isa AbstractVector{<:Real}, present) && length(Set(length.(present))) == 1 && + length(first(present)) >= 2 + width = length(first(present)) + integer = all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) + column = integer ? zeros(Int64, length(values), width) : fill(NaN, length(values), width) + for (row, value) in enumerate(values) + value === nothing || (column[row, :] .= value) + end + else + return nothing + end + return column, missing_rows, text_type +end diff --git a/bindings/julia/FspmTF/test/plantgraphs/Project.toml b/bindings/julia/FspmTF/test/plantgraphs/Project.toml new file mode 100644 index 0000000..638865a --- /dev/null +++ b/bindings/julia/FspmTF/test/plantgraphs/Project.toml @@ -0,0 +1,12 @@ +[deps] +FspmTF = "b7fd6adf-cc0c-448b-9b1a-23349074d866" +PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" +PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[compat] +PlantGeomPrimitives = "1" +PlantGeomTurtle = "1" +PlantGraphs = "1" +julia = "1.12" diff --git a/bindings/julia/FspmTF/test/plantgraphs/runtests.jl b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl new file mode 100644 index 0000000..c4a369f --- /dev/null +++ b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl @@ -0,0 +1,79 @@ +# FspmTF.jl's VirtualPlantLab extensions (PlantGraphs, PlantGeomTurtle; Julia 1.12 or later, as PlantGraphs). Their own +# project, so that the package's tests keep running on Julia 1.10: +# +# julia --project=bindings/julia/FspmTF/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/FspmTF"); Pkg.instantiate()' +# julia --project=bindings/julia/FspmTF/test/plantgraphs bindings/julia/FspmTF/test/plantgraphs/runtests.jl +using FspmTF +using PlantGeomPrimitives +using PlantGeomTurtle +using PlantGraphs +using Test + +# A small VPL plant: internodes as cylinders, leaves as ellipses, the turtle's rotations between them +struct Internode <: PlantGraphs.Node + length::Float64 + width::Float64 +end +struct Leaf <: PlantGraphs.Node + length::Float64 + width::Float64 + age::Int +end +struct Meristem <: PlantGraphs.Node end +struct Weather + temperature::Float64 +end + +PlantGeomTurtle.feed!(turtle::Turtle, node::Internode, data) = + HollowCylinder!(turtle; length = node.length, width = node.width, height = node.width, move = true) +PlantGeomTurtle.feed!(turtle::Turtle, node::Leaf, data) = + Ellipse!(turtle; length = node.length, width = node.width, move = false) + +plant() = Graph(axiom = Internode(0.1, 0.01) + (RU(30.0) + Leaf(0.05, 0.02, 3), RU(-30.0) + Leaf(0.04, 0.02, 1)) + + Internode(0.08, 0.008) + Meristem(), + data = Weather(20.5)) + +@testset "FspmTF and VirtualPlantLab" begin + @testset "A graph as a data structure, written, read and rebuilt" begin + graph = plant() + written = from_graph(graph; id = "plant", + variables = Dict("length" => Variable(unit = "m", standard_name = "length", kind = "extensive"))) + organs = entity_set(written, "Organ") + @test organs.columns["label"] == ["Internode", "Leaf", "Leaf", "Internode", "Meristem"] # no turtle moves + ids = organs.columns["id"] + @test organs.columns["parent"] == [-1, ids[1], ids[1], ids[1], ids[4]] # the leaves on the first internode + @test organs.columns["edge_type"] == ["none", "branch", "branch", "successor", "successor"] + @test isequal(organs.columns["length"], [0.1, 0.05, 0.04, 0.08, NaN]) # NaN: the meristem has none + @test organs.columns["age"] == [0, 3, 1, 0, 0] # missing where no field + @test written.extras["VirtualPlantLab"]["missing"]["Organ"]["age"] == [0, 3, 4] + @test written.scalars.columns["temperature"] == [20.5] + + frame = from_bytes(to_bytes(Frame(data_structures = [written]))) + read_back = frame.data_structures[1] + static, graph_ids = to_graph(read_back; types = Dict("Internode" => Internode, "Leaf" => Leaf, + "Meristem" => Meristem)) + again = from_graph(static; keep = node -> true) + again_organs = entity_set(again, "Organ") + @test again_organs.columns["label"] == organs.columns["label"] + @test isequal(again_organs.columns["length"], organs.columns["length"]) + @test again_organs.columns["age"] == organs.columns["age"] + @test again_organs.columns["edge_type"] == organs.columns["edge_type"] + @test again_organs.columns["parent"] == [-1; [graph_ids[parent] for parent in organs.columns["parent"][2:end]]] + end + + @testset "Its turtle geometry, each triangle with its node" begin + graph = plant() + written = from_graph(graph) + mesh = add_geometry!(written, graph) + reference = PlantGeomPrimitives.Mesh(graph) # VPL's own mesh (both packages export a Mesh) + @test size(mesh.positions, 1) == length(vertices(reference)) + @test all(mesh.positions[index, :] == collect(vertices(reference)[index]) for index in 1:length(vertices(reference))) + rows = mesh.feature_rows["Organ"] + labels = entity_set(written, "Organ").columns["label"] + @test all(row != NO_ID for row in rows) + @test Set(labels[row + 1] for row in rows) == Set(["Internode", "Leaf"]) # meristems have no geometry + path = joinpath(mktempdir(), "vpl_plant.glb") + write_frame(path, Frame(data_structures = [written])) + @test read_frame(path).data_structures[1].meshes[1].feature_rows == mesh.feature_rows + end +end diff --git a/bindings/julia/README.md b/bindings/julia/README.md index 6ab7077..13381a7 100644 --- a/bindings/julia/README.md +++ b/bindings/julia/README.md @@ -5,8 +5,11 @@ MultiScaleTreeGraph.jl MTG to and from a `graph` data structure (`from_mtg`, `to - `FspmTF/src/`: the GLB container (`glb.jl`), the data model (`model.jl`), reading and writing (`codec.jl`), sequences (`sequence.jl`), the conformance dump (`dump.jl`). -- `FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl`: the MTG adapter, loaded with MultiScaleTreeGraph. -- `FspmTF/test/`: the tests, the conformance files included. +- `FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl`: the MTG adapter, loaded with MultiScaleTreeGraph (PlantSimEngine's, + PlantGeom's graphs). +- `FspmTF/ext/FspmTFPlantGraphsExt.jl`, `FspmTFPlantGeomTurtleExt.jl`: VirtualPlantLab's graphs and their turtle + geometry, loaded with PlantGraphs (and PlantGeomTurtle, PlantGeomPrimitives). +- `FspmTF/test/`: the tests, the conformance files included; `test/plantgraphs/`: VirtualPlantLab's (Julia ≥ 1.12). - `conformance_run.jl`: the codec's dumps of the conformance files and its written scenes, for `.github/actions/conformance`. diff --git a/doc/julia.md b/doc/julia.md index 00e2fdb..163634d 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -1,8 +1,11 @@ # Reading and writing in Julia FspmTF.jl, in `bindings/julia/FspmTF/`, reads and writes fspm-TF files in Julia (1.10 or later). It passes the -conformance check like the Python codec. With MultiScaleTreeGraph.jl loaded, it also turns an MTG into a data -structure, and back. This is the MTG library of VirtualPlantLab's PlantGeom and of PlantSimEngine. +conformance check like the Python codec. Two VirtualPlantLab models are covered: + +- with MultiScaleTreeGraph.jl loaded, it turns an MTG into a data structure, and back. This is the multiscale graph + of PlantSimEngine and PlantGeom; +- with PlantGraphs.jl loaded, it does the same for VPL's own graphs, and with PlantGeomTurtle for their geometry. ## Installing @@ -55,6 +58,33 @@ Some MTG content has no place in fspm-TF 0.1: missing attributes, dates, the MTG the order of children. It is kept in the data structure's `extras["MultiScaleTreeGraph"]`, so that `to_mtg` rebuilds the MTG exactly. Other readers see the missing values as NaN, `false`, 0 or "". +## A VirtualPlantLab graph + +VPL's own graphs (PlantGraphs.jl) have one scale, and their nodes are the model's types (`Internode`, `Leaf`, …). +With PlantGraphs loaded (Julia 1.12 or later), `from_graph` writes one entity set, `Organ` by default, with: + +- each node's type as its label, and its fields as variables; +- the graph's data as scalars; +- its parent as the nearest kept node. + +The turtle's movements (`RU`, `T`, `F`, …) are left out. PlantGraphs stores no edge type, so the last child of a +node is written as its successor and the others as branches; `extras` says the edge types were guessed. With +PlantGeomTurtle loaded too, `add_geometry!` adds the graph's turtle mesh, each triangle carrying the row of the node +that made it. + +```julia +using FspmTF, PlantGraphs, PlantGeomTurtle, PlantGeomPrimitives + +plants = from_graph(graph; id = "plant", + variables = Dict("length" => Variable(unit = "m", standard_name = "length"))) +add_geometry!(plants, graph) # the mesh of PlantGeomPrimitives.Mesh(graph), node by node (both export Mesh) +write_frame("plant.glb", Frame(data_structures = [plants])) + +static, ids = to_graph(read_frame("plant.glb").data_structures[1]; + types = Dict("Internode" => Internode, "Leaf" => Leaf)) # nodes rebuilt from their fields +graph = Graph(axiom = static, rules = rules) +``` + ## The format's own tables ```julia diff --git a/implementations.md b/implementations.md index 7e3b9d8..69b768f 100644 --- a/implementations.md +++ b/implementations.md @@ -9,7 +9,7 @@ | GroIMP | Java | | | | | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.1 | | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.1 | -| VirtualPlantLab (PlantGraphs) | Julia | planned (FspmTF.jl extensions: PlantGeom meshes, PlantGraphs) | | | | +| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.1 | Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters (a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional From 36e9c8a9315af14fa3e1e6a8bbe6431d8358022a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 14:02:58 +0100 Subject: [PATCH 08/39] The Java codec (bindings/java: Java 17, Maven, Jackson): the GLB container, the data model, reading and writing frames and sequences, the conformance dump and tool (every conformance file read as expected; what it writes passes both validators with no warning); GroIMP's adapter (bindings/java/groimp, on GroIMP's Graph module): one entity set per refinement depth (names and ranks given), complex and parent from each node's incoming edges, other edge bits as relations, managed fields as variables, and graphs (GroIMP's or another platform's) back into the model's node classes; CI on Linux, macOS and Windows, conformance, the adapter on a pinned GroIMP; doc/java.md --- .github/workflows/java-codec.yml | 72 ++ CHANGELOG.md | 3 + README.md | 2 +- bindings/java/README.md | 16 + bindings/java/groimp/pom.xml | 67 ++ .../org/openalea/fspmtf/groimp/GroImp.java | 463 ++++++++++ .../openalea/fspmtf/groimp/GroImpTest.java | 123 +++ .../org/openalea/fspmtf/groimp/Organs.java | 168 ++++ bindings/java/pom.xml | 79 ++ .../main/java/org/openalea/fspmtf/Codec.java | 794 ++++++++++++++++++ .../main/java/org/openalea/fspmtf/Column.java | 91 ++ .../org/openalea/fspmtf/ComponentType.java | 33 + .../org/openalea/fspmtf/ConformanceRun.java | 46 + .../org/openalea/fspmtf/DataStructure.java | 40 + .../main/java/org/openalea/fspmtf/Dump.java | 191 +++++ .../java/org/openalea/fspmtf/EntitySet.java | 36 + .../main/java/org/openalea/fspmtf/Frame.java | 27 + .../org/openalea/fspmtf/FspmTfException.java | 14 + .../main/java/org/openalea/fspmtf/Glb.java | 102 +++ .../main/java/org/openalea/fspmtf/Grid.java | 20 + .../java/org/openalea/fspmtf/Mapping.java | 25 + .../main/java/org/openalea/fspmtf/Mesh.java | 28 + .../java/org/openalea/fspmtf/Relation.java | 21 + .../java/org/openalea/fspmtf/Sequences.java | 72 ++ .../main/java/org/openalea/fspmtf/Table.java | 81 ++ .../java/org/openalea/fspmtf/Variable.java | 45 + .../java/org/openalea/fspmtf/CodecTest.java | 192 +++++ doc/index.md | 1 + doc/java.md | 52 ++ doc/platforms.md | 2 +- implementations.md | 3 +- 31 files changed, 2906 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/java-codec.yml create mode 100644 bindings/java/README.md create mode 100644 bindings/java/groimp/pom.xml create mode 100644 bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java create mode 100644 bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java create mode 100644 bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/Organs.java create mode 100644 bindings/java/pom.xml create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Codec.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Column.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/ComponentType.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Dump.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Frame.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/FspmTfException.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Glb.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Grid.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Mapping.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Mesh.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Relation.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Sequences.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Table.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Variable.java create mode 100644 bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java create mode 100644 doc/java.md diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml new file mode 100644 index 0000000..fc9e194 --- /dev/null +++ b/.github/workflows/java-codec.yml @@ -0,0 +1,72 @@ +# The Java codec (bindings/java): its tests on Linux, macOS and Windows; the conformance check other codecs run; and +# GroIMP's adapter (bindings/java/groimp), built on GroIMP's Graph module (a pinned commit). OpenAlea's conda CI builds +# the Python package only. +name: Java codec + +on: + push: + branches: + - main + pull_request: + +env: + GROIMP_COMMIT: 6c5bbdef616f799c2eadec1097e1a312e43f308d + +jobs: + test: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Tests, the conformance files included + run: mvn -B -f bindings/java/pom.xml verify + + conformance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + cache: maven + - name: Dump and write the conformance scenes with the Java codec + run: | + mvn -B -q -f bindings/java/pom.xml package -DskipTests dependency:copy-dependencies -DincludeScope=runtime + java -cp "bindings/java/target/fspm-tf-0.1.0.jar:bindings/java/target/dependency/*" org.openalea.fspmtf.ConformanceRun conformance out + - uses: ./.github/actions/conformance + with: + dumps: out/dumps + written: out/written + + groimp-adapter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + cache: maven + - name: GroIMP's Graph module + run: | + git clone --quiet https://gitlab.com/grogra/groimp.git groimp + git -C groimp checkout --quiet "$GROIMP_COMMIT" + mvn -B -q -f groimp/pom.xml -pl Graph -am install -DskipTests -Dmaven.javadoc.skip=true + - name: The codec and the adapter + run: | + mvn -B -q -f bindings/java/pom.xml install -DskipTests + mvn -B -f bindings/java/groimp/pom.xml verify + - name: What the adapter wrote is valid fspm-TF + run: | + python -m pip install numpy jsonschema . + fspm-tf validate bindings/java/groimp/target/groimp_out/*.glb diff --git a/CHANGELOG.md b/CHANGELOG.md index c25317e..7073796 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ - The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph with segment ends); its CI, `.github/workflows/cpp-codec.yml`. +- The Java codec (`bindings/java/`): conformant reading and writing, sequences; GroIMP's adapter (`bindings/java/groimp/`: + scales from refinement edges, topology, edge bits as relations, managed fields; graphs back into the model's node + classes); its CI, `.github/workflows/java-codec.yml`. - FspmTF.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry (PlantGeomTurtle: `add_geometry!`), as package extensions. - FspmTF.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `fspm_tf_id`), and parents diff --git a/README.md b/README.md index 1192f0e..c329976 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ change there, released as a new version. Everything else implements, checks or d | `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | | `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter | no | +| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter; `bindings/java/`: the Java codec, with GroIMP's adapter | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | | `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | diff --git a/bindings/java/README.md b/bindings/java/README.md new file mode 100644 index 0000000..7065f76 --- /dev/null +++ b/bindings/java/README.md @@ -0,0 +1,16 @@ +# fspm-tf (Java) + +The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.1), with one dependency, +Jackson. GroIMP's adapter is in `groimp/`. + +- `src/main/java/org/openalea/fspmtf/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB + container (`Glb`), reading and writing (`Codec`), sequences (`Sequences`), the conformance dump (`Dump`) and tool + (`ConformanceRun`). +- `src/test/`: the tests, the conformance files included. +- `groimp/`: GroIMP's adapter (`org.openalea.fspmtf.groimp.GroImp`, Java 21), on GroIMP's Graph module. + +```bash +mvn -f bindings/java/pom.xml verify +``` + +See [the documentation](../../doc/java.md). diff --git a/bindings/java/groimp/pom.xml b/bindings/java/groimp/pom.xml new file mode 100644 index 0000000..ffe5842 --- /dev/null +++ b/bindings/java/groimp/pom.xml @@ -0,0 +1,67 @@ + + + + 4.0.0 + + org.openalea + fspm-tf-groimp + 0.1.0 + jar + fspm-TF, GroIMP's adapter + + + 21 + UTF-8 + 3.0.0 + 5.11.4 + + + + + org.openalea + fspm-tf + 0.1.0 + + + de.grogra + graph + ${groimp.version} + provided + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + false + + ${project.basedir}/../../../conformance + ${project.build.directory}/groimp_out + + + + + + diff --git a/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java b/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java new file mode 100644 index 0000000..443c57b --- /dev/null +++ b/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java @@ -0,0 +1,463 @@ +package org.openalea.fspmtf.groimp; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import org.openalea.fspmtf.Column; +import org.openalea.fspmtf.ComponentType; +import org.openalea.fspmtf.DataStructure; +import org.openalea.fspmtf.EntitySet; +import org.openalea.fspmtf.FspmTfException; +import org.openalea.fspmtf.Relation; +import org.openalea.fspmtf.Variable; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.Edge; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ManageableType; +import de.grogra.reflect.TypeId; + +/** + * A GroIMP graph as an fspm-TF graph, and back. + * + *
+ * + * + * + * + * + * + * + * + *
fspm-TFGroIMP
one entity set per scale (rank: the scale, from 1)the nodes at that refinement depth below the + * root; a node reached by a successor or branch edge has its parent's scale, by a refinement edge one more
id{@code Node.getId()}
labelthe node's class's simple name
complexthe source of its incoming refinement edge, else its parent's complex
parent, edge_typethe source of its incoming successor (successor) or branch (branch) edge at the same + * scale
relationsthe other edge bits between exported nodes (containment, mark, notifies, user edges), one + * relation per bit and pair of entity sets
variablesthe nodes' managed fields declared below {@code Node} (numbers, booleans, chars and Strings; + * 3-component vecmath tuples as vectors)
+ * + *

The graph's root (e.g. an RGG root) is the data structure itself, not an entity. What fspm-TF 0.1 cannot say is + * kept in {@code extras["groimp"]}: the rows where a node's class has no such field, and the classes' names.

+ */ +public final class GroImp { + /** The name of GroIMP's entry in a data structure's extras. */ + public static final String PLATFORM = "groimp"; + private static final int TREE_EDGES = Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE | Graph.REFINEMENT_EDGE; + private static final Map RELATION_NAMES = Map.of(Graph.CONTAINMENT_EDGE, "containment", + Graph.CONTAINMENT_END_EDGE, "containment_end", Graph.MARK_EDGE, "mark", Graph.NOTIFIES_EDGE, "notifies"); + + /** How a graph is written. */ + public static final class Options { + /** The data structure's id. */ + public String id = "plants"; + /** The entity set name of each scale (1: the coarsest); by default its nodes' class name when they share one, + * else {@code Scale}. */ + public final Map scaleNames = new HashMap<>(); + /** The rank of each scale's entity set (by default its refinement depth); e.g. the vocabulary's base scales' + * positions (Plant 1, Axis 2, Organ 5, SubOrgan 6). */ + public final Map scaleRanks = new HashMap<>(); + /** The declarations of the fields (by name): unit, standard name, kind, role. */ + public final Map variables = new HashMap<>(); + public String model; + } + + private GroImp() {} + + /** A node's place in the exported graph. */ + private static final class Entity { + final Node node; + final int scale; + Node complex; + Node parent; + String edgeType; + + Entity(Node node, int scale, Node complex, Node parent, String edgeType) { + this.node = node; + this.scale = scale; + this.complex = complex; + this.parent = parent; + this.edgeType = edgeType; + } + } + + /** The graph below {@code root} (not included) as a {@code graph} data structure. */ + public static DataStructure toDataStructure(Node root, Options options) { + // Depth first along successor, branch and refinement edges, each node once, children in edge order + Map entities = new IdentityHashMap<>(); + List order = new ArrayList<>(); + ArrayDeque stack = new ArrayDeque<>(); + List rootChildren = children(new Entity(root, 0, null, null, "none"), root, true); + for (int index = rootChildren.size() - 1; index >= 0; index--) stack.push(rootChildren.get(index)); + while (!stack.isEmpty()) { + Entity entity = stack.pop(); + if (entities.containsKey(entity.node)) continue; + if (entity.node.getId() < 0) + throw new FspmTfException("node " + entity.node + " has no id: it is not in a graph"); + entities.put(entity.node, entity); + order.add(entity); + List next = children(entity, root, false); + for (int index = next.size() - 1; index >= 0; index--) stack.push(next.get(index)); + } + + // Complex and parent from each node's incoming edges, whichever edge reached it first: a lateral's first + // component has both a refinement edge from its complex and a branch edge from its bearer + for (Entity entity : order) { + Node node = entity.node; + Node complex = null, parent = null; + String edgeType = "none"; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getTarget() != node || !entities.containsKey(edge.getSource())) continue; + int bits = edge.getEdgeBits(); + if ((bits & Graph.REFINEMENT_EDGE) != 0) complex = edge.getSource(); + else if ((bits & (Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE)) != 0 + && entities.get(edge.getSource()).scale == entity.scale) { + parent = edge.getSource(); + edgeType = (bits & Graph.SUCCESSOR_EDGE) != 0 ? "successor" : "branch"; + } + } + entity.complex = complex; + entity.parent = parent; + entity.edgeType = edgeType; + } + for (Entity entity : order) { // no refinement edge: its parent's complex (MTG's rule) + Entity ancestor = entity; + while (ancestor.complex == null && ancestor.parent != null) ancestor = entities.get(ancestor.parent); + entity.complex = ancestor.complex; + } + + Map> byScale = new java.util.TreeMap<>(); + for (Entity entity : order) byScale.computeIfAbsent(entity.scale, scale -> new ArrayList<>()).add(entity); + Map setNames = new HashMap<>(); + for (Map.Entry> item : byScale.entrySet()) { + Set classes = new LinkedHashSet<>(); + for (Entity entity : item.getValue()) classes.add(entity.node.getClass().getSimpleName()); + setNames.put(item.getKey(), options.scaleNames.getOrDefault(item.getKey(), + classes.size() == 1 ? classes.iterator().next() : "Scale" + item.getKey())); + } + + DataStructure dataStructure = new DataStructure(options.id, "graph"); + dataStructure.platform = "GroIMP"; + dataStructure.model = options.model; + ObjectNode extras = JsonNodeFactory.instance.objectNode(); + ObjectNode missing = extras.putObject("missing"); + ObjectNode classes = extras.putObject("classes"); + int coarsest = byScale.isEmpty() ? 0 : byScale.keySet().iterator().next(); + for (Map.Entry> item : byScale.entrySet()) { + String setName = setNames.get(item.getKey()); + List rows = item.getValue(); + EntitySet set = new EntitySet(setName, options.scaleRanks.getOrDefault(item.getKey(), (long) item.getKey())); + long[] ids = new long[rows.size()], complexes = new long[rows.size()], parents = new long[rows.size()]; + String[] labels = new String[rows.size()], edgeTypes = new String[rows.size()]; + for (int row = 0; row < rows.size(); row++) { + Entity entity = rows.get(row); + ids[row] = entity.node.getId(); + labels[row] = entity.node.getClass().getSimpleName(); + complexes[row] = entity.complex == null ? EntitySet.NO_ID : entity.complex.getId(); + parents[row] = entity.parent == null ? EntitySet.NO_ID : entity.parent.getId(); + edgeTypes[row] = entity.edgeType; + classes.put(labels[row], entity.node.getClass().getName()); + } + set.set("id", Column.of(ids)).set("label", Column.of(labels)); + if (item.getKey() != coarsest) set.set("complex", Column.of(complexes)); + set.set("parent", Column.of(parents)).set("edge_type", Column.of(edgeTypes)); + ObjectNode missingOfSet = JsonNodeFactory.instance.objectNode(); + addFields(set, rows, options, missingOfSet); + if (!missingOfSet.isEmpty()) missing.set(setName, missingOfSet); + dataStructure.entitySets.add(set); + } + if (missing.isEmpty()) extras.remove("missing"); + addRelations(dataStructure, order, entities, setNames); + dataStructure.extras.set(PLATFORM, extras); + return dataStructure; + } + + /** The nodes {@code entity}'s outgoing tree edges lead to, as entities. */ + private static List children(Entity entity, Node root, boolean fromRoot) { + List found = new ArrayList<>(); + Node node = entity.node; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getSource() != node) continue; + int bits = edge.getEdgeBits(); + if ((bits & TREE_EDGES) == 0) continue; + Node target = edge.getTarget(); + if (target == root) continue; + if (fromRoot) { // the root's children: the coarsest scale's first nodes + found.add(new Entity(target, 1, null, null, "none")); + } else if ((bits & Graph.REFINEMENT_EDGE) != 0) { // one scale finer: its complex + found.add(new Entity(target, entity.scale + 1, node, null, "none")); + } else { // the same scale: its parent + found.add(new Entity(target, entity.scale, entity.complex, node, + (bits & Graph.SUCCESSOR_EDGE) != 0 ? "successor" : "branch")); + } + } + return found; + } + + private static boolean isOwnField(ManageableType.Field field) { + return field.getDeclaringType() != Node.$TYPE && (field.getModifiers() & ManageableType.Field.HIDDEN) == 0; + } + + /** A field's value, as a column value: Double, Float, Long, Integer, Short, Byte, Boolean, String, double[3]; null + * when not written. */ + private static Object fieldValue(ManageableType.Field field, Node node) { + return switch (field.getType().getTypeId()) { + case TypeId.BOOLEAN -> field.getBoolean(node); + case TypeId.BYTE -> field.getByte(node); + case TypeId.SHORT -> field.getShort(node); + case TypeId.CHAR -> String.valueOf(field.getChar(node)); + case TypeId.INT -> field.getInt(node); + case TypeId.LONG -> field.getLong(node); + case TypeId.FLOAT -> field.getFloat(node); + case TypeId.DOUBLE -> field.getDouble(node); + case TypeId.OBJECT -> objectValue(field.getObject(node)); + default -> null; + }; + } + + /** Strings as they are, vecmath's Tuple3d and Tuple3f (public x, y, z) as 3 doubles; anything else is not written. */ + private static Object objectValue(Object value) { + if (value instanceof String) return value; + if (value == null) return null; + for (Class type = value.getClass(); type != null; type = type.getSuperclass()) + if (type.getName().equals("javax.vecmath.Tuple3d") || type.getName().equals("javax.vecmath.Tuple3f")) { + try { + double[] vector = new double[3]; + String[] names = {"x", "y", "z"}; + for (int axis = 0; axis < 3; axis++) + vector[axis] = ((Number) type.getField(names[axis]).get(value)).doubleValue(); + return vector; + } catch (ReflectiveOperationException error) { + return null; + } + } + return null; + } + + /** The rows' managed fields as columns (in order of first appearance), the missing rows of each in {@code missing}. */ + private static void addFields(EntitySet set, List rows, Options options, ObjectNode missing) { + Map values = new LinkedHashMap<>(); + for (int row = 0; row < rows.size(); row++) { + Node node = rows.get(row).node; + ManageableType type = node.getNType(); + for (int index = 0; index < type.getManagedFieldCount(); index++) { + ManageableType.Field field = type.getManagedField(index); + if (!isOwnField(field) || set.isReserved(field.getName())) continue; + Object value = fieldValue(field, node); + if (value == null) continue; + values.computeIfAbsent(field.getName(), name -> new Object[rows.size()])[row] = value; + } + } + for (Map.Entry item : values.entrySet()) { + Column column = column(item.getValue()); + if (column == null) continue; // values of several kinds: not written + set.set(item.getKey(), column); + var absent = JsonNodeFactory.instance.arrayNode(); + for (int row = 0; row < item.getValue().length; row++) if (item.getValue()[row] == null) absent.add(row); + if (!absent.isEmpty()) missing.set(item.getKey(), absent); + Variable declared = options.variables.get(item.getKey()); + if (declared != null) set.declare(item.getKey(), declared); + } + } + + /** A column of values of one kind (null where a row has none: NaN, 0, false or ""); null for mixed kinds. */ + private static Column column(Object[] values) { + Class kind = null; + for (Object value : values) + if (value != null) { + if (kind != null && kind != value.getClass()) return null; + kind = value.getClass(); + } + int rows = values.length; + if (kind == Double.class) { + double[] column = new double[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? Double.NaN : (Double) values[row]; + return Column.of(column); + } else if (kind == Float.class) { + float[] column = new float[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? Float.NaN : (Float) values[row]; + return Column.of(column); + } else if (kind == Long.class) { + long[] column = new long[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Long) values[row]; + return Column.of(column); + } else if (kind == Integer.class) { + int[] column = new int[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Integer) values[row]; + return Column.of(column); + } else if (kind == Short.class) { + short[] column = new short[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Short) values[row]; + return new Column(ComponentType.INT16, column); + } else if (kind == Byte.class) { + byte[] column = new byte[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? 0 : (Byte) values[row]; + return new Column(ComponentType.INT8, column); + } else if (kind == Boolean.class) { + boolean[] column = new boolean[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] != null && (Boolean) values[row]; + return Column.of(column); + } else if (kind == String.class) { + String[] column = new String[rows]; + for (int row = 0; row < rows; row++) column[row] = values[row] == null ? "" : (String) values[row]; + return Column.of(column); + } else if (kind == double[].class) { + double[] column = new double[rows * 3]; + for (int row = 0; row < rows; row++) + for (int axis = 0; axis < 3; axis++) + column[row * 3 + axis] = values[row] == null ? Double.NaN : ((double[]) values[row])[axis]; + return Column.vectors(3, column); + } + return null; + } + + /** The other edge bits between exported nodes (also on tree edges): one relation per bit and pair of sets. */ + private static void addRelations(DataStructure dataStructure, List order, Map entities, + Map setNames) { + Map> edges = new LinkedHashMap<>(); + Map ends = new HashMap<>(); + for (Entity entity : order) { + Node node = entity.node; + for (Edge edge = node.getFirstEdge(); edge != null; edge = edge.getNext(node)) { + if (edge.getSource() != node || !entities.containsKey(edge.getTarget())) continue; + int others = edge.getEdgeBits() & ~TREE_EDGES; + for (int bit = 1; bit != 0 && others != 0; bit <<= 1) { + if ((others & bit) == 0) continue; + others &= ~bit; + String kind = RELATION_NAMES.getOrDefault(bit, "edge_" + Integer.toHexString(bit)); + String source = setNames.get(entity.scale), target = setNames.get(entities.get(edge.getTarget()).scale); + String key = kind + "|" + source + "|" + target; + edges.computeIfAbsent(key, k -> new ArrayList<>()).add(new long[] {node.getId(), edge.getTarget().getId()}); + ends.put(key, new String[] {kind, source, target}); + } + } + } + Map kindCount = new HashMap<>(); + for (String[] end : ends.values()) kindCount.merge(end[0], 1, Integer::sum); + for (Map.Entry> item : edges.entrySet()) { + String[] end = ends.get(item.getKey()); + String name = kindCount.get(end[0]) == 1 ? end[0] : end[0] + "_" + end[1] + "_" + end[2]; + Relation relation = new Relation(name, end[1], end[2]); + long[] sources = new long[item.getValue().size()], targets = new long[item.getValue().size()]; + for (int index = 0; index < sources.length; index++) { + sources[index] = item.getValue().get(index)[0]; + targets[index] = item.getValue().get(index)[1]; + } + relation.set("source", Column.of(sources)).set("target", Column.of(targets)); + dataStructure.relations.add(relation); + } + } + + // ── Data structure → graph ────────────────────────────────────────── + + /** + * A graph data structure as GroIMP nodes below a new root: each entity a node made by {@code nodes} from its label + * (e.g. {@code label -> new Internode()}), its fields set from its row, linked by refinement edges from its complex + * (the first of each complex's components only, as GroIMP's multiscale graphs), successor or branch edges from its + * parent, and its relations' edge bits. The root's children: the coarsest scale's entities without parent. Returns + * the root; GroIMP gives the nodes new ids when the root is put in a graph, {@code idsOf} maps each entity's id to + * its node. + */ + public static Node toGraph(DataStructure dataStructure, Function nodes, Map idsOf) { + List sets = new ArrayList<>(); + for (EntitySet set : dataStructure.entitySets) if (set.rank != null && set.has("id")) sets.add(set); + sets.sort((left, right) -> Long.compare(left.rank, right.rank)); + ObjectNode missing = dataStructure.extras.path(PLATFORM).path("missing") instanceof ObjectNode found ? found : null; + Node root = new Node(); + for (EntitySet set : sets) { + long[] ids = set.column("id").longs(); + String[] labels = set.has("label") ? set.column("label").strings() : null; + for (int row = 0; row < ids.length; row++) { + Node node = nodes.apply(labels == null ? set.name : labels[row]); + if (node == null) throw new FspmTfException("no node made for label '" + labels[row] + "'"); + setFields(node, set, row, missing != null && missing.get(set.name) instanceof ObjectNode ofSet ? ofSet : null); + idsOf.put(ids[row], node); + } + } + Set refined = new java.util.HashSet<>(); // complexes already linked to a first component + for (EntitySet set : sets) { + long[] ids = set.column("id").longs(); + long[] parents = set.has("parent") ? set.column("parent").longs() : null; + String[] edgeTypes = set.has("edge_type") ? set.column("edge_type").strings() : null; + long[] complexes = set.has("complex") ? set.column("complex").longs() : null; + for (int row = 0; row < ids.length; row++) { + Node node = idsOf.get(ids[row]); + long parent = parents == null ? EntitySet.NO_ID : parents[row]; + long complex = complexes == null ? EntitySet.NO_ID : complexes[row]; + if (parent != EntitySet.NO_ID) { + idsOf.get(parent).addEdgeBitsTo(node, + "successor".equals(edgeTypes[row]) ? Graph.SUCCESSOR_EDGE : Graph.BRANCH_EDGE, null); + Long parentComplex = complexOf(sets, parent); + if (complex != EntitySet.NO_ID && (parentComplex == null || parentComplex != complex) + && refined.add(complex)) // the first component of its complex + idsOf.get(complex).addEdgeBitsTo(node, Graph.REFINEMENT_EDGE, null); + } else if (complex != EntitySet.NO_ID) { + idsOf.get(complex).addEdgeBitsTo(node, Graph.REFINEMENT_EDGE, null); + refined.add(complex); + } else { + root.addEdgeBitsTo(node, Graph.BRANCH_EDGE, null); + } + } + } + for (Relation relation : dataStructure.relations) { + int bit = 0; + for (Map.Entry item : RELATION_NAMES.entrySet()) + if (relation.name.equals(item.getValue()) || relation.name.startsWith(item.getValue() + "_")) bit = item.getKey(); + if (bit == 0 && relation.name.startsWith("edge_")) { + String hex = relation.name.substring(5).split("_")[0]; + bit = Integer.parseInt(hex, 16); + } + if (bit == 0) continue; // another platform's relation: not an edge bit + long[] sources = relation.column("source").longs(), targets = relation.column("target").longs(); + for (int index = 0; index < sources.length; index++) + idsOf.get(sources[index]).addEdgeBitsTo(idsOf.get(targets[index]), bit, null); + } + return root; + } + + private static Long complexOf(List sets, long id) { + for (EntitySet set : sets) { + if (!set.has("complex")) continue; + long[] ids = set.column("id").longs(); + for (int row = 0; row < ids.length; row++) if (ids[row] == id) return set.column("complex").longs()[row]; + } + return null; + } + + private static void setFields(Node node, EntitySet set, int row, ObjectNode missing) { + ManageableType type = node.getNType(); + for (String name : set.variableNames()) { + ManageableType.Field field = type.getManagedField(name); + if (field == null || !isOwnField(field)) continue; + if (missing != null && missing.has(name)) { + boolean absent = false; + for (var absentRow : missing.get(name)) absent |= absentRow.asInt() == row; + if (absent) continue; + } + Column column = set.column(name); + switch (field.getType().getTypeId()) { + case TypeId.BOOLEAN -> field.setBoolean(node, column.booleans()[row]); + case TypeId.BYTE -> field.setByte(node, (byte) column.doubleAt(row)); + case TypeId.SHORT -> field.setShort(node, (short) column.doubleAt(row)); + case TypeId.CHAR -> field.setChar(node, column.strings()[row].charAt(0)); + case TypeId.INT -> field.setInt(node, (int) column.doubleAt(row)); + case TypeId.LONG -> field.setLong(node, (long) column.doubleAt(row)); + case TypeId.FLOAT -> field.setFloat(node, (float) column.doubleAt(row)); + case TypeId.DOUBLE -> field.setDouble(node, column.doubleAt(row)); + case TypeId.OBJECT -> { + if (column.type == ComponentType.STRING) field.setObject(node, column.strings()[row]); + } + default -> { } + } + } + } +} diff --git a/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java new file mode 100644 index 0000000..691c584 --- /dev/null +++ b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java @@ -0,0 +1,123 @@ +package org.openalea.fspmtf.groimp; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.openalea.fspmtf.Codec; +import org.openalea.fspmtf.DataStructure; +import org.openalea.fspmtf.EntitySet; +import org.openalea.fspmtf.Frame; +import org.openalea.fspmtf.Variable; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** GroIMP's adapter: a multiscale GroIMP graph (plant, internodes and leaves) exported, written, read and rebuilt. */ +class GroImpTest { + static final Path OUT = Path.of(System.getProperty("fspmtf.out", "target/groimp_out")); + + /** root -> Plant /-> Internode < Internode, each internode bearing a leaf (+); a mark edge from a leaf to the plant. */ + static Node plant(GraphManager graph) { + Node root = new Node(); + Organs.Plant plant = new Organs.Plant(); + plant.species = "Wheat"; + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Organs.Internode first = new Organs.Internode(), second = new Organs.Internode(); + first.length = 0.12; first.radius = 0.004f; first.rank = 1; + second.length = 0.08; second.radius = 0.003f; second.rank = 2; + Organs.Leaf firstLeaf = new Organs.Leaf(), secondLeaf = new Organs.Leaf(); + firstLeaf.length = 0.2; secondLeaf.length = 0.15; secondLeaf.alive = false; + plant.addEdgeBitsTo(first, Graph.REFINEMENT_EDGE, null); + first.addEdgeBitsTo(second, Graph.SUCCESSOR_EDGE, null); + first.addEdgeBitsTo(firstLeaf, Graph.BRANCH_EDGE, null); + second.addEdgeBitsTo(secondLeaf, Graph.BRANCH_EDGE, null); + firstLeaf.addEdgeBitsTo(plant, Graph.MARK_EDGE, null); + graph.setRoot(Graph.MAIN_GRAPH, root); + return root; + } + + static GraphManager newGraph() { + return new GraphManager(new ServerConnection(null), "test", false, false); + } + + @Test + void exportedWrittenAndRebuilt() throws IOException { + Node root = plant(newGraph()); + GroImp.Options options = new GroImp.Options(); + options.scaleNames.put(2, "Organ"); + options.scaleRanks.put(2, 5L); // the vocabulary's Organ + options.variables.put("length", new Variable("m", "length", "extensive", "state")); + DataStructure written = GroImp.toDataStructure(root, options); + + assertEquals(List.of("Plant", "Organ"), written.entitySets.stream().map(set -> set.name).toList()); + EntitySet plants = written.entitySet("Plant"), organs = written.entitySet("Organ"); + assertArrayEquals(new String[] {"Wheat"}, plants.column("species").strings()); + assertEquals(5L, organs.rank); + assertArrayEquals(new String[] {"Internode", "Internode", "Leaf", "Leaf"}, organs.column("label").strings()); + long[] ids = organs.column("id").longs(); + long plantId = plants.column("id").longs()[0]; + assertArrayEquals(new long[] {plantId, plantId, plantId, plantId}, organs.column("complex").longs()); + assertArrayEquals(new long[] {-1, ids[0], ids[1], ids[0]}, organs.column("parent").longs()); + assertArrayEquals(new String[] {"none", "successor", "branch", "branch"}, organs.column("edge_type").strings()); + double[] lengths = organs.column("length").doubles(); + assertArrayEquals(new double[] {0.12, 0.08, 0.15, 0.2}, lengths); + assertTrue(Float.isNaN(organs.column("radius").floats()[2])); // leaves have no radius + assertArrayEquals(new boolean[] {false, false, false, true}, organs.column("alive").booleans()); + assertEquals("mark", written.relations.get(0).name); + assertArrayEquals(new long[] {ids[3]}, written.relations.get(0).column("source").longs()); + assertEquals(List.of(0, 1), List.of(written.extras.get("groimp").get("missing").get("Organ").get("alive").get(0).asInt(), + written.extras.get("groimp").get("missing").get("Organ").get("alive").get(1).asInt())); + + Files.createDirectories(OUT); + Frame frame = new Frame(); + frame.dataStructures.add(written); + Codec.write(OUT.resolve("groimp_plant.glb"), frame); + DataStructure readBack = Codec.read(OUT.resolve("groimp_plant.glb")).dataStructures.get(0); + + Map nodes = new HashMap<>(); + Map> classes = Map.of("Plant", Organs.Plant::new, + "Internode", Organs.Internode::new, "Leaf", Organs.Leaf::new); + Node rebuilt = GroImp.toGraph(readBack, label -> classes.get(label).get(), nodes); + newGraph().setRoot(Graph.MAIN_GRAPH, rebuilt); + DataStructure again = GroImp.toDataStructure(rebuilt, options); + EntitySet againOrgans = again.entitySet("Organ"); + assertArrayEquals(organs.column("label").strings(), againOrgans.column("label").strings()); + assertArrayEquals(lengths, againOrgans.column("length").doubles()); + assertArrayEquals(organs.column("edge_type").strings(), againOrgans.column("edge_type").strings()); + assertArrayEquals(organs.column("alive").booleans(), againOrgans.column("alive").booleans()); + assertEquals(0.004f, ((Organs.Internode) nodes.get(ids[0])).radius); + assertEquals("Wheat", ((Organs.Plant) nodes.get(plantId)).species); + assertEquals(1, again.relations.size()); + assertNull(again.entitySet("Plant").has("complex") ? "complex" : null); + } + + @Test + void anotherPlatformsGraph() throws IOException { + // The conformance root system (metafspm: Plant, Axis, SubOrgan) as GroIMP nodes: three scales by refinement + Path conformance = Path.of(System.getProperty("fspmtf.conformance", "../../../conformance")); + DataStructure roots = Codec.read(conformance.resolve("root_system.glb")).dataStructures.get(0); + Node root = GroImp.toGraph(roots, label -> new Node(), new HashMap<>()); + newGraph().setRoot(Graph.MAIN_GRAPH, root); + DataStructure again = GroImp.toDataStructure(root, new GroImp.Options()); + assertEquals(3, again.entitySets.size()); + for (int scale = 0; scale < 3; scale++) { + EntitySet written = roots.entitySets.get(scale), read = again.entitySets.get(scale); + assertEquals(written.rowCount(), read.rowCount(), written.name); + if (written.has("edge_type")) + assertEquals(List.of(written.column("edge_type").strings()).stream().sorted().toList(), + List.of(read.column("edge_type").strings()).stream().sorted().toList(), written.name); + } + } +} diff --git a/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/Organs.java b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/Organs.java new file mode 100644 index 0000000..334b729 --- /dev/null +++ b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/Organs.java @@ -0,0 +1,168 @@ +package org.openalea.fspmtf.groimp; + +import de.grogra.graph.impl.Node; + +/** Node classes as GroIMP models declare them (managed fields, as the XL compiler generates them): a plant, internodes + * and leaves. */ +final class Organs { + private Organs() {} + + static final class Plant extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + String species = ""; + + static { + $TYPE = new NType(Plant.class); + $TYPE.addManagedField(new Field("species", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.STRING, null, 0)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Plant.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + protected void setObjectImpl(Object o, Object value) { + if (id == 0) ((Plant) o).species = (String) value; else super.setObjectImpl(o, value); + } + + @Override + public Object getObject(Object o) { + return id == 0 ? ((Plant) o).species : super.getObject(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Plant(); + } + } + + static final class Internode extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + double length; + float radius; + int rank; + + static { + $TYPE = new NType(Internode.class); + $TYPE.addManagedField(new Field("length", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.DOUBLE, null, 0)); + $TYPE.addManagedField(new Field("radius", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.FLOAT, null, 1)); + $TYPE.addManagedField(new Field("rank", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.INT, null, 2)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Internode.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + public void setDouble(Object o, double value) { + if (id == 0) ((Internode) o).length = value; else super.setDouble(o, value); + } + + @Override + public double getDouble(Object o) { + return id == 0 ? ((Internode) o).length : super.getDouble(o); + } + + @Override + public void setFloat(Object o, float value) { + if (id == 1) ((Internode) o).radius = value; else super.setFloat(o, value); + } + + @Override + public float getFloat(Object o) { + return id == 1 ? ((Internode) o).radius : super.getFloat(o); + } + + @Override + public void setInt(Object o, int value) { + if (id == 2) ((Internode) o).rank = value; else super.setInt(o, value); + } + + @Override + public int getInt(Object o) { + return id == 2 ? ((Internode) o).rank : super.getInt(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Internode(); + } + } + + static final class Leaf extends Node { + private static final long serialVersionUID = 1L; + public static final NType $TYPE; + double length; + boolean alive = true; + + static { + $TYPE = new NType(Leaf.class); + $TYPE.addManagedField(new Field("length", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.DOUBLE, null, 0)); + $TYPE.addManagedField(new Field("alive", Field.PUBLIC | Field.SCO, de.grogra.reflect.Type.BOOLEAN, null, 1)); + $TYPE.validate(); + } + + private static final class Field extends NType.Field { + private final int id; + + Field(String name, int modifiers, de.grogra.reflect.Type type, de.grogra.reflect.Type componentType, int id) { + super(Leaf.$TYPE, name, modifiers, type, componentType); + this.id = id; + } + + @Override + public void setDouble(Object o, double value) { + if (id == 0) ((Leaf) o).length = value; else super.setDouble(o, value); + } + + @Override + public double getDouble(Object o) { + return id == 0 ? ((Leaf) o).length : super.getDouble(o); + } + + @Override + public void setBoolean(Object o, boolean value) { + if (id == 1) ((Leaf) o).alive = value; else super.setBoolean(o, value); + } + + @Override + public boolean getBoolean(Object o) { + return id == 1 ? ((Leaf) o).alive : super.getBoolean(o); + } + } + + @Override + protected NType getNTypeImpl() { + return $TYPE; + } + + @Override + protected Node newInstance() { + return new Leaf(); + } + } +} diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml new file mode 100644 index 0000000..8df2552 --- /dev/null +++ b/bindings/java/pom.xml @@ -0,0 +1,79 @@ + + + + 4.0.0 + + org.openalea + fspm-tf + 0.1.0 + jar + fspm-TF (Java) + Reading and writing fspm-TF files: functional-structural plant model scenes as glTF 2.0 binaries + https://github.com/openalea/fspm-TF + + + 17 + UTF-8 + 2.18.2 + 5.11.4 + ${project.basedir}/../../conformance + + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + -Xlint:all + + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.2 + + + ${fspmtf.conformance} + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + org.openalea.fspmtf.ConformanceRun + + + org.openalea.fspmtf + + + + + + + diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java new file mode 100644 index 0000000..ce93a4f --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java @@ -0,0 +1,794 @@ +package org.openalea.fspmtf; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * Reading and writing frames as GLB files. Tables are EXT_structural_metadata property tables (one class per table), + * meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data + * structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, + * standard names, kinds and roles. + */ +public final class Codec { + /** The version of the specification this library reads and writes. */ + public static final String SPECIFICATION_VERSION = "0.1"; + public static final List EDGE_TYPES = List.of("none", "successor", "branch"); + + private static final String EXTENSION = "FSPM_scene", METADATA = "EXT_structural_metadata", + FEATURES = "EXT_mesh_features", EDGE_TYPE_ENUM = "FSPM_edge_type"; + private static final List ENUM_COLUMNS = List.of("label", "edge_type", "relation"); + private static final List ID_COLUMNS = List.of("id", "complex", "parent", "source", "target", "level", "index"); + private static final int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; + private static final int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; + private static final long MAX_FLOAT_FEATURE = 1L << 24; // feature rows are float vertex attributes + private static final Pattern IDENTIFIER = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); + // The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about + // x, column-major). Readers of the tables use the scene's frame, not this node. + private static final double[] Z_UP_TO_Y_UP = {1, 0, 0, 0, 0, 0, -1, 0, 0, 1, 0, 0, 0, 0, 0, 1}; + + private Codec() {} + + // ── Writing ────────────────────────────────────────────────────────── + + /** The GLB file of a frame. */ + public static byte[] toBytes(Frame frame) { + Glb.BufferBuilder buffers = new Glb.BufferBuilder(); + ObjectNode schema = Glb.MAPPER.createObjectNode().put("id", "fspm_scene"); + schema.putObject("classes"); + ObjectNode edgeEnum = schema.putObject("enums").putObject(EDGE_TYPE_ENUM).put("valueType", "UINT8"); + ArrayNode edgeValues = edgeEnum.putArray("values"); + for (int value = 0; value < EDGE_TYPES.size(); value++) + edgeValues.addObject().put("name", EDGE_TYPES.get(value)).put("value", value); + ArrayNode tables = Glb.MAPPER.createArrayNode(); + ObjectNode variables = Glb.MAPPER.createObjectNode(); + ObjectNode document = Glb.MAPPER.createObjectNode(); + document.putObject("asset").put("version", "2.0").put("generator", "fspm-tf (Java) " + SPECIFICATION_VERSION); + document.put("scene", 0); + document.putArray("scenes").addObject().putArray("nodes").add(0); + ArrayNode nodes = document.putArray("nodes"); + ObjectNode sceneNode = nodes.addObject().put("name", "fspm_scene"); + putNumbers(sceneNode.putArray("matrix"), Z_UP_TO_Y_UP); + sceneNode.putArray("children"); + ArrayNode accessors = Glb.MAPPER.createArrayNode(), meshes = Glb.MAPPER.createArrayNode(); + + ArrayNode dataStructures = Glb.MAPPER.createArrayNode(); + for (DataStructure dataStructure : frame.dataStructures) { + checkIdentifier(dataStructure.id, "data structure id"); + ObjectNode entry = dataStructures.addObject().put("id", dataStructure.id).put("kind", dataStructure.kind); + if (dataStructure.platform != null) entry.put("platform", dataStructure.platform); + if (dataStructure.model != null) entry.put("model", dataStructure.model); + double[] columnMajor = columnMajor(dataStructure.origin); + putNumbers(entry.putArray("origin"), columnMajor); // column-major, as glTF matrices + ArrayNode sets = entry.putArray("entitySets"); + for (EntitySet set : dataStructure.entitySets) { + ObjectNode item = sets.addObject().put("name", set.name); + if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + item.put("propertyTable", addTable(set, dataStructure.id + "__" + set.name, buffers, schema, tables, variables)); + } + if (!dataStructure.relations.isEmpty()) { + ArrayNode relations = entry.putArray("relations"); + for (Relation relation : dataStructure.relations) + relations.addObject().put("name", relation.name).put("sourceSet", relation.sourceSet) + .put("targetSet", relation.targetSet).put("propertyTable", + addTable(relation, dataStructure.id + "__" + relation.name, buffers, schema, tables, variables)); + } + if (dataStructure.scalars != null) + entry.put("scalars", addTable(dataStructure.scalars, dataStructure.id + "__scalars", buffers, schema, tables, variables)); + if (dataStructure.grid != null) entry.set("grid", gridJson(dataStructure.grid)); + if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); + ObjectNode node = nodes.addObject().put("name", dataStructure.id); + if (!isIdentity(dataStructure.origin)) putNumbers(node.putArray("matrix"), columnMajor); + int nodeIndex = nodes.size() - 1; + ((ArrayNode) sceneNode.get("children")).add(nodeIndex); + entry.put("node", nodeIndex); + if (!dataStructure.meshes.isEmpty()) { + ArrayNode children = node.putArray("children"); + ArrayNode meshEntries = entry.putArray("meshes"); + for (Mesh mesh : dataStructure.meshes) { + meshes.add(encodeMesh(mesh, dataStructure, entry, buffers, accessors)); + nodes.addObject().put("name", mesh.name).put("mesh", meshes.size() - 1); + children.add(nodes.size() - 1); + ObjectNode meshEntry = meshEntries.addObject().put("name", mesh.name).put("mesh", meshes.size() - 1); + if (!mesh.display.isEmpty()) { + ArrayNode display = meshEntry.putArray("display"); + for (String variable : mesh.display.keySet()) { + ObjectNode item = display.addObject().put("attribute", displayAttribute(variable)) + .put("variable", variable); + String source = mesh.displaySources.get(variable); + if (source == null) item.putNull("entitySet"); else item.put("entitySet", source); + } + } + } + } + } + + ArrayNode mappings = Glb.MAPPER.createArrayNode(); + for (Mapping mapping : frame.mappings) { + checkIdentifier(mapping.name, "mapping name"); + ObjectNode entry = mappings.addObject().put("name", mapping.name); + entry.putObject("source").put("dataStructure", mapping.sourceDataStructure).put("entitySet", mapping.sourceSet); + entry.putObject("target").put("dataStructure", mapping.targetDataStructure).put("entitySet", mapping.targetSet); + if (mapping.method != null) entry.put("method", mapping.method); + entry.put("propertyTable", addTable(mapping, "mapping__" + mapping.name, buffers, schema, tables, variables)); + } + + ObjectNode scene = Glb.MAPPER.createObjectNode().put("version", SPECIFICATION_VERSION); + scene.putObject("frame").put("time", frame.time).put("timeUnit", frame.timeUnit).put("step", frame.step); + if (frame.vocabularyVersion != null) scene.put("vocabulary", frame.vocabularyVersion); + if (!frame.writer.isEmpty()) scene.set("writer", frame.writer); + scene.set("dataStructures", dataStructures); + if (!mappings.isEmpty()) scene.set("mappings", mappings); + if (!variables.isEmpty()) scene.set("variables", variables); + + ObjectNode extensions = Glb.MAPPER.createObjectNode(); + ObjectNode metadata = extensions.putObject(METADATA); + metadata.set("schema", schema); + metadata.set("propertyTables", tables); + extensions.set(EXTENSION, scene); + ArrayNode used = document.putArray("extensionsUsed").add(METADATA).add(EXTENSION); + if (!meshes.isEmpty()) used.add(FEATURES); + JsonNode others = frame.unknown.get("extensions"); + if (others != null) + for (Iterator> items = others.fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + extensions.set(item.getKey(), item.getValue()); + used.add(item.getKey()); + } + document.set("extensions", extensions); + if (frame.unknown.has("extras")) document.set("extras", frame.unknown.get("extras")); + if (!meshes.isEmpty()) { + document.set("meshes", meshes); + document.set("accessors", accessors); + } + byte[] binary = buffers.data.toByteArray(); + if (!buffers.views.isEmpty()) { + // the buffer holds the chunk's padding to 8 bytes: glTF-Validator warns of more than 3 bytes beyond it + binary = java.util.Arrays.copyOf(binary, binary.length + Glb.padding(binary.length)); + document.putArray("buffers").addObject().put("byteLength", binary.length); + document.set("bufferViews", buffers.views); + } + return Glb.pack(document, binary); + } + + /** Writes a frame to the GLB file {@code path}. */ + public static void write(Path path, Frame frame) throws IOException { + Files.write(path, toBytes(frame)); + } + + static void checkIdentifier(String name, String what) { + if (name == null || !IDENTIFIER.matcher(name).matches()) + throw new FspmTfException(what + " '" + name + "' must be an identifier (letters, digits, '_'; not starting with a digit)"); + } + + private static void putNumbers(ArrayNode array, double[] values) { + for (double value : values) array.add(value); + } + + private static double[] columnMajor(double[] rowMajor) { + double[] values = new double[16]; + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) values[column * 4 + row] = rowMajor[row * 4 + column]; + return values; + } + + private static boolean isIdentity(double[] origin) { + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) + if (origin[row * 4 + column] != (row == column ? 1.0 : 0.0)) return false; + return true; + } + + private static ObjectNode declarationJson(Variable variable) { + ObjectNode declaration = Glb.MAPPER.createObjectNode().put("unit", variable.unit).put("role", variable.role); + if (variable.standardName != null) declaration.put("standardName", variable.standardName); + if (variable.kind != null) declaration.put("kind", variable.kind); + if (variable.minimum != null) declaration.put("minimum", variable.minimum); + if (variable.maximum != null) declaration.put("maximum", variable.maximum); + if (!variable.references.isEmpty()) variable.references.forEach(declaration.putArray("references")::add); + if (!variable.doi.isEmpty()) variable.doi.forEach(declaration.putArray("doi")::add); + return declaration; + } + + private static int addTable(Table table, String classId, Glb.BufferBuilder buffers, ObjectNode schema, ArrayNode tables, + ObjectNode variables) { + checkIdentifier(classId, "class"); + table.checkLengths(); + ObjectNode properties = Glb.MAPPER.createObjectNode(), stored = Glb.MAPPER.createObjectNode(); + for (Map.Entry item : table.columns.entrySet()) + encodeColumn(table, classId, item.getKey(), item.getValue(), buffers, schema, properties, stored); + ((ObjectNode) schema.get("classes")).putObject(classId).put("name", table.name).set("properties", properties); + ObjectNode description = tables.addObject().put("name", table.name).put("class", classId).put("count", table.rowCount()); + description.set("properties", stored); + ObjectNode declared = Glb.MAPPER.createObjectNode(); + for (String name : table.variableNames()) + if (table.variables.containsKey(name)) declared.set(name, declarationJson(table.variables.get(name))); + if (!declared.isEmpty()) variables.set(classId, declared); + return tables.size() - 1; + } + + private static void encodeColumn(Table table, String classId, String name, Column column, Glb.BufferBuilder buffers, + ObjectNode schema, ObjectNode properties, ObjectNode stored) { + checkIdentifier(name, "column of '" + table.name + "'"); + String where = "column '" + name + "' of '" + table.name + "'"; + ObjectNode property = properties.putObject(name); + Variable declared = table.variables.get(name); + if (declared != null && declared.description != null) property.put("description", declared.description); + boolean reserved = table.isReserved(name); + if (reserved && ENUM_COLUMNS.contains(name)) { + if (column.type != ComponentType.STRING) throw new FspmTfException(where + ": labels are text"); + property.put("type", "ENUM"); + property.put("enumType", encodeEnum(name, column.strings(), classId, schema, buffers, stored)); + } else if (column.type == ComponentType.STRING) { + if (column.width != 1) throw new FspmTfException(where + ": text vectors are not supported"); + property.put("type", "STRING"); + String[] values = column.strings(); + byte[][] encoded = new byte[values.length][]; + long[] offsets = new long[values.length + 1]; + for (int row = 0; row < values.length; row++) { + encoded[row] = values[row].getBytes(StandardCharsets.UTF_8); + offsets[row + 1] = offsets[row] + encoded[row].length; + } + ByteBuffer text = ByteBuffer.allocate((int) offsets[values.length]); + for (byte[] value : encoded) text.put(value); + ObjectNode item = stored.putObject(name).put("values", buffers.add(text.array())); + boolean narrow = offsets[values.length] < (1L << 32); + ByteBuffer offsetBytes = little(offsets.length * (narrow ? 4 : 8)); + for (long offset : offsets) if (narrow) offsetBytes.putInt((int) offset); else offsetBytes.putLong(offset); + item.put("stringOffsets", buffers.add(offsetBytes.array())).put("stringOffsetType", narrow ? "UINT32" : "UINT64"); + } else if (column.type == ComponentType.BOOLEAN) { + if (column.width != 1) throw new FspmTfException(where + ": boolean vectors are not supported"); + property.put("type", "BOOLEAN"); + boolean[] values = column.booleans(); + byte[] packed = new byte[(values.length + 7) / 8]; + for (int row = 0; row < values.length; row++) + if (values[row]) packed[row / 8] |= (byte) (1 << (row % 8)); // bit order: little + stored.putObject(name).put("values", buffers.add(packed)); + } else { + ComponentType type = column.type; + Column data = column; + if (reserved && ID_COLUMNS.contains(name) && !type.isFloat() && type != ComponentType.INT64) { + long[] ids = new long[column.size()]; // ids are INT64, whatever the platform's + for (int index = 0; index < ids.length; index++) ids[index] = (long) column.doubleAt(index); + data = new Column(ComponentType.INT64, ids, column.width); + type = ComponentType.INT64; + } + property.put("type", "SCALAR").put("componentType", type.name()); + if (column.width != 1) { + if (column.width < 2) throw new FspmTfException(where + ": vectors have at least 2 components"); + property.put("array", true).put("count", column.width); + } + if (type == ComponentType.FLOAT64) data = standIns(data.doubles().clone(), property, where); + else if (type == ComponentType.FLOAT32) data = standIns(data.floats().clone(), property, where); + else if (reserved && (name.equals("complex") || name.equals("parent"))) property.put("noData", EntitySet.NO_ID); + if (data.width != column.width) data = new Column(data.type, data.values, column.width); + stored.putObject(name).put("values", buffers.add(bytesOf(data))); + } + if (name.equals("id") && table.kind() == Table.Kind.ENTITY_SET) property.put("required", true); + } + + // EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; + // +inf and -inf as the largest finite value and the next one after the most negative, declared in the property's + // extras (FSPM_infinity). + private static Column standIns(double[] values, ObjectNode property, String where) { + double positive = Double.MAX_VALUE, negative = Math.nextUp(-Double.MAX_VALUE), noData = -Double.MAX_VALUE; + boolean infinite = false, nan = false; + for (double value : values) { + infinite |= Double.isInfinite(value); + nan |= Double.isNaN(value); + } + if (infinite) { + for (double value : values) + if (value == positive || value == negative) + throw new FspmTfException(where + " holds both infinities and the values standing for them"); + property.putObject("extras").putObject("FSPM_infinity").put("positive", positive).put("negative", negative); + for (int index = 0; index < values.length; index++) + if (Double.isInfinite(values[index])) values[index] = values[index] > 0 ? positive : negative; + } + if (nan) { + for (double value : values) + if (value == noData) throw new FspmTfException(where + " holds both NaN and the value standing for NaN"); + property.put("noData", noData); + for (int index = 0; index < values.length; index++) if (Double.isNaN(values[index])) values[index] = noData; + } + return new Column(ComponentType.FLOAT64, values); + } + + private static Column standIns(float[] values, ObjectNode property, String where) { + float positive = Float.MAX_VALUE, negative = Math.nextUp(-Float.MAX_VALUE), noData = -Float.MAX_VALUE; + boolean infinite = false, nan = false; + for (float value : values) { + infinite |= Float.isInfinite(value); + nan |= Float.isNaN(value); + } + if (infinite) { + for (float value : values) + if (value == positive || value == negative) + throw new FspmTfException(where + " holds both infinities and the values standing for them"); + property.putObject("extras").putObject("FSPM_infinity").put("positive", (double) positive) + .put("negative", (double) negative); + for (int index = 0; index < values.length; index++) + if (Float.isInfinite(values[index])) values[index] = values[index] > 0 ? positive : negative; + } + if (nan) { + for (float value : values) + if (value == noData) throw new FspmTfException(where + " holds both NaN and the value standing for NaN"); + property.put("noData", (double) noData); + for (int index = 0; index < values.length; index++) if (Float.isNaN(values[index])) values[index] = noData; + } + return new Column(ComponentType.FLOAT32, values); + } + + private static String encodeEnum(String name, String[] values, String classId, ObjectNode schema, + Glb.BufferBuilder buffers, ObjectNode stored) { + if (name.equals("edge_type")) { + byte[] codes = new byte[values.length]; + for (int row = 0; row < values.length; row++) { + int code = EDGE_TYPES.indexOf(values[row]); + if (code < 0) + throw new FspmTfException("edge_type of '" + classId + "': unknown value '" + values[row] + + "' (allowed: none, successor, branch)"); + codes[row] = (byte) code; + } + stored.putObject(name).put("values", buffers.add(codes)); + return EDGE_TYPE_ENUM; + } + List names = new ArrayList<>(new TreeSet<>(List.of(values))); + String enumName = classId + "__" + name; + boolean narrow = names.size() <= 256; + ObjectNode enumeration = ((ObjectNode) schema.get("enums")).putObject(enumName).put("valueType", narrow ? "UINT8" : "UINT16"); + ArrayNode enumValues = enumeration.putArray("values"); + Map codeOf = new HashMap<>(); + for (int code = 0; code < names.size(); code++) { + enumValues.addObject().put("name", names.get(code)).put("value", code); + codeOf.put(names.get(code), code); + } + ByteBuffer codes = little(values.length * (narrow ? 1 : 2)); + for (String value : values) if (narrow) codes.put((byte) (int) codeOf.get(value)); else codes.putShort((short) (int) codeOf.get(value)); + stored.putObject(name).put("values", buffers.add(codes.array())); + return enumName; + } + + private static ByteBuffer little(int size) { + return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + } + + /** A numeric column's values, little-endian. */ + static byte[] bytesOf(Column column) { + ByteBuffer bytes = little(column.size() * column.type.size); + switch (column.type) { + case INT8, UINT8 -> bytes.put((byte[]) column.values); + case INT16, UINT16 -> { for (short value : (short[]) column.values) bytes.putShort(value); } + case INT32, UINT32 -> { for (int value : (int[]) column.values) bytes.putInt(value); } + case INT64, UINT64 -> { for (long value : (long[]) column.values) bytes.putLong(value); } + case FLOAT32 -> { for (float value : (float[]) column.values) bytes.putFloat(value); } + case FLOAT64 -> { for (double value : (double[]) column.values) bytes.putDouble(value); } + default -> throw new FspmTfException("a " + column.type + " column has no numbers"); + } + return bytes.array(); + } + + private static ObjectNode gridJson(Grid grid) { + ObjectNode entry = Glb.MAPPER.createObjectNode().put("kind", grid.kind); + ArrayNode shape = entry.putArray("shape"); + for (long size : grid.shape) shape.add(size); + putNumbers(entry.putArray("origin"), grid.origin); + putNumbers(entry.putArray("spacing"), grid.spacing); + grid.axes.forEach(entry.putArray("axes")::add); + ArrayNode periodic = entry.putArray("periodic"); + for (boolean value : grid.periodic) periodic.add(value); + entry.put("zUp", grid.zUp).put("cells", grid.cells); + if (grid.kind.equals("adaptive")) entry.put("maxLevel", grid.maxLevel); + return entry; + } + + private static String displayAttribute(String name) { + return "_" + name.toUpperCase(java.util.Locale.ROOT); + } + + private static ObjectNode encodeMesh(Mesh mesh, DataStructure dataStructure, ObjectNode entry, + Glb.BufferBuilder buffers, ArrayNode accessors) { + int vertices = mesh.vertexCount(); + if (mesh.positions.length != vertices * 3) throw new FspmTfException("mesh '" + mesh.name + "': 3 coordinates per vertex"); + float[] positions = new float[mesh.positions.length]; + for (int index = 0; index < positions.length; index++) positions[index] = (float) mesh.positions[index]; + ObjectNode attributes = Glb.MAPPER.createObjectNode(); + ObjectNode position = accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, positions)), "VEC3", + FLOAT, ARRAY_BUFFER, vertices, attributes, "POSITION"); + if (vertices > 0) { + double[] minimum = {Double.MAX_VALUE, Double.MAX_VALUE, Double.MAX_VALUE}, maximum = {-Double.MAX_VALUE, -Double.MAX_VALUE, -Double.MAX_VALUE}; + for (int vertex = 0; vertex < vertices; vertex++) + for (int axis = 0; axis < 3; axis++) { + minimum[axis] = Math.min(minimum[axis], positions[vertex * 3 + axis]); + maximum[axis] = Math.max(maximum[axis], positions[vertex * 3 + axis]); + } + putNumbers(position.putArray("min"), minimum); + putNumbers(position.putArray("max"), maximum); + } + Map tables = new HashMap<>(); + for (JsonNode set : entry.get("entitySets")) tables.put(set.get("name").asText(), set.get("propertyTable").asInt()); + ArrayNode featureIds = Glb.MAPPER.createArrayNode(); + int setIndex = 0; + for (Map.Entry item : mesh.featureRows.entrySet()) { + String setName = item.getKey(); + long[] rows = item.getValue(); + if (!tables.containsKey(setName)) + throw new FspmTfException("mesh '" + mesh.name + "': no entity set '" + setName + "' in '" + dataStructure.id + "'"); + if (rows.length != vertices) + throw new FspmTfException("mesh '" + mesh.name + "': feature rows of '" + setName + "' are not one per vertex"); + long count = dataStructure.entitySet(setName).rowCount(); + if (count >= MAX_FLOAT_FEATURE) + throw new FspmTfException("mesh '" + mesh.name + "': too many entities in '" + setName + "', feature IDs stop at 2^24"); + float[] storedRows = new float[rows.length]; + TreeSet present = new TreeSet<>(); + boolean hasNull = false; + for (int vertex = 0; vertex < rows.length; vertex++) { + long row = rows[vertex]; + if (row < EntitySet.NO_ID || row >= count) + throw new FspmTfException("mesh '" + mesh.name + "': feature rows of '" + setName + "' outside its rows"); + hasNull |= row == EntitySet.NO_ID; + if (row != EntitySet.NO_ID) present.add(row); + storedRows[vertex] = row == EntitySet.NO_ID ? count : row; // null: one past the last row + } + accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, storedRows)), "SCALAR", FLOAT, ARRAY_BUFFER, + vertices, attributes, "_FEATURE_ID_" + setIndex); + ObjectNode feature = featureIds.addObject().put("featureCount", present.size()).put("attribute", setIndex) + .put("propertyTable", tables.get(setName)).put("label", setName); + if (hasNull) feature.put("nullFeatureId", count); + setIndex++; + } + for (Map.Entry item : mesh.display.entrySet()) { + if (item.getValue().length != vertices) + throw new FspmTfException("mesh '" + mesh.name + "': display copy '" + item.getKey() + "' is not one value per vertex"); + accessor(accessors, buffers, bytesOf(new Column(ComponentType.FLOAT32, item.getValue())), "SCALAR", FLOAT, + ARRAY_BUFFER, vertices, attributes, displayAttribute(item.getKey())); + } + ObjectNode primitive = Glb.MAPPER.createObjectNode(); + primitive.set("attributes", attributes); + primitive.put("mode", 4); + if (mesh.indices != null) { + int[] indices = new int[mesh.indices.length]; + for (int index = 0; index < indices.length; index++) indices[index] = (int) mesh.indices[index]; + ObjectNode holder = Glb.MAPPER.createObjectNode(); + accessor(accessors, buffers, bytesOf(new Column(ComponentType.UINT32, indices)), "SCALAR", UNSIGNED_INT, + ELEMENT_ARRAY_BUFFER, indices.length, holder, "indices"); + primitive.set("indices", holder.get("indices")); + } + if (!featureIds.isEmpty()) primitive.putObject("extensions").putObject(FEATURES).set("featureIds", featureIds); + ObjectNode encoded = Glb.MAPPER.createObjectNode().put("name", mesh.name); + encoded.putArray("primitives").add(primitive); + return encoded; + } + + private static ObjectNode accessor(ArrayNode accessors, Glb.BufferBuilder buffers, byte[] bytes, String type, + int componentType, int target, int count, ObjectNode attributes, String attribute) { + ObjectNode item = accessors.addObject().put("bufferView", buffers.add(bytes, target)) + .put("componentType", componentType).put("count", count).put("type", type); + attributes.put(attribute, accessors.size() - 1); + return item; + } + + // ── Reading ────────────────────────────────────────────────────────── + + /** The frame of a GLB file's bytes. */ + public static Frame fromBytes(byte[] data) { + Glb.Contents contents = Glb.unpack(data); + JsonNode document = contents.document(); + byte[] binary = contents.binary(); + JsonNode extensions = document.path("extensions"); + if (!extensions.has(EXTENSION)) throw new FspmTfException("not an fspm-TF file: no " + EXTENSION + " extension"); + JsonNode scene = extensions.get(EXTENSION); + JsonNode metadata = extensions.path(METADATA); + JsonNode variables = scene.path("variables"); + + Frame frame = new Frame(); + JsonNode frameEntry = scene.path("frame"); + frame.time = frameEntry.path("time").asDouble(0.0); + frame.timeUnit = frameEntry.path("timeUnit").asText("d"); + frame.step = frameEntry.path("step").asLong(0); + if (scene.get("writer") instanceof ObjectNode writer) frame.writer = writer; + if (scene.hasNonNull("vocabulary")) frame.vocabularyVersion = scene.get("vocabulary").asText(); + for (JsonNode entry : scene.path("dataStructures")) { + DataStructure dataStructure = new DataStructure(entry.get("id").asText(), entry.path("kind").asText("graph")); + if (entry.hasNonNull("platform")) dataStructure.platform = entry.get("platform").asText(); + if (entry.hasNonNull("model")) dataStructure.model = entry.get("model").asText(); + if (entry.get("extras") instanceof ObjectNode extras) dataStructure.extras = extras; + if (entry.has("origin")) { + JsonNode origin = entry.get("origin"); + if (origin.size() != 16) throw new FspmTfException("data structure '" + dataStructure.id + "': an origin of 16 numbers"); + for (int row = 0; row < 4; row++) + for (int column = 0; column < 4; column++) + dataStructure.origin[row * 4 + column] = origin.get(column * 4 + row).asDouble(); + } + Map tableNames = new HashMap<>(); + for (JsonNode item : entry.path("entitySets")) { + EntitySet set = new EntitySet("", item.hasNonNull("rank") ? item.get("rank").asLong() : null); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), set); + tableNames.put(item.get("propertyTable").asInt(), item.get("name").asText()); + dataStructure.entitySets.add(set); + } + for (JsonNode item : entry.path("relations")) { + Relation relation = new Relation("", item.get("sourceSet").asText(), item.get("targetSet").asText()); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), relation); + dataStructure.relations.add(relation); + } + if (entry.has("scalars")) { + Table scalars = new Table(""); + decodeTable(document, binary, metadata, variables, entry.get("scalars").asInt(), scalars); + dataStructure.scalars = scalars; + } + if (entry.has("grid")) dataStructure.grid = decodeGrid(entry.get("grid")); + for (JsonNode meshEntry : entry.path("meshes")) + dataStructure.meshes.add(decodeMesh(document, binary, meshEntry, tableNames)); + frame.dataStructures.add(dataStructure); + } + for (JsonNode entry : scene.path("mappings")) { + Mapping mapping = new Mapping("", entry.get("source").get("dataStructure").asText(), + entry.get("source").get("entitySet").asText(), entry.get("target").get("dataStructure").asText(), + entry.get("target").get("entitySet").asText(), entry.hasNonNull("method") ? entry.get("method").asText() : null); + decodeTable(document, binary, metadata, variables, entry.get("propertyTable").asInt(), mapping); + frame.mappings.add(mapping); + } + ObjectNode others = Glb.MAPPER.createObjectNode(); + for (Iterator> items = extensions.fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + if (!List.of(METADATA, EXTENSION, FEATURES).contains(item.getKey())) others.set(item.getKey(), item.getValue()); + } + if (!others.isEmpty()) frame.unknown.set("extensions", others); + if (document.has("extras")) frame.unknown.set("extras", document.get("extras")); + return frame; + } + + /** The frame of the GLB file {@code path}. */ + public static Frame read(Path path) throws IOException { + return fromBytes(Files.readAllBytes(path)); + } + + private static ByteBuffer view(JsonNode document, byte[] binary, int index) { + JsonNode view = document.get("bufferViews").get(index); + if (view.path("buffer").asInt(0) != 0) throw new FspmTfException("fspm-TF files keep every buffer view in the GLB binary chunk"); + int start = view.path("byteOffset").asInt(0), length = view.get("byteLength").asInt(); + if (start < 0 || length < 0 || start + (long) length > binary.length) throw new FspmTfException("a buffer view runs past the binary chunk"); + return ByteBuffer.wrap(binary, start, length).slice().order(ByteOrder.LITTLE_ENDIAN); + } + + private static long[] unsignedValues(String type, ByteBuffer bytes, int count) { + long[] values = new long[count]; + for (int index = 0; index < count; index++) + values[index] = switch (type) { + case "UINT8" -> Byte.toUnsignedLong(bytes.get(index)); + case "UINT16" -> Short.toUnsignedLong(bytes.getShort(index * 2)); + case "UINT32" -> Integer.toUnsignedLong(bytes.getInt(index * 4)); + case "UINT64" -> bytes.getLong(index * 8); + default -> throw new FspmTfException("offsets or enum values of type " + type + " are not part of fspm-TF 0.1"); + }; + return values; + } + + private static Column readScalars(ComponentType type, ByteBuffer bytes, int count, int width) { + int size = count * width; + if ((long) size * type.size > bytes.remaining()) throw new FspmTfException("a buffer view is too short for its values"); + return switch (type) { + case INT8, UINT8 -> { byte[] values = new byte[size]; bytes.get(values); yield new Column(type, values, width); } + case INT16, UINT16 -> { short[] values = new short[size]; bytes.asShortBuffer().get(values); yield new Column(type, values, width); } + case INT32, UINT32 -> { int[] values = new int[size]; bytes.asIntBuffer().get(values); yield new Column(type, values, width); } + case INT64, UINT64 -> { long[] values = new long[size]; bytes.asLongBuffer().get(values); yield new Column(type, values, width); } + case FLOAT32 -> { float[] values = new float[size]; bytes.asFloatBuffer().get(values); yield new Column(type, values, width); } + case FLOAT64 -> { double[] values = new double[size]; bytes.asDoubleBuffer().get(values); yield new Column(type, values, width); } + default -> throw new FspmTfException("component type " + type + " is not a number"); + }; + } + + private static void decodeTable(JsonNode document, byte[] binary, JsonNode metadata, JsonNode variables, int index, + Table table) { + JsonNode description = metadata.get("propertyTables").get(index); + String classId = description.get("class").asText(); + table.name = description.path("name").asText(classId); + JsonNode schema = metadata.get("schema"); + JsonNode properties = schema.get("classes").get(classId).get("properties"); + int count = description.get("count").asInt(); + for (Iterator> items = description.get("properties").fields(); items.hasNext(); ) { + Map.Entry item = items.next(); + String name = item.getKey(); + JsonNode stored = item.getValue(), property = properties.get(name); + ByteBuffer raw = view(document, binary, stored.get("values").asInt()); + String type = property.get("type").asText(); + Column column; + switch (type) { + case "ENUM" -> { + JsonNode enumeration = schema.get("enums").get(property.get("enumType").asText()); + long[] codes = unsignedValues(enumeration.path("valueType").asText("UINT16"), raw, count); + Map nameOf = new HashMap<>(); + for (JsonNode value : enumeration.get("values")) nameOf.put(value.get("value").asLong(), value.get("name").asText()); + String[] labels = new String[count]; + for (int row = 0; row < count; row++) labels[row] = nameOf.get(codes[row]); + column = new Column(ComponentType.STRING, labels); + } + case "STRING" -> { + long[] offsets = unsignedValues(stored.path("stringOffsetType").asText("UINT32"), + view(document, binary, stored.get("stringOffsets").asInt()), count + 1); + String[] text = new String[count]; + byte[] bytes = new byte[raw.remaining()]; + raw.get(bytes); + for (int row = 0; row < count; row++) { + if (offsets[row + 1] > bytes.length || offsets[row] > offsets[row + 1]) + throw new FspmTfException("'" + classId + "." + name + "': string offsets outside its values"); + text[row] = new String(bytes, (int) offsets[row], (int) (offsets[row + 1] - offsets[row]), StandardCharsets.UTF_8); + } + column = new Column(ComponentType.STRING, text); + } + case "BOOLEAN" -> { + if (raw.remaining() * 8L < count) throw new FspmTfException("'" + classId + "." + name + "': too few bits"); + boolean[] values = new boolean[count]; + for (int row = 0; row < count; row++) values[row] = ((raw.get(row / 8) >> (row % 8)) & 1) == 1; + column = new Column(ComponentType.BOOLEAN, values); + } + case "SCALAR" -> { + boolean vector = property.path("array").asBoolean(false); + if (vector && !property.has("count")) + throw new FspmTfException("'" + classId + "." + name + "': variable-length arrays are not part of fspm-TF 0.1"); + int width = vector ? property.get("count").asInt() : 1; + ComponentType componentType; + try { + componentType = ComponentType.valueOf(property.get("componentType").asText()); + } catch (IllegalArgumentException error) { + throw new FspmTfException("component type " + property.get("componentType") + " is not part of fspm-TF 0.1"); + } + column = readScalars(componentType, raw, count, width); + restoreStandIns(column, property); + } + default -> throw new FspmTfException("'" + classId + "." + name + "': property type " + type + " is not part of fspm-TF 0.1"); + } + table.columns.put(name, column); + JsonNode declared = variables.path(classId).get(name); + if (declared != null || property.has("description")) { + Variable variable = new Variable(); + if (declared != null) { + variable.unit = declared.path("unit").asText("1"); + variable.role = declared.path("role").asText("state"); + if (declared.hasNonNull("standardName")) variable.standardName = declared.get("standardName").asText(); + if (declared.hasNonNull("kind")) variable.kind = declared.get("kind").asText(); + if (declared.hasNonNull("minimum")) variable.minimum = declared.get("minimum").asDouble(); + if (declared.hasNonNull("maximum")) variable.maximum = declared.get("maximum").asDouble(); + for (JsonNode reference : declared.path("references")) variable.references.add(reference.asText()); + for (JsonNode doi : declared.path("doi")) variable.doi.add(doi.asText()); + } + if (property.has("description")) variable.description = property.get("description").asText(); + table.variables.put(name, variable); + } + } + } + + private static void restoreStandIns(Column column, JsonNode property) { + JsonNode infinity = property.path("extras").path("FSPM_infinity"); + if (column.type == ComponentType.FLOAT64) { + double[] values = column.doubles(); + if (property.has("noData")) { + double noData = property.get("noData").asDouble(); + for (int index = 0; index < values.length; index++) if (values[index] == noData) values[index] = Double.NaN; + } + if (!infinity.isMissingNode()) { + double positive = infinity.get("positive").asDouble(), negative = infinity.get("negative").asDouble(); + for (int index = 0; index < values.length; index++) { + if (values[index] == positive) values[index] = Double.POSITIVE_INFINITY; + else if (values[index] == negative) values[index] = Double.NEGATIVE_INFINITY; + } + } + } else if (column.type == ComponentType.FLOAT32) { + float[] values = column.floats(); + if (property.has("noData")) { + float noData = (float) property.get("noData").asDouble(); + for (int index = 0; index < values.length; index++) if (values[index] == noData) values[index] = Float.NaN; + } + if (!infinity.isMissingNode()) { + float positive = (float) infinity.get("positive").asDouble(), negative = (float) infinity.get("negative").asDouble(); + for (int index = 0; index < values.length; index++) { + if (values[index] == positive) values[index] = Float.POSITIVE_INFINITY; + else if (values[index] == negative) values[index] = Float.NEGATIVE_INFINITY; + } + } + } + } + + private static Grid decodeGrid(JsonNode item) { + Grid grid = new Grid(); + grid.kind = item.get("kind").asText(); + grid.shape = new long[item.get("shape").size()]; + for (int index = 0; index < grid.shape.length; index++) grid.shape[index] = item.get("shape").get(index).asLong(); + grid.origin = doubles(item.get("origin")); + grid.spacing = doubles(item.get("spacing")); + List axes = new ArrayList<>(); + for (JsonNode axis : item.get("axes")) axes.add(axis.asText()); + grid.axes = axes; + grid.periodic = new boolean[item.get("periodic").size()]; + for (int index = 0; index < grid.periodic.length; index++) grid.periodic[index] = item.get("periodic").get(index).asBoolean(); + grid.zUp = item.path("zUp").asBoolean(true); + grid.maxLevel = item.path("maxLevel").asLong(0); + grid.cells = item.get("cells").asText(); + return grid; + } + + private static double[] doubles(JsonNode array) { + double[] values = new double[array.size()]; + for (int index = 0; index < values.length; index++) values[index] = array.get(index).asDouble(); + return values; + } + + /** An accessor's values, as doubles, row after row. */ + private static double[] accessorValues(JsonNode document, byte[] binary, int index) { + JsonNode accessor = document.get("accessors").get(index); + int width = switch (accessor.get("type").asText()) { + case "SCALAR" -> 1; + case "VEC2" -> 2; + case "VEC3" -> 3; + case "VEC4" -> 4; + default -> throw new FspmTfException("accessor type " + accessor.get("type") + " is not used by fspm-TF"); + }; + int componentType = accessor.get("componentType").asInt(); + int size = switch (componentType) { + case FLOAT, UNSIGNED_INT -> 4; + case UNSIGNED_SHORT -> 2; + case UNSIGNED_BYTE -> 1; + default -> throw new FspmTfException("accessor component type " + componentType + " is not used by fspm-TF"); + }; + int viewIndex = accessor.get("bufferView").asInt(); + ByteBuffer raw = view(document, binary, viewIndex); + int stride = document.get("bufferViews").get(viewIndex).path("byteStride").asInt(size * width); + int start = accessor.path("byteOffset").asInt(0), count = accessor.get("count").asInt(); + double[] values = new double[count * width]; + for (int item = 0; item < count; item++) + for (int component = 0; component < width; component++) { + int at = start + item * stride + component * size; + if (at + size > raw.limit()) throw new FspmTfException("an accessor runs past its buffer view"); + values[item * width + component] = switch (componentType) { + case FLOAT -> raw.getFloat(at); + case UNSIGNED_INT -> Integer.toUnsignedLong(raw.getInt(at)); + case UNSIGNED_SHORT -> Short.toUnsignedInt(raw.getShort(at)); + default -> Byte.toUnsignedInt(raw.get(at)); + }; + } + return values; + } + + private static Mesh decodeMesh(JsonNode document, byte[] binary, JsonNode meshEntry, Map tableNames) { + JsonNode primitive = document.get("meshes").get(meshEntry.get("mesh").asInt()).get("primitives").get(0); + JsonNode attributes = primitive.get("attributes"); + Mesh mesh = new Mesh(meshEntry.get("name").asText(), accessorValues(document, binary, attributes.get("POSITION").asInt())); + for (JsonNode feature : primitive.path("extensions").path(FEATURES).path("featureIds")) { + double[] stored = accessorValues(document, binary, attributes.get("_FEATURE_ID_" + feature.get("attribute").asInt()).asInt()); + long[] rows = new long[stored.length]; + long nullRow = feature.has("nullFeatureId") ? feature.get("nullFeatureId").asLong() : Long.MIN_VALUE; + for (int vertex = 0; vertex < rows.length; vertex++) + rows[vertex] = (long) stored[vertex] == nullRow ? EntitySet.NO_ID : (long) stored[vertex]; + mesh.featureRows.put(tableNames.get(feature.get("propertyTable").asInt()), rows); + } + for (JsonNode item : meshEntry.path("display")) { + double[] stored = accessorValues(document, binary, attributes.get(item.get("attribute").asText()).asInt()); + float[] values = new float[stored.length]; + for (int vertex = 0; vertex < values.length; vertex++) values[vertex] = (float) stored[vertex]; + String variable = item.get("variable").asText(); + mesh.display.put(variable, values); + if (item.hasNonNull("entitySet")) mesh.displaySources.put(variable, item.get("entitySet").asText()); + } + if (primitive.has("indices")) { + double[] stored = accessorValues(document, binary, primitive.get("indices").asInt()); + mesh.indices = new long[stored.length]; + for (int index = 0; index < stored.length; index++) mesh.indices[index] = (long) stored[index]; + } + return mesh; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Column.java b/bindings/java/src/main/java/org/openalea/fspmtf/Column.java new file mode 100644 index 0000000..de3c7cc --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Column.java @@ -0,0 +1,91 @@ +package org.openalea.fspmtf; + +import java.lang.reflect.Array; +import java.util.Arrays; + +/** + * A column: its values (a Java array, see {@link ComponentType}), {@code width} per row (1: one value per row; 2 or more: + * a fixed-size vector per row, row after row). + */ +public final class Column { + public final ComponentType type; + public final Object values; + public final int width; + + public Column(ComponentType type, Object values, int width) { + if (!values.getClass().isArray()) throw new FspmTfException("a column's values are an array"); + Class expected = switch (type) { + case INT8, UINT8 -> byte.class; + case INT16, UINT16 -> short.class; + case INT32, UINT32 -> int.class; + case INT64, UINT64 -> long.class; + case FLOAT32 -> float.class; + case FLOAT64 -> double.class; + case BOOLEAN -> boolean.class; + case STRING -> String.class; + }; + if (values.getClass().getComponentType() != expected) + throw new FspmTfException("a " + type + " column holds " + expected.getSimpleName() + " values"); + if (width < 1) throw new FspmTfException("a column's width is at least 1"); + this.type = type; + this.values = values; + this.width = width; + } + + public Column(ComponentType type, Object values) { + this(type, values, 1); + } + + public static Column of(long... values) { return new Column(ComponentType.INT64, values); } + public static Column of(int... values) { return new Column(ComponentType.INT32, values); } + public static Column of(double... values) { return new Column(ComponentType.FLOAT64, values); } + public static Column of(float... values) { return new Column(ComponentType.FLOAT32, values); } + public static Column of(boolean... values) { return new Column(ComponentType.BOOLEAN, values); } + public static Column of(String... values) { return new Column(ComponentType.STRING, values); } + /** A vector column: {@code width} values per row, row after row. */ + public static Column vectors(int width, double... values) { return new Column(ComponentType.FLOAT64, values, width); } + + /** The number of values (rows × width). */ + public int size() { + return Array.getLength(values); + } + + public int rows() { + return size() / width; + } + + public long[] longs() { return (long[]) values; } + public int[] ints() { return (int[]) values; } + public double[] doubles() { return (double[]) values; } + public float[] floats() { return (float[]) values; } + public boolean[] booleans() { return (boolean[]) values; } + public String[] strings() { return (String[]) values; } + + /** The value at {@code index} as a double (unsigned integers read as unsigned). */ + public double doubleAt(int index) { + return switch (type) { + case INT8 -> ((byte[]) values)[index]; + case UINT8 -> Byte.toUnsignedInt(((byte[]) values)[index]); + case INT16 -> ((short[]) values)[index]; + case UINT16 -> Short.toUnsignedInt(((short[]) values)[index]); + case INT32 -> ((int[]) values)[index]; + case UINT32 -> Integer.toUnsignedLong(((int[]) values)[index]); + case INT64 -> ((long[]) values)[index]; + case UINT64 -> Double.parseDouble(Long.toUnsignedString(((long[]) values)[index])); + case FLOAT32 -> ((float[]) values)[index]; + case FLOAT64 -> ((double[]) values)[index]; + default -> throw new FspmTfException("a " + type + " column has no numbers"); + }; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Column column)) return false; + return type == column.type && width == column.width && Arrays.deepEquals(new Object[] {values}, new Object[] {column.values}); + } + + @Override + public int hashCode() { + return type.hashCode() * 31 + width; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/ComponentType.java b/bindings/java/src/main/java/org/openalea/fspmtf/ComponentType.java new file mode 100644 index 0000000..38d4297 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/ComponentType.java @@ -0,0 +1,33 @@ +package org.openalea.fspmtf; + +/** + * The type of a column's values. Integers are held in the Java array of their width (UINT8 in a byte[], UINT64 in a + * long[]), their bits as stored; booleans in a boolean[]; text, labels and edge types in a String[]. + */ +public enum ComponentType { + INT8(1, "int8"), UINT8(1, "uint8"), INT16(2, "int16"), UINT16(2, "uint16"), INT32(4, "int32"), + UINT32(4, "uint32"), INT64(8, "int64"), UINT64(8, "uint64"), FLOAT32(4, "float32"), FLOAT64(8, "float64"), + BOOLEAN(0, "bool"), STRING(0, "string"); + + /** Bytes per value (0: booleans and text). */ + public final int size; + /** The type's name in the conformance files' expected values (numpy's, or "string"). */ + public final String dtype; + + ComponentType(int size, String dtype) { + this.size = size; + this.dtype = dtype; + } + + public boolean isNumber() { + return size > 0; + } + + public boolean isFloat() { + return this == FLOAT32 || this == FLOAT64; + } + + public boolean isUnsigned() { + return this == UINT8 || this == UINT16 || this == UINT32 || this == UINT64; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java b/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java new file mode 100644 index 0000000..4c8e3da --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java @@ -0,0 +1,46 @@ +package org.openalea.fspmtf; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.stream.Stream; + +/** + * The Java codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump it + * as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). Then + * .github/actions/conformance compares the dumps with the expected values and validates the written files. + * + * 
java -jar fspm-tf.jar CONFORMANCE OUT
+ */ +public final class ConformanceRun { + private ConformanceRun() {} + + public static void main(String[] arguments) throws IOException { + if (arguments.length != 2) { + System.err.println("usage: java -jar fspm-tf.jar CONFORMANCE OUT"); + System.exit(2); + } + Path conformance = Path.of(arguments[0]), out = Path.of(arguments[1]); + List files; + try (Stream paths = Files.walk(conformance)) { + files = paths.filter(path -> path.toString().endsWith(".glb")).sorted().toList(); + } + for (Path file : files) { + Path relative = conformance.relativize(file); + Frame frame = Codec.read(file); + String name = relative.toString(); + Path dump = out.resolve("dumps").resolve(name.substring(0, name.length() - ".glb".length()) + ".json"); + Path written = out.resolve("written").resolve(relative); + Files.createDirectories(dump.getParent()); + Files.createDirectories(written.getParent()); + Files.writeString(dump, Glb.MAPPER.writeValueAsString(Dump.frameToJson(frame)) + "\n"); + Codec.write(written, frame); // written again from what was read + } + Path index = conformance.resolve("growing_sequence").resolve(Sequences.INDEX_FILE); + if (Files.exists(index)) + Files.copy(index, out.resolve("written").resolve("growing_sequence").resolve(Sequences.INDEX_FILE), + StandardCopyOption.REPLACE_EXISTING); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java b/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java new file mode 100644 index 0000000..72dab03 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java @@ -0,0 +1,40 @@ +package org.openalea.fspmtf; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * One plant population's graph, or one environment model's grid. origin: its frame in the scene, a 4×4 matrix row by + * row. extras: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); other + * readers keep them without interpreting them. + */ +public final class DataStructure { + public String id; + public String kind = "graph"; + public final List entitySets = new ArrayList<>(); + public final List relations = new ArrayList<>(); + public Table scalars; + public Grid grid; + public final List meshes = new ArrayList<>(); + public double[] origin = {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + public String platform; + public String model; + public ObjectNode extras = JsonNodeFactory.instance.objectNode(); + + public DataStructure(String id) { + this.id = id; + } + + public DataStructure(String id, String kind) { + this.id = id; + this.kind = kind; + } + + public EntitySet entitySet(String name) { + for (EntitySet set : entitySets) if (set.name.equals(name)) return set; + throw new FspmTfException("data structure '" + id + "' has no entity set '" + name + "'"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java new file mode 100644 index 0000000..0b24fd3 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java @@ -0,0 +1,191 @@ +package org.openalea.fspmtf; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md), and + * the comparison {@code fspm-tf compare} makes. + */ +public final class Dump { + private Dump() {} + + private static void addValue(ArrayNode array, Column column, int index) { + switch (column.type) { + case FLOAT32, FLOAT64 -> { + double value = column.type == ComponentType.FLOAT32 ? column.floats()[index] : column.doubles()[index]; + if (Double.isNaN(value)) array.addNull(); + else if (Double.isInfinite(value)) array.add(value > 0 ? "Infinity" : "-Infinity"); + else array.add(value); + } + case BOOLEAN -> array.add(column.booleans()[index]); + case STRING -> array.add(column.strings()[index]); + case UINT64 -> array.add(new BigInteger(Long.toUnsignedString(column.longs()[index]))); + case INT64 -> array.add(column.longs()[index]); + default -> array.add((long) column.doubleAt(index)); + } + } + + private static ArrayNode columnJson(Column column) { + ArrayNode items = Glb.MAPPER.createArrayNode(); + if (column.width == 1) { + for (int index = 0; index < column.size(); index++) addValue(items, column, index); + } else { + for (int row = 0; row < column.rows(); row++) { + ArrayNode vector = items.addArray(); + for (int component = 0; component < column.width; component++) addValue(vector, column, row * column.width + component); + } + } + return items; + } + + private static ObjectNode tableJson(Table table) { + ObjectNode entry = Glb.MAPPER.createObjectNode().put("name", table.name).put("count", table.rowCount()); + ObjectNode columns = entry.putObject("columns"); + for (Map.Entry item : table.columns.entrySet()) { + ObjectNode column = columns.putObject(item.getKey()).put("dtype", item.getValue().type.dtype); + column.set("values", columnJson(item.getValue())); + } + ObjectNode declarations = Glb.MAPPER.createObjectNode(); + for (Map.Entry item : table.variables.entrySet()) { + Variable variable = item.getValue(); + ObjectNode declared = declarations.putObject(item.getKey()).put("unit", variable.unit).put("role", variable.role); + if (variable.standardName != null) declared.put("standard_name", variable.standardName); + if (variable.kind != null) declared.put("kind", variable.kind); + if (variable.description != null) declared.put("description", variable.description); + if (variable.minimum != null) declared.put("minimum", variable.minimum); + if (variable.maximum != null) declared.put("maximum", variable.maximum); + if (!variable.references.isEmpty()) variable.references.forEach(declared.putArray("references")::add); + if (!variable.doi.isEmpty()) variable.doi.forEach(declared.putArray("doi")::add); + } + if (!declarations.isEmpty()) entry.set("variables", declarations); + return entry; + } + + private static ArrayNode numbers(double[] values) { + ArrayNode array = Glb.MAPPER.createArrayNode(); + for (double value : values) array.add(value); + return array; + } + + /** A frame as plain JSON values: NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. */ + public static ObjectNode frameToJson(Frame frame) { + ObjectNode dump = Glb.MAPPER.createObjectNode().put("time", frame.time).put("time_unit", frame.timeUnit).put("step", frame.step); + ArrayNode dataStructures = dump.putArray("data_structures"); + for (DataStructure dataStructure : frame.dataStructures) { + ObjectNode entry = dataStructures.addObject().put("id", dataStructure.id).put("kind", dataStructure.kind); + ArrayNode origin = entry.putArray("origin"); + for (int row = 0; row < 4; row++) { + ArrayNode values = origin.addArray(); + for (int column = 0; column < 4; column++) values.add(dataStructure.origin[row * 4 + column]); + } + ArrayNode sets = entry.putArray("entity_sets"); + for (EntitySet set : dataStructure.entitySets) { + ObjectNode item = tableJson(set); + if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + sets.add(item); + } + if (dataStructure.platform != null) entry.put("platform", dataStructure.platform); + if (dataStructure.model != null) entry.put("model", dataStructure.model); + if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); + if (!dataStructure.relations.isEmpty()) { + ArrayNode relations = entry.putArray("relations"); + for (Relation relation : dataStructure.relations) + relations.add(tableJson(relation).put("source_set", relation.sourceSet).put("target_set", relation.targetSet)); + } + if (dataStructure.scalars != null) entry.set("scalars", tableJson(dataStructure.scalars)); + if (dataStructure.grid != null) { + Grid grid = dataStructure.grid; + ObjectNode item = entry.putObject("grid").put("kind", grid.kind); + ArrayNode shape = item.putArray("shape"); + for (long size : grid.shape) shape.add(size); + item.set("origin", numbers(grid.origin)); + item.set("spacing", numbers(grid.spacing)); + grid.axes.forEach(item.putArray("axes")::add); + ArrayNode periodic = item.putArray("periodic"); + for (boolean value : grid.periodic) periodic.add(value); + item.put("z_up", grid.zUp).put("max_level", grid.maxLevel).put("cells", grid.cells); + } + if (!dataStructure.meshes.isEmpty()) { + ArrayNode meshes = entry.putArray("meshes"); + for (Mesh mesh : dataStructure.meshes) { + ObjectNode item = meshes.addObject().put("name", mesh.name); + ArrayNode positions = item.putArray("positions_float32"); + for (int vertex = 0; vertex < mesh.vertexCount(); vertex++) { + ArrayNode position = positions.addArray(); + for (int axis = 0; axis < 3; axis++) position.add((double) (float) mesh.positions[vertex * 3 + axis]); + } + ObjectNode featureRows = item.putObject("feature_rows"); + for (Map.Entry rows : mesh.featureRows.entrySet()) { + ArrayNode values = featureRows.putArray(rows.getKey()); + for (long row : rows.getValue()) values.add(row); + } + if (mesh.indices != null) { + ArrayNode indices = item.putArray("indices"); + for (long index : mesh.indices) indices.add(index); + } + if (!mesh.display.isEmpty()) { + ObjectNode display = item.putObject("display"); + for (Map.Entry values : mesh.display.entrySet()) + display.set(values.getKey(), columnJson(new Column(ComponentType.FLOAT32, values.getValue()))); + } + } + } + } + ArrayNode mappings = dump.putArray("mappings"); + for (Mapping mapping : frame.mappings) { + ObjectNode item = tableJson(mapping); + item.putArray("source").add(mapping.sourceDataStructure).add(mapping.sourceSet); + item.putArray("target").add(mapping.targetDataStructure).add(mapping.targetSet); + if (mapping.method == null) item.putNull("method"); else item.put("method", mapping.method); + mappings.add(item); + } + return dump; + } + + /** + * The differences between two JSON values, as {@code fspm-tf compare} finds them: objects by key, arrays in order, + * numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). + */ + public static List compare(JsonNode actual, JsonNode expected, String path) { + List found = new ArrayList<>(); + if (expected.isObject()) { + if (!actual.isObject()) return List.of(path + ": an object expected"); + for (Iterator names = expected.fieldNames(); names.hasNext(); ) { + String name = names.next(); + if (!actual.has(name)) found.add(path + ": missing key '" + name + "'"); + else found.addAll(compare(actual.get(name), expected.get(name), path + "." + name)); + } + for (Iterator names = actual.fieldNames(); names.hasNext(); ) { + String name = names.next(); + if (!expected.has(name)) found.add(path + ": unexpected key '" + name + "'"); + } + return found; + } + if (expected.isArray()) { + if (!actual.isArray()) return List.of(path + ": an array expected"); + if (actual.size() != expected.size()) + return List.of(path + ": " + actual.size() + " items, " + expected.size() + " expected"); + for (int index = 0; index < expected.size(); index++) + found.addAll(compare(actual.get(index), expected.get(index), path + "[" + index + "]")); + return found; + } + if (expected.isNumber()) { + if (actual.isNumber() && actual.asDouble() == expected.asDouble()) return found; + return List.of(path + ": " + actual + ", " + expected + " expected"); + } + if (!actual.equals(expected)) return List.of(path + ": " + actual + ", " + expected + " expected"); + return found; + } + + public static List compare(JsonNode actual, JsonNode expected) { + return compare(actual, expected, "$"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java b/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java new file mode 100644 index 0000000..7424789 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java @@ -0,0 +1,36 @@ +package org.openalea.fspmtf; + +/** + * The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (no rank). Reserved + * columns: id (INT64, stable across frames, never reused), label (text), complex (the id of the containing entity at a + * coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), edge_type ("none", + * "successor", "branch"); an adaptive grid's cells: level and index. + */ +public class EntitySet extends Table { + /** The id of no entity. */ + public static final long NO_ID = -1; + + public Long rank; + + public EntitySet(String name, Long rank) { + super(name); + this.rank = rank; + } + + public EntitySet(String name) { + this(name, null); + } + + @Override + public Kind kind() { + return Kind.ENTITY_SET; + } + + /** The ids of its entities: its id column, or a grid's row numbers (from 0) when it has none. */ + public long[] ids() { + if (has("id")) return column("id").longs(); + long[] rows = new long[rowCount()]; + for (int row = 0; row < rows.length; row++) rows[row] = row; + return rows; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java b/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java new file mode 100644 index 0000000..2fb2300 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java @@ -0,0 +1,27 @@ +package org.openalea.fspmtf; + +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A scene at one time step (time in timeUnit, step), or a parameter set. unknown: the content of a read file this + * library does not interpret (other root extensions, the root's extras), written back as it was. + */ +public final class Frame { + public final List dataStructures = new ArrayList<>(); + public final List mappings = new ArrayList<>(); + public double time = 0.0; + public String timeUnit = "d"; + public long step = 0; + public ObjectNode writer = JsonNodeFactory.instance.objectNode(); + public String vocabularyVersion; + public ObjectNode unknown = JsonNodeFactory.instance.objectNode(); + + public DataStructure dataStructure(String id) { + for (DataStructure dataStructure : dataStructures) if (dataStructure.id.equals(id)) return dataStructure; + throw new FspmTfException("the frame has no data structure '" + id + "'"); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/FspmTfException.java b/bindings/java/src/main/java/org/openalea/fspmtf/FspmTfException.java new file mode 100644 index 0000000..bae6912 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/FspmTfException.java @@ -0,0 +1,14 @@ +package org.openalea.fspmtf; + +/** What the specification forbids, or a file that is not fspm-TF. */ +public class FspmTfException extends RuntimeException { + private static final long serialVersionUID = 1L; + + public FspmTfException(String message) { + super(message); + } + + public FspmTfException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Glb.java b/bindings/java/src/main/java/org/openalea/fspmtf/Glb.java new file mode 100644 index 0000000..14c3cce --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Glb.java @@ -0,0 +1,102 @@ +package org.openalea.fspmtf; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary chunk + * padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. + */ +public final class Glb { + private static final int MAGIC = 0x46546C67; // "glTF" + private static final int VERSION = 2; + private static final int JSON_CHUNK = 0x4E4F534A; // "JSON" + private static final int BINARY_CHUNK = 0x004E4942; // "BIN\0" + static final ObjectMapper MAPPER = new ObjectMapper(); + + /** A GLB file's JSON document and binary buffer. */ + public record Contents(JsonNode document, byte[] binary) {} + + private Glb() {} + + static int padding(long length) { + return (int) ((8 - length % 8) % 8); + } + + /** The bytes of a GLB file from its JSON document and its binary buffer (both padded to 8 bytes). */ + public static byte[] pack(JsonNode document, byte[] binary) { + byte[] text; + try { + text = MAPPER.writeValueAsBytes(document); + } catch (IOException error) { + throw new FspmTfException("the document cannot be written as JSON", error); + } + int textLength = text.length + padding(text.length); + int binaryLength = binary.length + padding(binary.length); + long total = 12L + 8 + textLength + (binary.length == 0 ? 0 : 8 + binaryLength); + if (total > 0xFFFFFFFFL) throw new FspmTfException("a GLB file is at most 4 GiB"); + ByteBuffer buffer = ByteBuffer.allocate((int) total).order(ByteOrder.LITTLE_ENDIAN); + buffer.putInt(MAGIC).putInt(VERSION).putInt((int) total).putInt(textLength).putInt(JSON_CHUNK).put(text); + for (int index = text.length; index < textLength; index++) buffer.put((byte) ' '); + if (binary.length > 0) { + buffer.putInt(binaryLength).putInt(BINARY_CHUNK).put(binary); + buffer.put(new byte[binaryLength - binary.length]); + } + return buffer.array(); + } + + /** The JSON document (keys in their order) and the binary buffer of a GLB file's bytes. */ + public static Contents unpack(byte[] data) { + if (data.length < 20) throw new FspmTfException("not a GLB file: too short"); + ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN); + if (buffer.getInt(0) != MAGIC) throw new FspmTfException("not a GLB file: wrong magic"); + if (buffer.getInt(4) != VERSION) throw new FspmTfException("GLB version " + buffer.getInt(4) + ", 2 expected"); + long total = Integer.toUnsignedLong(buffer.getInt(8)); + if (total != data.length) + throw new FspmTfException("GLB header length " + total + ", the file holds " + data.length + " bytes"); + int offset = 12; + JsonNode document = null; + byte[] binary = new byte[0]; + while (offset < total) { + int length = buffer.getInt(offset), type = buffer.getInt(offset + 4); + if (length < 0 || offset + 8L + length > total) throw new FspmTfException("a GLB chunk runs past the end of the file"); + if (type == JSON_CHUNK) { + try { + document = MAPPER.readTree(Arrays.copyOfRange(data, offset + 8, offset + 8 + length)); + } catch (IOException error) { + throw new FspmTfException("the GLB file's JSON chunk is not JSON", error); + } + } else if (type == BINARY_CHUNK) { + binary = Arrays.copyOfRange(data, offset + 8, offset + 8 + length); + } + offset += 8 + length; + } + if (document == null) throw new FspmTfException("GLB file without a JSON chunk"); + return new Contents(document, binary); + } + + /** Appends arrays to one binary buffer, each at an 8-byte aligned offset, and records their buffer views. */ + static final class BufferBuilder { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final com.fasterxml.jackson.databind.node.ArrayNode views = MAPPER.createArrayNode(); + + int add(byte[] bytes, int target) { + data.write(new byte[padding(data.size())], 0, padding(data.size())); + var view = views.addObject(); + view.put("buffer", 0).put("byteOffset", data.size()).put("byteLength", bytes.length); + if (target != 0) view.put("target", target); + data.write(bytes, 0, bytes.length); + return views.size() - 1; + } + + int add(byte[] bytes) { + return add(bytes, 0); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Grid.java b/bindings/java/src/main/java/org/openalea/fspmtf/Grid.java new file mode 100644 index 0000000..1bd78cd --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Grid.java @@ -0,0 +1,20 @@ +package org.openalea.fspmtf; + +import java.util.List; + +/** + * A grid's layout. Regular: shape cells of size spacing from origin, along axes, periodic per axis, z pointing up or + * down (zUp). Adaptive: the same for its base level, refined up to maxLevel, its leaf cells (an entity set with level + * and index columns) explicit. cells: the name of the cells' entity set. + */ +public final class Grid { + public String kind = "regular"; + public long[] shape = {}; + public double[] origin = {}; + public double[] spacing = {}; + public List axes = List.of("x", "y", "z"); + public boolean[] periodic = {}; + public boolean zUp = true; + public long maxLevel = 0; + public String cells = "Cell"; +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Mapping.java b/bindings/java/src/main/java/org/openalea/fspmtf/Mapping.java new file mode 100644 index 0000000..d186d8a --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Mapping.java @@ -0,0 +1,25 @@ +package org.openalea.fspmtf; + +/** + * Weights between the entities of two data structures: source and target ids, weight. source and target: a data + * structure's id and an entity set's name. + */ +public class Mapping extends Table { + public String sourceDataStructure, sourceSet, targetDataStructure, targetSet; + public String method; + + public Mapping(String name, String sourceDataStructure, String sourceSet, String targetDataStructure, + String targetSet, String method) { + super(name); + this.sourceDataStructure = sourceDataStructure; + this.sourceSet = sourceSet; + this.targetDataStructure = targetDataStructure; + this.targetSet = targetSet; + this.method = method; + } + + @Override + public Kind kind() { + return Kind.MAPPING; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Mesh.java b/bindings/java/src/main/java/org/openalea/fspmtf/Mesh.java new file mode 100644 index 0000000..61a539b --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Mesh.java @@ -0,0 +1,28 @@ +package org.openalea.fspmtf; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Triangles: positions (3 per vertex, in the data structure's frame), three consecutive vertices per triangle, or + * indices (3 per triangle, from 0) into them; featureRows: per entity set, the row (from 0) of each vertex's entity in + * that set, NO_ID for none; display: per variable, its float copy per vertex, from the entity set displaySources. + */ +public final class Mesh { + public String name; + public double[] positions; + public final Map featureRows = new LinkedHashMap<>(); + public long[] indices; + public final Map display = new LinkedHashMap<>(); + public final Map displaySources = new HashMap<>(); + + public Mesh(String name, double[] positions) { + this.name = name; + this.positions = positions; + } + + public int vertexCount() { + return positions.length / 3; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Relation.java b/bindings/java/src/main/java/org/openalea/fspmtf/Relation.java new file mode 100644 index 0000000..44f8310 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Relation.java @@ -0,0 +1,21 @@ +package org.openalea.fspmtf; + +/** + * An edge table: source and target entity ids (in the entity sets sourceSet and targetSet), an optional id per edge, + * an optional relation label per edge, and edge variables. + */ +public class Relation extends Table { + public String sourceSet; + public String targetSet; + + public Relation(String name, String sourceSet, String targetSet) { + super(name); + this.sourceSet = sourceSet; + this.targetSet = targetSet; + } + + @Override + public Kind kind() { + return Kind.RELATION; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Sequences.java b/bindings/java/src/main/java/org/openalea/fspmtf/Sequences.java new file mode 100644 index 0000000..7f1de15 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Sequences.java @@ -0,0 +1,72 @@ +package org.openalea.fspmtf; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeSet; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. */ +public final class Sequences { + public static final String INDEX_FILE = "scene.fspm.json"; + + /** One frame of a sequence's index; path: its file's absolute path, when read. */ + public record Entry(double time, long step, String file, Path path) {} + + /** A sequence's index: its frames, in time order. */ + public record Index(String version, String timeUnit, List frames) {} + + private Sequences() {} + + public static void writeIndex(Path path, Index index) throws IOException { + ObjectNode document = Glb.MAPPER.createObjectNode().put("fspmIndex", index.version()).put("timeUnit", index.timeUnit()); + ArrayNode frames = document.putArray("frames"); + for (Entry entry : index.frames()) frames.addObject().put("time", entry.time()).put("step", entry.step()).put("file", entry.file()); + Files.writeString(path, Glb.MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(document) + "\n", + StandardCharsets.UTF_8); + } + + public static Index readIndex(Path path) throws IOException { + JsonNode document = Glb.MAPPER.readTree(path.toFile()); + Path directory = path.toAbsolutePath().getParent(); + List frames = new ArrayList<>(); + for (JsonNode item : document.path("frames")) + frames.add(new Entry(item.get("time").asDouble(), item.get("step").asLong(), item.get("file").asText(), + directory.resolve(item.get("file").asText()))); + return new Index(document.path("fspmIndex").asText(Codec.SPECIFICATION_VERSION), document.path("timeUnit").asText("d"), frames); + } + + /** Writes frames to {@code directory}, one GLB each (name_000000.glb, ...), and their index; the index's path. */ + public static Path writeSequence(Path directory, List frames, String name) throws IOException { + Files.createDirectories(directory); + List entries = new ArrayList<>(); + TreeSet timeUnits = new TreeSet<>(); + for (int number = 0; number < frames.size(); number++) { + String file = String.format("%s_%06d.glb", name, number); + Codec.write(directory.resolve(file), frames.get(number)); + entries.add(new Entry(frames.get(number).time, frames.get(number).step, file, null)); + timeUnits.add(frames.get(number).timeUnit); + } + if (timeUnits.size() > 1) throw new FspmTfException("the frames' times are in several units: " + timeUnits); + Path index = directory.resolve(INDEX_FILE); + writeIndex(index, new Index(Codec.SPECIFICATION_VERSION, timeUnits.isEmpty() ? "d" : timeUnits.first(), entries)); + return index; + } + + public static Path writeSequence(Path directory, List frames) throws IOException { + return writeSequence(directory, frames, "frame"); + } + + /** The frames of the sequence whose index is {@code path}. */ + public static List readSequence(Path path) throws IOException { + List frames = new ArrayList<>(); + for (Entry entry : readIndex(path).frames()) frames.add(Codec.read(entry.path())); + return frames; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Table.java b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java new file mode 100644 index 0000000..f56dc20 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java @@ -0,0 +1,81 @@ +package org.openalea.fspmtf; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Columns of equal length, in their order, and the declarations of its variables: a data structure's scalars. */ +public class Table { + /** The kinds of tables, each with its reserved columns. */ + public enum Kind { + SCALARS(List.of()), + ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index")), + RELATION(List.of("id", "source", "target", "relation")), + MAPPING(List.of("source", "target", "weight")); + + public final List reserved; + + Kind(List reserved) { + this.reserved = reserved; + } + } + + public String name; + public final Map columns = new LinkedHashMap<>(); + public final Map variables = new LinkedHashMap<>(); + + public Table(String name) { + this.name = name; + } + + public Kind kind() { + return Kind.SCALARS; + } + + /** Sets a column (replacing it, or appending it); returns the table, for chaining. */ + public Table set(String columnName, Column column) { + columns.put(columnName, column); + return this; + } + + public Table declare(String columnName, Variable variable) { + variables.put(columnName, variable); + return this; + } + + public Column column(String columnName) { + Column column = columns.get(columnName); + if (column == null) throw new FspmTfException("table '" + name + "' has no column '" + columnName + "'"); + return column; + } + + public boolean has(String columnName) { + return columns.containsKey(columnName); + } + + public int rowCount() { + return columns.isEmpty() ? 0 : columns.values().iterator().next().rows(); + } + + public boolean isReserved(String columnName) { + return kind().reserved.contains(columnName); + } + + /** The variable columns: all but the reserved ones. */ + public List variableNames() { + List names = new ArrayList<>(); + for (String columnName : columns.keySet()) if (!isReserved(columnName)) names.add(columnName); + return names; + } + + /** Throws when columns have different numbers of rows. */ + public void checkLengths() { + for (Map.Entry entry : columns.entrySet()) { + Column column = entry.getValue(); + if (column.size() % column.width != 0) + throw new FspmTfException("column '" + entry.getKey() + "' of '" + name + "': its values are not whole rows"); + if (column.rows() != rowCount()) throw new FspmTfException("table '" + name + "': columns of different lengths"); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java b/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java new file mode 100644 index 0000000..b7386f9 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java @@ -0,0 +1,45 @@ +package org.openalea.fspmtf; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind + * (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance. + */ +public final class Variable { + public String unit = "1"; + public String standardName; + public String kind; + public String role = "state"; + public String description; + public Double minimum; + public Double maximum; + public List references = new ArrayList<>(); + public List doi = new ArrayList<>(); + + public Variable() {} + + public Variable(String unit, String standardName, String kind, String role) { + this.unit = unit; + this.standardName = standardName; + this.kind = kind; + this.role = role; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Variable variable)) return false; + return Objects.equals(unit, variable.unit) && Objects.equals(standardName, variable.standardName) + && Objects.equals(kind, variable.kind) && Objects.equals(role, variable.role) + && Objects.equals(description, variable.description) && Objects.equals(minimum, variable.minimum) + && Objects.equals(maximum, variable.maximum) && references.equals(variable.references) + && doi.equals(variable.doi); + } + + @Override + public int hashCode() { + return Objects.hash(unit, standardName, kind, role); + } +} diff --git a/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java new file mode 100644 index 0000000..7e1f28a --- /dev/null +++ b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java @@ -0,0 +1,192 @@ +package org.openalea.fspmtf; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import com.fasterxml.jackson.databind.JsonNode; + +/** The Java codec: the conformance files read as their expected values, written and read back the same, every column type, meshes, sequences, and what the specification forbids refused. */ +class CodecTest { + static final Path CONFORMANCE = Path.of(System.getProperty("fspmtf.conformance", "../../conformance")); + + static Stream conformanceFiles() throws IOException { + try (Stream paths = Files.walk(CONFORMANCE)) { + return paths.filter(path -> path.toString().endsWith(".glb")).sorted().toList().stream(); + } + } + + static JsonNode expectedOf(Path glb) throws IOException { + String name = glb.getFileName().toString(); + return Glb.MAPPER.readTree(glb.resolveSibling(name.substring(0, name.length() - 4) + ".expected.json").toFile()); + } + + /** The dump as it is written and read as JSON (numbers as JSON numbers). */ + static JsonNode asJson(Frame frame) throws IOException { + return Glb.MAPPER.readTree(Glb.MAPPER.writeValueAsString(Dump.frameToJson(frame))); + } + + @Test + void nineConformanceFiles() throws IOException { + assertEquals(9, conformanceFiles().count()); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void glbRepackedAsItWas(Path file) throws IOException { + Glb.Contents contents = Glb.unpack(Files.readAllBytes(file)); + Glb.Contents again = Glb.unpack(Glb.pack(contents.document(), contents.binary())); + assertArrayEquals(contents.binary(), again.binary()); + assertEquals(contents.document(), again.document()); + assertEquals(0, contents.binary().length % 8); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void readAsExpected(Path file) throws IOException { + List differences = Dump.compare(asJson(Codec.read(file)), expectedOf(file)); + assertTrue(differences.isEmpty(), () -> String.join("\n", differences.subList(0, Math.min(5, differences.size())))); + } + + @ParameterizedTest + @MethodSource("conformanceFiles") + void writtenAndReadBack(Path file) throws IOException { + Frame frame = Codec.fromBytes(Codec.toBytes(Codec.read(file))); + List differences = Dump.compare(asJson(frame), expectedOf(file)); + assertTrue(differences.isEmpty(), () -> String.join("\n", differences.subList(0, Math.min(5, differences.size())))); + } + + @Test + void columnsOfEveryType() { + EntitySet organs = new EntitySet("Organ", 1L); + organs.set("id", Column.of(0, 1, 2)).set("label", Column.of("Leaf", "Internode", "Leaf")) + .set("parent", Column.of(-1L, 0L, 1L)).set("edge_type", Column.of("none", "successor", "branch")) + .set("with_nan", Column.of(1.0, Double.NaN, -2.5)) + .set("with_infinities", Column.of(Float.POSITIVE_INFINITY, 1f, Float.NEGATIVE_INFINITY)) + .set("flag", Column.of(true, false, true)).set("name", Column.of("é", "", "a b")) + .set("position", Column.vectors(3, 1, 2, 3, 4, 5, 6, 7, 8, 9)) + .set("column_int8", new Column(ComponentType.INT8, new byte[] {1, 2, -3})) + .set("column_uint8", new Column(ComponentType.UINT8, new byte[] {1, 2, (byte) 255})) + .set("column_int16", new Column(ComponentType.INT16, new short[] {1, 2, 3})) + .set("column_uint16", new Column(ComponentType.UINT16, new short[] {1, 2, (short) 65535})) + .set("column_uint32", new Column(ComponentType.UINT32, new int[] {1, 2, -1})) + .set("column_uint64", new Column(ComponentType.UINT64, new long[] {1, 2, -1})); + Variable length = new Variable("m", "length", "extensive", "state"); + length.description = "a length"; + length.minimum = 0.0; + length.references.add("a reference"); + organs.declare("with_nan", length); + DataStructure plants = new DataStructure("plants"); + plants.entitySets.add(organs); + plants.origin = new double[] {1, 0, 0, 0.5, 0, 1, 0, 0, 0, 0, 1, -0.1, 0, 0, 0, 1}; + plants.extras.putArray("platform").add(1).add(2); + Frame frame = new Frame(); + frame.dataStructures.add(plants); + frame.time = 2.5; + frame.timeUnit = "h"; + frame.step = 3; + + Frame readBack = Codec.fromBytes(Codec.toBytes(frame)); + EntitySet set = readBack.dataStructure("plants").entitySet("Organ"); + assertEquals(ComponentType.INT64, set.column("id").type); // ids are INT64 + assertArrayEquals(new double[] {1.0, Double.NaN, -2.5}, set.column("with_nan").doubles()); + assertArrayEquals(new float[] {Float.POSITIVE_INFINITY, 1f, Float.NEGATIVE_INFINITY}, set.column("with_infinities").floats()); + assertArrayEquals(new boolean[] {true, false, true}, set.column("flag").booleans()); + assertArrayEquals(new String[] {"é", "", "a b"}, set.column("name").strings()); + assertEquals(3, set.column("position").width); + assertArrayEquals(new double[] {1, 2, 3, 4, 5, 6, 7, 8, 9}, set.column("position").doubles()); + assertArrayEquals(new String[] {"Leaf", "Internode", "Leaf"}, set.column("label").strings()); + for (String name : List.of("column_int8", "column_uint8", "column_int16", "column_uint16", "column_uint32", "column_uint64")) + assertEquals(organs.column(name), set.column(name), name); + assertEquals(255.0, set.column("column_uint8").doubleAt(2)); + assertEquals(length, set.variables.get("with_nan")); + assertArrayEquals(plants.origin, readBack.dataStructure("plants").origin); + assertEquals(plants.extras, readBack.dataStructure("plants").extras); + assertEquals(2.5, readBack.time); + assertEquals("h", readBack.timeUnit); + assertEquals(3, readBack.step); + } + + @Test + void meshes() { + EntitySet segments = new EntitySet("SubOrgan", 6L); + segments.set("id", Column.of(10L, 11L)); + Mesh mesh = new Mesh("roots", new double[] {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1}); + mesh.featureRows.put("SubOrgan", new long[] {0, 1, 1, -1}); + mesh.indices = new long[] {0, 1, 2, 0, 2, 3}; + mesh.display.put("radius", new float[] {1, 2, 2, 0}); + mesh.displaySources.put("radius", "SubOrgan"); + DataStructure roots = new DataStructure("roots"); + roots.entitySets.add(segments); + roots.meshes.add(mesh); + Frame frame = new Frame(); + frame.dataStructures.add(roots); + Mesh readBack = Codec.fromBytes(Codec.toBytes(frame)).dataStructures.get(0).meshes.get(0); + assertArrayEquals(mesh.positions, readBack.positions); + assertArrayEquals(mesh.featureRows.get("SubOrgan"), readBack.featureRows.get("SubOrgan")); + assertArrayEquals(mesh.indices, readBack.indices); + assertArrayEquals(mesh.display.get("radius"), readBack.display.get("radius")); + assertEquals(mesh.displaySources, readBack.displaySources); + } + + static Frame frameWith(EntitySet set) { + DataStructure plants = new DataStructure("plants"); + plants.entitySets.add(set); + Frame frame = new Frame(); + frame.dataStructures.add(plants); + return frame; + } + + @Test + void refused() { + EntitySet badColumn = new EntitySet("Organ"); + badColumn.set("id", Column.of(0L)).set("1st", Column.of(1.0)); + assertThrows(FspmTfException.class, () -> Codec.toBytes(frameWith(badColumn))); + Frame badId = new Frame(); + badId.dataStructures.add(new DataStructure("bad id")); + assertThrows(FspmTfException.class, () -> Codec.toBytes(badId)); + EntitySet badEdge = new EntitySet("Organ"); + badEdge.set("id", Column.of(0L)).set("parent", Column.of(-1L)).set("edge_type", Column.of("root")); + assertThrows(FspmTfException.class, () -> Codec.toBytes(frameWith(badEdge))); + EntitySet both = new EntitySet("Organ"); + both.set("id", Column.of(0L, 1L)).set("x", Column.of(Double.NaN, -Double.MAX_VALUE)); + assertThrows(FspmTfException.class, () -> Codec.toBytes(frameWith(both))); + EntitySet lengths = new EntitySet("Organ"); + lengths.set("id", Column.of(0L, 1L)).set("x", Column.of(1.0)); + assertThrows(FspmTfException.class, () -> Codec.toBytes(frameWith(lengths))); + assertThrows(FspmTfException.class, () -> Glb.unpack(new byte[] {1, 2, 3})); + assertThrows(FspmTfException.class, () -> new Column(ComponentType.INT64, new double[] {1.0})); + } + + @Test + void sequences(@TempDir Path folder) throws IOException { + List frames = new ArrayList<>(); + for (int step = 0; step < 3; step++) { + EntitySet plant = new EntitySet("Plant", 1L); + plant.set("id", Column.of(0L)).set("biomass", Column.of((double) step)); + Frame frame = frameWith(plant); + frame.time = step / 24.0; + frame.step = step; + frames.add(frame); + } + Path index = Sequences.writeSequence(folder, frames); + assertEquals(2, Sequences.readIndex(index).frames().get(2).step()); + List readBack = Sequences.readSequence(index); + assertEquals(2.0, readBack.get(2).dataStructures.get(0).entitySets.get(0).column("biomass").doubles()[0]); + for (Sequences.Entry entry : Sequences.readIndex(CONFORMANCE.resolve("growing_sequence").resolve(Sequences.INDEX_FILE)).frames()) + assertTrue(Files.exists(entry.path())); + } +} diff --git a/doc/index.md b/doc/index.md index e0d8852..5cd94e6 100644 --- a/doc/index.md +++ b/doc/index.md @@ -35,6 +35,7 @@ Reading and writing in Python A notebook: fspm-TF in Python Reading and writing in Julia Reading and writing in C++, and CPlantBox +Reading and writing in Java, and GroIMP ``` ## Implementing fspm-TF diff --git a/doc/java.md b/doc/java.md new file mode 100644 index 0000000..db3e973 --- /dev/null +++ b/doc/java.md @@ -0,0 +1,52 @@ +# Reading and writing in Java, and GroIMP + +`bindings/java/` holds the Java codec (Java 17, Maven; one dependency, Jackson). It reads and writes fspm-TF files and +passes the conformance check. In `bindings/java/groimp/` is GroIMP's adapter. Both have tests, run in +`.github/workflows/java-codec.yml`. + +## The format's own tables + +```java +import org.openalea.fspmtf.*; + +Frame frame = Codec.read(Path.of("scene.glb")); +EntitySet segments = frame.dataStructure("RootPopulation").entitySet("SubOrgan"); +double[] length = segments.column("length").doubles(); // one value per segment +String unit = segments.variables.get("length").unit; // its declared unit, e.g. "m" + +EntitySet organs = new EntitySet("Organ", 5L); +organs.set("id", Column.of(1L, 2L)).set("parent", Column.of(-1L, 1L)) + .set("edge_type", Column.of("none", "branch")).set("length", Column.of(0.12, 0.05)); +organs.declare("length", new Variable("m", "length", "extensive", "state")); +DataStructure plants = new DataStructure("plants"); +plants.entitySets.add(organs); +Frame written = new Frame(); +written.dataStructures.add(plants); +Codec.write(Path.of("plants.glb"), written); +``` + +A `Column` holds a Java array of its type (`long[]`, `double[]`, `boolean[]`, `String[]`, …; unsigned integers in +the array of their width), with a `width` of 2 or more for a fixed-size vector per row. NaN and infinities are written +with the specification's stand-ins and read back as they were. + +## GroIMP + +`GroImp.toDataStructure(root, options)` writes the graph below a GroIMP root: + +| fspm-TF | GroIMP | +|---|---| +| one entity set per scale | the nodes at one refinement depth (a successor or branch edge keeps the scale, a refinement edge adds one) | +| `id`, `label` | `Node.getId()`, the node's class's simple name | +| `complex` | the source of its incoming refinement edge, else its parent's complex | +| `parent`, `edge_type` | the source of its incoming successor (`successor`) or branch (`branch`) edge at the same scale | +| relations | the other edge bits (containment, mark, notifies, user edges), one relation per bit and pair of sets | +| variables | the managed fields of the model's classes (numbers, booleans, text; vecmath tuples as vectors) | + +`options.scaleNames` and `options.scaleRanks` give each scale its entity set's name and rank, e.g. the vocabulary's +`Organ`, rank 5. `options.variables` declares the fields' units and standard names. GroIMP's fields have no units. +`GroImp.toGraph(dataStructure, label -> new Internode(), ids)` builds the nodes back: their fields, refinement, +successor and branch edges, and edge bits. It does this for any graph, e.g. metafspm's Plant, Axis and SubOrgan +become three scales linked by refinement edges. + +The adapter depends on GroIMP's Graph module only. It is meant to become a GroIMP plugin: an export filter with the +scene's geometry, and an import filter. diff --git a/doc/platforms.md b/doc/platforms.md index bc4e9b4..9a9a32f 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -27,7 +27,7 @@ What it has to handle: This is a few hundred lines in any language. Existing libraries help: - **C++:** done, `bindings/cpp/` ([C++ and CPlantBox](cpp.md)): hand-written, with nlohmann_json. -- **Java:** JglTF has its JSON classes. +- **Java:** done, `bindings/java/` ([Java and GroIMP](java.md)): hand-written, with Jackson. - **Julia:** done, `bindings/julia/FspmTF/` ([FspmTF.jl](julia.md)): a hand-written GLB container and JSON.jl. Each codec goes in its own folder, with its usual build files and tests: diff --git a/implementations.md b/implementations.md index 69b768f..1efb03e 100644 --- a/implementations.md +++ b/implementations.md @@ -6,7 +6,8 @@ | openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.1 | | fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.1 | | CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.1 | -| GroIMP | Java | | | | | +| fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.1 | +| GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.1 | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.1 | | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.1 | | VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.1 | From f986f282cc0720ca343855213c15d5b7c11cd619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 15:19:46 +0100 Subject: [PATCH 09/39] Each binding under its platform's licence: bindings/cpp GPL-3.0 (CPlantBox), bindings/java GPL-3.0-or-later (GroIMP); the rest of fspm-TF stays CeCILL-C --- bindings/cpp/CMakeLists.txt | 3 + bindings/cpp/LICENSE | 674 +++++++++++++++++++++++++++++++++++ bindings/cpp/README.md | 2 + bindings/java/LICENSE | 674 +++++++++++++++++++++++++++++++++++ bindings/java/README.md | 3 +- bindings/java/groimp/pom.xml | 6 + bindings/java/pom.xml | 7 + doc/cpp.md | 2 + doc/java.md | 2 + 9 files changed, 1372 insertions(+), 1 deletion(-) create mode 100644 bindings/cpp/LICENSE create mode 100644 bindings/java/LICENSE diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index b2f8ada..78a6148 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -2,6 +2,8 @@ # CPlantBox is given (FSPM_TF_CPLANTBOX_SOURCE_DIR and FSPM_TF_CPLANTBOX_LIBRARY), CPlantBox's adapter. # # cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build +# +# Licence: GPL-3.0 (LICENSE), as CPlantBox; the rest of fspm-TF is CeCILL-C. cmake_minimum_required(VERSION 3.16) project(fspm_tf VERSION 0.1.0 LANGUAGES CXX) @@ -51,6 +53,7 @@ install(TARGETS fspm_tf EXPORT fspm_tfTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/fspm_tf) install(EXPORT fspm_tfTargets NAMESPACE fspm_tf:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) configure_package_config_file(cmake/fspm_tfConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfig.cmake INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) diff --git a/bindings/cpp/LICENSE b/bindings/cpp/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/bindings/cpp/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 477ad47..2cc9b8b 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -18,3 +18,5 @@ cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build Its one dependency is nlohmann_json (header-only, MIT), found installed or fetched. It installs as a CMake package: `find_package(fspm_tf)`, then `target_link_libraries(... fspm_tf::fspm_tf)`. See [the documentation](../../doc/cpp.md). + +Licence: GPL-3.0 (`LICENSE`), as CPlantBox; the rest of fspm-TF is CeCILL-C. diff --git a/bindings/java/LICENSE b/bindings/java/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/bindings/java/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/bindings/java/README.md b/bindings/java/README.md index 7065f76..6546d9b 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -13,4 +13,5 @@ Jackson. GroIMP's adapter is in `groimp/`. mvn -f bindings/java/pom.xml verify ``` -See [the documentation](../../doc/java.md). +Licence: GPL-3.0-or-later (`LICENSE`), as GroIMP; the rest of fspm-TF is CeCILL-C. See +[the documentation](../../doc/java.md). diff --git a/bindings/java/groimp/pom.xml b/bindings/java/groimp/pom.xml index ffe5842..a8ee495 100644 --- a/bindings/java/groimp/pom.xml +++ b/bindings/java/groimp/pom.xml @@ -15,6 +15,12 @@ 0.1.0 jar fspm-TF, GroIMP's adapter + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + + 21 diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml index 8df2552..c8f763f 100644 --- a/bindings/java/pom.xml +++ b/bindings/java/pom.xml @@ -14,6 +14,13 @@ fspm-TF (Java) Reading and writing fspm-TF files: functional-structural plant model scenes as glTF 2.0 binaries https://github.com/openalea/fspm-TF + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + As GroIMP, the platform this codec serves (bindings/java/LICENSE); the rest of fspm-TF is CeCILL-C + + 17 diff --git a/doc/cpp.md b/doc/cpp.md index 6fa1332..bb5dd14 100644 --- a/doc/cpp.md +++ b/doc/cpp.md @@ -4,6 +4,8 @@ `bindings/cpp/cplantbox/` is CPlantBox's adapter. Both have tests, run in `.github/workflows/cpp-codec.yml` on Linux, macOS and Windows. +Licence: GPL-3.0, as CPlantBox (the rest of fspm-TF is CeCILL-C). + ## Building ```bash diff --git a/doc/java.md b/doc/java.md index db3e973..1936d29 100644 --- a/doc/java.md +++ b/doc/java.md @@ -4,6 +4,8 @@ passes the conformance check. In `bindings/java/groimp/` is GroIMP's adapter. Both have tests, run in `.github/workflows/java-codec.yml`. +Licence: GPL-3.0-or-later, as GroIMP (the rest of fspm-TF is CeCILL-C). + ## The format's own tables ```java From 70f94793b25e6600ca6fb251ffcc91a1668c1876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 18:59:47 +0100 Subject: [PATCH 10/39] Specification 0.2 (draft, additive): from the bindings' points as the user decided them: missing values in integer, text and boolean columns (UINT8 0/1 with noData 255); an entity set's scale (a base scale or mixed) and ranks as base scale positions; reading by content (standard names, the complex chain); position columns in their declared length unit; a data structure's labels map; the extras key, the platform's name in lower case; unique relation names, bearer relations' names and labels; edge_type unknown; the order column; parents in another complex; date formats; sparse ids; conformance/ as part of the agreement and the sources' precedence. Vocabulary 0.2 (Seed, mtg_index, the new enumerations); the reference codec, validator (each new rule) and ParaView reader; the conformance files regenerated with a new scene, missing_values --- CHANGELOG.md | 12 +- README.md | 2 +- .../python/src/openalea/fspm_tf/__init__.py | 2 +- bindings/python/src/openalea/fspm_tf/codec.py | 80 ++++- bindings/python/src/openalea/fspm_tf/dump.py | 17 +- bindings/python/src/openalea/fspm_tf/model.py | 37 +- .../python/src/openalea/fspm_tf/validate.py | 62 +++- bindings/python/test/test_codec.py | 8 +- bindings/python/test/test_compare.py | 2 +- bindings/python/test/test_paraview_reader.py | 11 + bindings/python/test/test_validate.py | 47 +++ conformance/README.md | 14 +- conformance/adaptive_soil.glb | Bin 2396 -> 2428 bytes conformance/generate.py | 59 ++- conformance/growing_sequence/frame_000000.glb | Bin 12860 -> 12892 bytes conformance/growing_sequence/frame_000001.glb | Bin 14820 -> 14852 bytes conformance/growing_sequence/frame_000002.glb | Bin 17732 -> 17764 bytes conformance/growing_sequence/scene.fspm.json | 2 +- conformance/missing_values.expected.json | 337 ++++++++++++++++++ conformance/missing_values.glb | Bin 0 -> 6972 bytes conformance/parameter_set.glb | Bin 3460 -> 3484 bytes conformance/root_system.glb | Bin 15788 -> 15820 bytes conformance/section_cells.glb | Bin 5260 -> 5284 bytes conformance/soil_and_roots.glb | Bin 18324 -> 18356 bytes conformance/wheat_shoot.glb | Bin 6924 -> 6956 bytes doc/governance.md | 5 +- implementations.md | 4 +- paraview/fspm_tf_reader.py | 7 +- spec/schema/fspm_scene.schema.json | 10 +- spec/schema/index.schema.json | 4 +- spec/specification.md | 120 +++++-- spec/vocabulary/enums.json | 13 +- spec/vocabulary/labels.csv | 1 + spec/vocabulary/standard_names.csv | 1 + 34 files changed, 777 insertions(+), 80 deletions(-) create mode 100644 conformance/missing_values.expected.json create mode 100644 conformance/missing_values.glb diff --git a/CHANGELOG.md b/CHANGELOG.md index 7073796..b64b61b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ # Changelog -## Unreleased: specification 0.1 (draft) +## Unreleased: specification 0.2 (draft) + +- Specification 0.2, additive (0.1 files stay valid), from the bindings' points (`spec/specification.md`, "Changes + from 0.1"): missing values in integer, text and boolean columns; an entity set's `scale` and ranks as base scale + positions; reading by content; position columns in their declared unit; a data structure's `labels`; the extras + key; `edge_type` `unknown`; the `order` column; parents in another complex; bearer relations' labels and names; + dates (`format`); sparse ids; the conformance files' status and the sources' precedence. +- Vocabulary 0.2: `Seed`, `mtg_index`, the new enumerations. +- The reference codec, the validator, the ParaView reader and the conformance files (a new scene, `missing_values`). + +## Specification 0.1 (draft) - The specification (`spec/specification.md`), its JSON Schemas and the first vocabulary (41 standard names, the labels with their Plant Ontology terms, the enumerations, a column of names per platform). diff --git a/README.md b/README.md index c329976..f3fbe65 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ An exchange format for whole functional-structural plant model (FSPM) scenes, ac Each time step is one glTF file, and the variables share one vocabulary. It also aims to serve as an in-memory interface for coupling components across platforms. -Status: specification 0.1, draft. +Status: specification 0.2, draft (0.1 files stay valid). - [The documentation](doc/index.md): for users (viewing, checking, Python) and for platform developers (implementing a codec, conformance, changing the format), built with Sphinx (`.readthedocs.yml`; locally: diff --git a/bindings/python/src/openalea/fspm_tf/__init__.py b/bindings/python/src/openalea/fspm_tf/__init__.py index bbc30cf..49a90b7 100644 --- a/bindings/python/src/openalea/fspm_tf/__init__.py +++ b/bindings/python/src/openalea/fspm_tf/__init__.py @@ -3,7 +3,7 @@ (openalea.fspm_tf.codec), the validator (openalea.fspm_tf.validate), the display copies (openalea.fspm_tf.display) and the vocabulary (openalea.fspm_tf.vocabulary). See the specification in the repository's spec/ folder. """ -SPECIFICATION_VERSION = "0.1" +SPECIFICATION_VERSION = "0.2" from openalea.fspm_tf.model import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, # noqa: E402 Table, Variable) diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/fspm_tf/codec.py index 4badc61..7d5d950 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/fspm_tf/codec.py @@ -10,6 +10,7 @@ IDs, and the FSPM_scene root extension holds what these two cannot: data structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, standard names, kinds and roles. """ +import itertools import json import os import re @@ -80,10 +81,13 @@ def add_table(table: Table, class_id: str) -> int: if getattr(data_structure, key) is not None: entry[key] = getattr(data_structure, key) entry["origin"] = data_structure.origin.T.reshape(-1).tolist() # column-major, as glTF matrices - entry["entitySets"] = [ - {"name": entity_set.name, "rank": entity_set.rank, - "propertyTable": add_table(entity_set, f"{data_structure.id}__{entity_set.name}")} - for entity_set in data_structure.entity_sets] + entry["entitySets"] = [] + for entity_set in data_structure.entity_sets: + set_entry = {"name": entity_set.name, "rank": entity_set.rank} + if entity_set.scale is not None: + set_entry["scale"] = entity_set.scale + set_entry["propertyTable"] = add_table(entity_set, f"{data_structure.id}__{entity_set.name}") + entry["entitySets"].append(set_entry) if data_structure.relations: entry["relations"] = [ {"name": relation.name, "sourceSet": relation.source_set, "targetSet": relation.target_set, @@ -93,6 +97,8 @@ def add_table(table: Table, class_id: str) -> int: entry["scalars"] = add_table(data_structure.scalars, f"{data_structure.id}__scalars") if data_structure.grid is not None: entry["grid"] = _grid_json(data_structure.grid) + if data_structure.labels: + entry["labels"] = dict(data_structure.labels) if data_structure.extras: entry["extras"] = data_structure.extras node = {"name": data_structure.id} @@ -174,6 +180,8 @@ def _declaration_json(variable: Variable) -> dict: for key in ("references", "doi"): if getattr(variable, key): declaration[key] = list(getattr(variable, key)) + if variable.format is not None: + declaration["format"] = variable.format return declaration @@ -192,6 +200,10 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d values[name] = {"values": buffers.add(codes)} elif column.dtype == object: prop["type"] = "STRING" + if any(value is None for value in column): # missing text: noData, a string the column lacks + no_data = _free_string(column) + prop["noData"] = no_data + column = np.array([no_data if value is None else value for value in column], dtype=object) encoded = [value.encode("utf-8") for value in column] offsets = np.concatenate([[0], np.cumsum([len(value) for value in encoded])]) offset_type = "UINT32" if offsets[-1] < 2 ** 32 else "UINT64" @@ -201,8 +213,15 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d elif column.dtype == bool: if column.ndim != 1: raise ValueError(f"column '{name}' of '{table.name}': boolean vectors are not supported") - prop["type"] = "BOOLEAN" - values[name] = {"values": buffers.add(np.packbits(column, bitorder="little"))} + if np.ma.isMaskedArray(column): # booleans with missing values: UINT8 0, 1, 255 + prop.update(type="SCALAR", componentType="UINT8", noData=BOOLEAN_NO_DATA, + extras={"FSPM_boolean": True}) + stored = np.where(np.ma.getmaskarray(column), BOOLEAN_NO_DATA, + np.asarray(column.data, dtype=np.uint8)).astype(np.uint8) + values[name] = {"values": buffers.add(stored)} + else: + prop["type"] = "BOOLEAN" + values[name] = {"values": buffers.add(np.packbits(column, bitorder="little"))} else: if column.dtype not in COMPONENT_TYPES: raise TypeError(f"column '{name}' of '{table.name}': dtype {column.dtype} is not supported") @@ -231,13 +250,43 @@ def _encode_table(table: Table, class_id: str, buffers: BufferBuilder, schema: d data = np.where(np.isnan(column), no_data, column).astype(column.dtype) elif name in ("complex", "parent") and name in table.reserved: prop["noData"] = NO_ID - values[name] = {"values": buffers.add(data)} + elif np.ma.isMaskedArray(column): # missing integers: noData, a value the column lacks + no_data = _free_integer(column) + prop["noData"] = no_data + data = np.where(np.ma.getmaskarray(column), no_data, np.asarray(column.data)).astype(column.dtype) + values[name] = {"values": buffers.add(np.asarray(data))} if name == "id" and name in table.reserved and table.table_kind == "entity_set": prop["required"] = True properties[name] = prop return properties, values +BOOLEAN_NO_DATA = 255 # a boolean column with missing values: UINT8 0, 1, and 255 missing + + +def _free_integer(column: np.ndarray) -> int: + """A value of the column's integer type that it does not hold: its minimum (signed) or maximum (unsigned) first.""" + info = np.iinfo(column.dtype) + present = set(np.asarray(column.compressed() if np.ma.isMaskedArray(column) else column).reshape(-1).tolist()) + first, second = (info.min, info.max) if info.min < 0 else (info.max, info.min) + for candidate in (first, second): + if candidate not in present: + return int(candidate) + step = 1 if info.min < 0 else -1 # then inwards from the first: at most len(present) tries + for candidate in itertools.count(first + step, step): + if candidate not in present: + return int(candidate) + + +def _free_string(column: np.ndarray) -> str: + """A string the column does not hold: the empty string first, then NUL characters.""" + present = {value for value in column if value is not None} + candidate = "" + while candidate in present: + candidate += "\0" + return candidate + + def _enum_codes(name: str, column: np.ndarray, class_id: str, schema: dict) -> tuple: if name == "edge_type": unknown = set(column) - set(EDGE_TYPES) @@ -335,9 +384,10 @@ def table(index: int, kind, **fields): for entry in scene.get("dataStructures", []): data_structure = DataStructure( id=entry["id"], kind=entry.get("kind", "graph"), platform=entry.get("platform"), - model=entry.get("model"), extras=entry.get("extras", {}), + model=entry.get("model"), extras=entry.get("extras", {}), labels=entry.get("labels", {}), origin=np.asarray(entry.get("origin", np.eye(4).reshape(-1)), dtype=float).reshape(4, 4).T) - data_structure.entity_sets = [table(item["propertyTable"], EntitySet, rank=item.get("rank")) + data_structure.entity_sets = [table(item["propertyTable"], EntitySet, rank=item.get("rank"), + scale=item.get("scale")) for item in entry.get("entitySets", [])] data_structure.relations = [table(item["propertyTable"], Relation, source_set=item["sourceSet"], target_set=item["targetSet"]) for item in entry.get("relations", [])] @@ -391,6 +441,8 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, count=count + 1) column = np.array([raw[offsets[i]:offsets[i + 1]].decode("utf-8") for i in range(count)], dtype=object) + if "noData" in prop: # missing text + column = np.array([None if value == prop["noData"] else value for value in column], dtype=object) elif prop["type"] == "BOOLEAN": column = np.unpackbits(np.frombuffer(raw, dtype=np.uint8), bitorder="little", count=count).astype(bool) elif prop["type"] == "SCALAR": @@ -408,6 +460,13 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict column = np.where(column == np.asarray(infinity["positive"], dtype=dtype), np.inf, np.where(column == np.asarray(infinity["negative"], dtype=dtype), -np.inf, column) ).astype(dtype) + if prop.get("extras", {}).get("FSPM_boolean"): # booleans with missing values + missing = column == prop.get("noData", BOOLEAN_NO_DATA) + column = np.ma.MaskedArray(column == 1, mask=missing) if missing.any() else column == 1 + elif dtype.kind in "iu" and "noData" in prop and name not in ("complex", "parent"): # missing integers + missing = column == prop["noData"] + if missing.any(): + column = np.ma.MaskedArray(column, mask=missing) else: raise ValueError(f"'{class_id}.{name}': property type {prop['type']} is not part of fspm-TF 0.1") columns[name] = column @@ -418,7 +477,8 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict unit=declared.get("unit", "1"), standard_name=declared.get("standardName"), kind=declared.get("kind"), role=declared.get("role", "state"), description=prop.get("description"), minimum=declared.get("minimum"), maximum=declared.get("maximum"), - references=tuple(declared.get("references", ())), doi=tuple(declared.get("doi", ()))) + references=tuple(declared.get("references", ())), doi=tuple(declared.get("doi", ())), + format=declared.get("format")) return columns, declarations diff --git a/bindings/python/src/openalea/fspm_tf/dump.py b/bindings/python/src/openalea/fspm_tf/dump.py index fcda7f6..776f6b2 100644 --- a/bindings/python/src/openalea/fspm_tf/dump.py +++ b/bindings/python/src/openalea/fspm_tf/dump.py @@ -3,8 +3,8 @@ conformance files' expected values (``*.expected.json``) are written with it, so that a reader in any language can compare what it reads with them:: - frame_to_dict(frame) -> dict NaN as null, ±∞ as "Infinity" / "-Infinity"; floats as the shortest decimal - giving the same double + frame_to_dict(frame) -> dict NaN and missing values as null, ±∞ as "Infinity" / "-Infinity"; floats as + the shortest decimal giving the same double """ import math @@ -26,6 +26,12 @@ def _value(value): def _column(values: np.ndarray): + if np.ma.isMaskedArray(values): # missing values: null + missing = np.ma.getmaskarray(values) + if values.ndim == 2: + return [[None if gone else _value(item) for item, gone in zip(row, row_missing)] + for row, row_missing in zip(values.data, missing)] + return [None if gone else _value(item) for item, gone in zip(values.data, missing)] if values.ndim == 2: return [[_value(item) for item in row] for row in values] return [_value(item) for item in values] @@ -44,6 +50,8 @@ def _table(table: Table) -> dict: for key in ("references", "doi"): if getattr(variable, key): declaration[key] = list(getattr(variable, key)) + if variable.format is not None: + declaration["format"] = variable.format declarations[name] = declaration if declarations: entry["variables"] = declarations @@ -55,13 +63,16 @@ def frame_to_dict(frame: Frame) -> dict: for data_structure in frame.data_structures: entry = {"id": data_structure.id, "kind": data_structure.kind, "origin": data_structure.origin.tolist(), - "entity_sets": [dict(_table(entity_set), rank=entity_set.rank) + "entity_sets": [dict(_table(entity_set), rank=entity_set.rank, + **({"scale": entity_set.scale} if entity_set.scale is not None else {})) for entity_set in data_structure.entity_sets]} for key in ("platform", "model"): if getattr(data_structure, key) is not None: entry[key] = getattr(data_structure, key) if data_structure.extras: entry["extras"] = data_structure.extras + if data_structure.labels: + entry["labels"] = dict(data_structure.labels) if data_structure.relations: entry["relations"] = [dict(_table(relation), source_set=relation.source_set, target_set=relation.target_set) for relation in data_structure.relations] diff --git a/bindings/python/src/openalea/fspm_tf/model.py b/bindings/python/src/openalea/fspm_tf/model.py index b4aa0c2..c5a78d6 100644 --- a/bindings/python/src/openalea/fspm_tf/model.py +++ b/bindings/python/src/openalea/fspm_tf/model.py @@ -12,6 +12,9 @@ Every table holds columns of equal length (its rows), and declarations (Variable) for its variables. The columns reserved by the specification (RESERVED) are the structure: ids, the MTG relations, a relation's or a mapping's ends. + +Missing values: NaN in float columns; elsewhere a numpy masked array (np.ma, the masked rows missing) for integers and +booleans, None in a text column. """ from dataclasses import dataclass, field from typing import Optional @@ -22,13 +25,16 @@ # Columns whose meaning the specification fixes, per kind of table RESERVED = { - "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index"), + "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index", "order"), "relation": ("id", "source", "target", "relation"), "mapping": ("source", "target", "weight"), "scalars": (), } -ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index") # reserved columns of ids, int64 -EDGE_TYPES = ("none", "successor", "branch") # MTG: root of its scale, '<', '+' +ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "order") # reserved, int64 +EDGE_TYPES = ("none", "successor", "branch", "unknown") # MTG: root of its scale, '<', '+'; a link not known +BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", + "Connection") # an entity set's scale: one of these, or "mixed" +FORMATS = ("date", "date-time", "time") # a text variable holding ISO 8601 dates or times ENUM_COLUMNS = ("label", "edge_type", "relation") DATA_STRUCTURE_KINDS = ("graph", "regular_grid", "adaptive_grid", "table") ROLES = ("state", "input", "parameter", "derived", "setting") @@ -37,7 +43,8 @@ @dataclass class Variable: """The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its - kind (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance.""" + kind (extensive, intensive, ...: how it maps between scales and grids), its role and its provenance; the format + of a text column holding dates or times ("date", "date-time", "time").""" unit: str = "1" standard_name: Optional[str] = None kind: Optional[str] = None @@ -47,6 +54,7 @@ class Variable: maximum: Optional[float] = None references: tuple = () doi: tuple = () + format: Optional[str] = None @dataclass @@ -64,6 +72,8 @@ def __post_init__(self): for name in set(self.columns) & set(ID_COLUMNS) & set(self.reserved): # ids are int64, whatever the platform's if self.columns[name].dtype.kind in "iu": # default integer self.columns[name] = self.columns[name].astype(np.int64) + if np.ma.isMaskedArray(self.columns[name]): + raise ValueError(f"table '{self.name}': the reserved column '{name}' has no missing values") counts = {name: len(values) for name, values in self.columns.items()} if len(set(counts.values())) > 1: raise ValueError(f"table '{self.name}': columns of different lengths {counts}") @@ -88,8 +98,11 @@ class EntitySet(Table): """The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (rank None). Columns: id (int64, stable across frames, never reused), label (str), complex (the id of the containing entity at the coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), edge_type - (str: 'none', 'successor', 'branch'); an adaptive grid's cells: level and index; then the variables.""" + (str: 'none', 'successor', 'branch', 'unknown'); order (int64, optional: its position among its tree parent's + children); an adaptive grid's cells: level and index; then the variables. *scale*: its base scale + (BASE_SCALES), or "mixed".""" rank: Optional[int] = None + scale: Optional[str] = None table_kind = "entity_set" @@ -154,7 +167,7 @@ def __post_init__(self): class DataStructure: """One plant population's graph, or one environment model's grid. *origin*: its frame in the scene (4×4). *extras*: JSON values for the writing platform (e.g. what it needs to rebuild its own data structure); other - readers keep them without interpreting them.""" + readers keep them without interpreting them. *labels*: {the platform's label: the vocabulary's label}.""" id: str kind: str = "graph" entity_sets: list = field(default_factory=list) @@ -166,6 +179,7 @@ class DataStructure: platform: Optional[str] = None model: Optional[str] = None extras: dict = field(default_factory=dict) + labels: dict = field(default_factory=dict) def __post_init__(self): self.origin = np.asarray(self.origin, dtype=np.float64).reshape(4, 4) @@ -209,13 +223,18 @@ def data_structure(self, id: str) -> DataStructure: def _as_column(name: str, values) -> np.ndarray: - array = np.asarray(values) + if np.ma.isMaskedArray(values): # integers or booleans with missing values + array = values if np.ma.getmaskarray(values).any() else np.asarray(values.data) + else: + array = np.asarray(values) if array.ndim == 0: array = array.reshape(1) if array.dtype.kind in "OU": - if not all(isinstance(value, str) for value in array.reshape(-1)): - raise TypeError(f"column '{name}': object columns must hold strings") + if not all(isinstance(value, str) or value is None for value in array.reshape(-1)): + raise TypeError(f"column '{name}': object columns must hold strings (None: missing)") array = array.astype(object) + elif np.ma.isMaskedArray(array) and array.dtype.kind == "f": + array = array.filled(np.nan) # missing floats are NaN if array.ndim > 2: raise ValueError(f"column '{name}': one value or one vector per row (got shape {array.shape})") return array diff --git a/bindings/python/src/openalea/fspm_tf/validate.py b/bindings/python/src/openalea/fspm_tf/validate.py index 921b6fb..8036694 100644 --- a/bindings/python/src/openalea/fspm_tf/validate.py +++ b/bindings/python/src/openalea/fspm_tf/validate.py @@ -9,6 +9,7 @@ is likely wrong or incomplete). """ from dataclasses import dataclass, field +import datetime import json import os @@ -19,7 +20,7 @@ from openalea.fspm_tf.codec import (DTYPES, EDGE_TYPE_ENUM, EXTENSION, FEATURES, METADATA, Z_UP_TO_Y_UP, from_bytes) from openalea.fspm_tf.glb import unpack -from openalea.fspm_tf.model import EDGE_TYPES, NO_ID +from openalea.fspm_tf.model import BASE_SCALES, EDGE_TYPES, NO_ID @dataclass @@ -157,6 +158,9 @@ def length(key): return views[index]["byteLength"] kind = prop.get("type") + if prop.get("extras", {}).get("FSPM_boolean") and (kind != "SCALAR" or prop.get("componentType") != "UINT8" + or prop.get("noData") != 255 or prop.get("array")): + report.error(where, "a boolean column with missing values is UINT8 (0, 1) with noData 255") values_length = length("values") if values_length is None: return @@ -216,6 +220,19 @@ def _check_frame(frame, document: dict, report: Report) -> None: else: _check_references(f"{where}/{relation.name}/{end}", relation.columns[end], ids[set_name], report, allow_none=False) + names = [relation.name for relation in data_structure.relations] + if len(set(names)) != len(names): + report.error(where, "two relations have the same name") + for relation in data_structure.relations: + if relation.name.startswith("bearer") and "relation" in relation.columns: + unknown = set(relation.columns["relation"].tolist()) - set(enums["bearer_relation"]) + if unknown: + report.warning(f"{where}/{relation.name}/relation", + f"bearer labels {sorted(unknown)}: expected {sorted(enums['bearer_relation'])}") + _check_scales_and_labels(data_structure, report) + for key in data_structure.extras: + if key != key.lower(): + report.warning(f"{where}/extras", f"key '{key}': the platform's name, in lower case") if data_structure.grid is not None: _check_grid(data_structure, sets, report) elif data_structure.kind in ("regular_grid", "adaptive_grid"): @@ -323,6 +340,47 @@ def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: report.error(f"{where}/parent", "the parents form a cycle") +def _check_scales_and_labels(data_structure, report) -> None: + """Each entity set's scale and rank; its order column; the data structure's label map.""" + used = set() + for entity_set in data_structure.entity_sets: + where = f"{data_structure.id}/{entity_set.name}" + if entity_set.scale in BASE_SCALES and entity_set.rank is not None \ + and entity_set.rank != BASE_SCALES.index(entity_set.scale) + 1: + report.warning(where, f"rank {entity_set.rank}: the base scale {entity_set.scale}'s position is " + f"{BASE_SCALES.index(entity_set.scale) + 1}") + if "order" in entity_set.columns and (np.asarray(entity_set.columns["order"]) < 0).any(): + report.error(f"{where}/order", "positions among siblings are non-negative") + if "label" in entity_set.columns: + used |= {label for label in entity_set.columns["label"].tolist() if label is not None} + vocabulary_labels = vocabulary.labels() + for label, standard in data_structure.labels.items(): + if label not in used: + report.warning(f"{data_structure.id}/labels", f"'{label}' is no entity's label") + if standard not in vocabulary_labels: + report.warning(f"{data_structure.id}/labels", f"'{standard}' is not a label of the vocabulary " + f"{vocabulary.version()}") + + +def _check_format(where, values, variable_format, report) -> None: + """A text column's dates or times, as ISO 8601.""" + if np.asarray(values).dtype != object: + report.error(where, f"format '{variable_format}' is for text columns") + return + parse = {"date": datetime.date.fromisoformat, "date-time": datetime.datetime.fromisoformat, + "time": datetime.time.fromisoformat}[variable_format] + bad = [] + for value in values: + if value is None: + continue + try: + parse(value) + except ValueError: + bad.append(value) + if bad: + report.error(where, f"{len(bad)} value(s) not ISO 8601 {variable_format}, e.g. {bad[:3]}") + + def _check_grid(data_structure, sets: dict, report) -> None: grid, where = data_structure.grid, f"{data_structure.id}/grid" dimensions = len(grid.shape) @@ -369,6 +427,8 @@ def _check_variables(where, table, standard_names: dict, enums: dict, report) -> report.error(f"{where}/{name}", f"kind '{variable.kind}' is not one of {sorted(enums['kind'])}") if variable.role not in enums["role"]: report.error(f"{where}/{name}", f"role '{variable.role}' is not one of {sorted(enums['role'])}") + if variable.format is not None: + _check_format(f"{where}/{name}", table.columns[name], variable.format, report) if variable.standard_name is None: continue entry = standard_names.get(variable.standard_name) diff --git a/bindings/python/test/test_codec.py b/bindings/python/test/test_codec.py index d4d7749..15b4983 100644 --- a/bindings/python/test/test_codec.py +++ b/bindings/python/test/test_codec.py @@ -17,7 +17,11 @@ def same_columns(a, b): for name in a.columns: x, y = a.columns[name], b.columns[name] assert x.dtype == y.dtype and x.shape == y.shape, name - if x.dtype.kind == "f": + assert np.ma.isMaskedArray(x) == np.ma.isMaskedArray(y), name + if np.ma.isMaskedArray(x): # missing values: the same rows, the same others + assert np.array_equal(np.ma.getmaskarray(x), np.ma.getmaskarray(y)), name + assert np.array_equal(x.compressed(), y.compressed()), name + elif x.dtype.kind == "f": assert np.ascontiguousarray(x).tobytes() == np.ascontiguousarray(y).tobytes(), name # bit for bit else: assert np.array_equal(x, y), name @@ -36,6 +40,8 @@ def test_round_trip(name): for a, b in zip(written.entity_sets + written.relations, read.entity_sets + read.relations): same_columns(a, b) assert [entity_set.rank for entity_set in written.entity_sets] == [s.rank for s in read.entity_sets] + assert [entity_set.scale for entity_set in written.entity_sets] == [s.scale for s in read.entity_sets] + assert written.labels == read.labels if written.scalars is not None: same_columns(written.scalars, read.scalars) assert written.grid == read.grid diff --git a/bindings/python/test/test_compare.py b/bindings/python/test/test_compare.py index 8d4502f..15b8722 100644 --- a/bindings/python/test/test_compare.py +++ b/bindings/python/test/test_compare.py @@ -39,7 +39,7 @@ def test_compare_all(tmp_path): target.parent.mkdir(parents=True, exist_ok=True) shutil.copy(os.path.join(folder, name), target) results = compare_all(str(dumps), CONFORMANCE) - assert len(results) == 9 and not any(results.values()) + assert len(results) == 10 and not any(results.values()) assert main(["compare-all", str(dumps), CONFORMANCE]) == 0 with open(dumps / "root_system.json") as file: dump = json.load(file) diff --git a/bindings/python/test/test_paraview_reader.py b/bindings/python/test/test_paraview_reader.py index 0563251..7fcfaa6 100644 --- a/bindings/python/test/test_paraview_reader.py +++ b/bindings/python/test/test_paraview_reader.py @@ -114,3 +114,14 @@ def test_inside_paraview(): result = subprocess.run([pvpython, script, CONFORMANCE], capture_output=True, text=True, timeout=600) assert result.returncode == 0, result.stdout + result.stderr assert "fspm-TF reader: ok" in result.stdout + + +def test_missing_values_are_nan_and_code_minus_one(): + frame = generate.root_system() + segments = frame.data_structures[0].entity_set("SubOrgan") + count = segments.count + segments.columns["tissue"] = np.array([None] + ["cortex"] * (count - 1), dtype=object) + segments.columns["rank"] = np.ma.MaskedArray(np.arange(count, dtype=np.int32), mask=[True] + [False] * (count - 1)) + lines = blocks_of(frame)[("root_system", "SubOrgan segments")] + assert np.isnan(cell_array(lines, "SubOrgan.rank")[0]) and cell_array(lines, "SubOrgan.rank")[1] == 1. + assert cell_array(lines, "SubOrgan.tissue_code")[0] == -1 and cell_array(lines, "SubOrgan.tissue_code")[1] == 0 diff --git a/bindings/python/test/test_validate.py b/bindings/python/test/test_validate.py index d645ed2..d05a8f0 100644 --- a/bindings/python/test/test_validate.py +++ b/bindings/python/test/test_validate.py @@ -109,3 +109,50 @@ def test_index_order(tmp_path): with open(index, "w") as file: json.dump(content, file) assert "frames are in time order" in str(validate_sequence(index)) + + +def organs_of(frame): + return frame.data_structures[0].entity_set("Organ") + + +def test_missing_values_scene_is_valid(): + report = validate_bytes(to_bytes(generate.missing_values())) + assert report.ok and not report.warnings, str(report) + + +def test_rules_of_0_2(): + frame = generate.missing_values() + organs_of(frame).columns["emergence"][0] = "1 March 2026" + assert "1 value(s) not ISO 8601 date" in messages(frame) + frame = generate.missing_values() + organs_of(frame).columns["edge_type"][0] = "unknown" # an unknown link needs a parent + assert "'none' exactly for the entities without a parent" in messages(frame) + frame = generate.missing_values() + organs_of(frame).columns["order"][0] = -1 + assert "positions among siblings are non-negative" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].relations[1].name = "bearer_Axis_Organ" + assert "two relations have the same name" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].relations[0].columns["relation"][0] = "sideways" + assert "bearer labels ['sideways']" in messages(frame) + frame = generate.missing_values() + organs_of(frame).scale = "Organ" + organs_of(frame).rank = 3 + assert "rank 3: the base scale Organ's position is 5" in messages(frame) + frame = generate.missing_values() + frame.data_structures[0].labels["Petal"] = "Corolla" + text = messages(frame) + assert "'Petal' is no entity's label" in text and "'Corolla' is not a label of the vocabulary" in text + frame = generate.missing_values() + frame.data_structures[0].extras = {"MyPlatform": {}} + assert "key 'MyPlatform': the platform's name, in lower case" in messages(frame) + + +def test_boolean_with_missing_values_is_uint8(): + data = to_bytes(generate.missing_values()) + document, binary = unpack(data) + classes = document["extensions"]["EXT_structural_metadata"]["schema"]["classes"] + classes["plant__Organ"]["properties"]["alive"]["noData"] = 2 + assert "a boolean column with missing values is UINT8 (0, 1) with noData 255" in \ + str(validate_bytes(pack(document, binary))) diff --git a/conformance/README.md b/conformance/README.md index fa9c75a..95bb42a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -12,6 +12,7 @@ numbers, the same bytes on every run; the CI checks that the committed files are | `soil_and_roots` | a root system, a regular soil grid with z pointing down, and their mapping | | `adaptive_soil` | an adaptive grid's leaf cells (`level`, `index`) | | `parameter_set` | a `table` data structure: parameters per plant and in the scalars, references, an infinite value | +| `missing_values` | integer, text, boolean and float columns with missing values; a date column (`format`); `unknown` edge types and an `order` column; a `mixed` entity set; a labels map; a parent in another axis; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system, ids kept, and their index | ## The expected values @@ -23,12 +24,14 @@ numbers, the same bytes on every run; the CI checks that the committed files are "data_structures": [ {"id": "...", "kind": "graph", "origin": [[4 numbers] x 4, row by row], "platform": "...", "model": "...", "extras": {...}, (when present) - "entity_sets": [{"name": "SubOrgan", "rank": 6, "count": 10, + "labels": {"Internode": "StemInternode", ...}, (when present) + "entity_sets": [{"name": "SubOrgan", "rank": 6, "scale": "SubOrgan", "count": 10, (scale when present) "columns": {"id": {"dtype": "int64", "values": [...]}, "label": {"dtype": "string", "values": [...]}, (labels and enums as their names) "end": {"dtype": "float64", "values": [[x, y, z], ...]}, ...}, "variables": {"length": {"unit": "m", "role": "state", "standard_name": "length", - "kind": "extensive", "description": "..."}, ...}}], + "kind": "extensive", "description": "...", + "format": "date"}, ...}}], (format when present) "relations": [... like entity sets, with "source_set" and "target_set"], "scalars": {... like an entity set, one row}, "grid": {"kind", "shape", "origin", "spacing", "axes", "periodic", "z_up", "max_level", "cells"}, @@ -38,10 +41,13 @@ numbers, the same bytes on every run; the CI checks that the committed files are ``` - **Floats** are the shortest decimals giving the same double, compared exactly. -- **NaN** is `null`; +∞ and −∞ are the strings `"Infinity"` and `"-Infinity"`. +- **NaN and missing values** are `null`, whatever the column's type; +∞ and −∞ are the strings `"Infinity"` and + `"-Infinity"`. - **Positions** are the FLOAT32 values of the file, widened to doubles. - **`dtype`** is the column's numpy type name (`float64`, `int64`, `uint8`, `bool`...), or `string` for text and enums. -- Keys only appear when the frame has them (no `extras`, `relations`, `scalars`, `grid` or `meshes` when absent). + A boolean column with missing values (stored as UINT8) is `bool`. +- Keys only appear when the frame has them (no `extras`, `labels`, `relations`, `scalars`, `grid`, `meshes`, + `scale` or `format` when absent). ## Checking a codec diff --git a/conformance/adaptive_soil.glb b/conformance/adaptive_soil.glb index 80dd7a0e0b2f43d3bbc608f14d68f88dc99b9e72..0e5b314f5138926adbd4ac301ab8f975aa6d15b1 100644 GIT binary patch delta 82 zcmca3^hby{JtxGCiGhKkhLeF|!$e*SMx%{xpO|EHs+IB*b5oV9luGlm^YY8{lysEJ l5_3v{BF2-=nJ+UMP3C5?XEfUE%@WDXte~K Frame: @@ -155,7 +155,7 @@ def wheat_shoot() -> Frame: platform="fspm-tf", model="toy shoot", origin=np.array([[1., 0., 0., 0.5], [0., 1., 0., 0.25], [0., 0., 1., 0.], [0., 0., 0., 1.]])) - return Frame([data_structure], time=12.5, step=300, writer=WRITER, vocabulary_version="0.1") + return Frame([data_structure], time=12.5, step=300, writer=WRITER, vocabulary_version="0.2") def section_cells() -> Frame: @@ -189,7 +189,7 @@ def section_cells() -> Frame: data_structure = DataStructure("section", entity_sets=[cells], relations=[connections], meshes=[mesh], platform="fspm-tf", model="toy cell network", extras={"fspm-tf": {"note": "extras are the writer's, kept by readers"}}) - return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.2") def soil_and_roots() -> Frame: @@ -239,7 +239,7 @@ def adaptive_soil() -> Frame: grid=Grid(kind="adaptive", shape=(2, 1, 1), origin=(0., 0., 0.), spacing=(0.1, 0.1, 0.1), axes=("x", "y", "z"), periodic=(False, False, False), z_up=False, max_level=1, cells="Cell"), platform="fspm-tf", model="toy adaptive soil") - return Frame([soil], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + return Frame([soil], time=0., step=0, writer=WRITER, vocabulary_version="0.2") def parameter_set() -> Frame: @@ -263,7 +263,53 @@ def parameter_set() -> Frame: method=dict(unit="1", role="setting"))) data_structure = DataStructure("archisimple_parameters", kind="table", entity_sets=[plants], scalars=scalars, platform="fspm-tf", model="RootArchitecture parameters") - return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.1") + return Frame([data_structure], time=0., step=0, writer=WRITER, vocabulary_version="0.2") + + +def missing_values() -> Frame: + """A plant as platforms with per-class fields write it (MTG.jl, PlantGraphs, GroIMP): integer, text, boolean and + float columns with missing values, a date column, an edge type the writer does not know, the order of children, + an entity set mixing base scales (a whole root axis among organs), the platform's labels mapped to the + vocabulary's, a branch's first internode whose parent is in another axis, bearer relations between two pairs of + sets.""" + plant = EntitySet("Plant", rank=1, scale="Plant", columns=dict(id=np.array([0], dtype=np.int64), + label=np.array(["Wheat"], dtype=object))) + axes = EntitySet("Axis", rank=2, scale="Axis", columns=dict( + id=np.array([1, 2], dtype=np.int64), label=np.array(["Axis", "Axis"], dtype=object), + complex=np.array([0, 0], dtype=np.int64), parent=np.array([-1, -1], dtype=np.int64), + edge_type=np.array(["none", "none"], dtype=object))) + organs = EntitySet("Organ", rank=5, scale="mixed", columns=dict( + id=np.arange(10, 17, dtype=np.int64), + label=np.array(["Internode", "Internode", "Leaf", "Leaf", "Internode", "Bud", "Root"], dtype=object), + complex=np.array([1, 1, 1, 1, 2, 2, 0], dtype=np.int64), + parent=np.array([-1, 10, 10, 11, 11, 14, -1], dtype=np.int64), + edge_type=np.array(["none", "successor", "branch", "unknown", "branch", "successor", "none"], dtype=object), + order=np.array([0, 1, 0, 0, 0, 0, 1], dtype=np.int64), + length=np.array([0.12, 0.08, 0.2, 0.15, 0.05, np.nan, 0.3]), + rank=np.ma.MaskedArray(np.array([1, 2, 1, 2, 1, 0, 0], dtype=np.int32), + mask=[False, False, False, False, False, True, True]), + tissue=np.array(["stem", "stem", "blade", None, "stem", None, "root"], dtype=object), + alive=np.ma.MaskedArray(np.array([True, True, True, False, True, False, False]), + mask=[False, False, False, False, False, True, False]), + emergence=np.array(["2026-03-01", "2026-03-04", "2026-03-02", "2026-03-06", None, None, "2026-02-28"], + dtype=object)), + variables=_declare(length=dict(unit="m", standard_name="length", kind="extensive"), + rank=dict(unit="1", kind="descriptor", description="the organ's rank on its axis"), + tissue=dict(unit="1", kind="descriptor"), + alive=dict(unit="1", kind="descriptor"), + emergence=dict(unit="1", kind="descriptor", format="date", + description="the organ's emergence date"))) + bearer_axes = Relation("bearer_Axis_Organ", source_set="Axis", target_set="Organ", columns=dict( + source=np.array([2], dtype=np.int64), target=np.array([11], dtype=np.int64), + relation=np.array(["branch"], dtype=object))) + bearer_buds = Relation("bearer_Organ_Axis", source_set="Organ", target_set="Axis", columns=dict( + source=np.array([15], dtype=np.int64), target=np.array([1], dtype=np.int64), + relation=np.array(["decomposition"], dtype=object))) + data_structure = DataStructure("plant", entity_sets=[plant, axes, organs], relations=[bearer_axes, bearer_buds], + platform="fspm-tf", model="toy plant with per-class fields", + labels={"Internode": "StemInternode", "Leaf": "Leaf", "Bud": "Meristem", + "Root": "RootAxis", "Axis": "ShootAxis"}) + return Frame([data_structure], time=3., time_unit="d", step=72, writer=WRITER, vocabulary_version="0.2") def growing_sequence() -> list: @@ -272,7 +318,8 @@ def growing_sequence() -> list: SCENES = {"root_system": root_system, "wheat_shoot": wheat_shoot, "section_cells": section_cells, - "soil_and_roots": soil_and_roots, "adaptive_soil": adaptive_soil, "parameter_set": parameter_set} + "soil_and_roots": soil_and_roots, "adaptive_soil": adaptive_soil, "parameter_set": parameter_set, + "missing_values": missing_values} def files() -> dict: diff --git a/conformance/growing_sequence/frame_000000.glb b/conformance/growing_sequence/frame_000000.glb index 7d09190f858bbd62adf7c76f708b98d975096403..00afa7dc892ae9a49391e42f1d1a8615775fffc4 100644 GIT binary patch delta 80 zcmdm!awml^JtxGCiGhJ3#t2BrP2@9YG}`F)i$g}IS}89vH&w|>sWdM;FTXrbNk^$H iF{czLV!YX&^DYac(PUXZdq$(pv3%3{H-D7%76AaR+85sd delta 51 zcmcbUvL}TvJtxGCiGhK^#)yI8f$T&+b4J6BZofD-$8z3bVKkg<#b?iGxVegNI{#)7 HIS&y4l8_GL diff --git a/conformance/growing_sequence/frame_000001.glb b/conformance/growing_sequence/frame_000001.glb index ecb3a69c762e25e7fe22ea64a89a22de0d8ea639..cc55210f04043629f62a26a207b3aaed5fadf3f7 100644 GIT binary patch delta 80 zcmaD-+)~1qo)hB6#K6G7V#UC~AvclFoY82b+b<3oooc1L#N1RRE2YxB?7aN)JS82a jvc#NHposBid(OKoj7F1X`5YLHHplVJ<(MGpl95Oo9N_mO7sY+H#rFq$T`Q>>^ lI!a}UIi)}mCd=_TFdA)+=Ud9ZnNiMJ9RSyJ7ncA4 delta 65 zcmaFT#dxHPkuNzTU^1k;^O_(Gv%0&C%+9PkGb$`YtR$7B*-m$??##mw z@Zyiqf8fQFhdqhl$+N$}vj=~|`hE54>Q2%or)3+Os;>H8_0{)GM#0(r9m}#l`M|P% zf8Vk`KRr2~+a!)@Vt3~DC1tT6g?7hwcWZXV9?_68l0*zvhVgV#O@`&#u4^wUcHDx71Qw)i}z1Aw#deSOxA6E^E7Ge&~$geGJ_5(1~b zfBNX56f=W~8d`@1TXBC(Cj^0fV30j*KR_LWYUVbVsoQCwQTaPRp1 zA;8gaHj${NG~KR%Un~GsZX8_uTxuyL|9nwGB6BUH&_Y1Lf;LxB7=4CU+^ITvLIM`U z&3;q;X3*(iM7si03$UzTZc8bssoE1ZB4DxF0>t-=QZDEIvTL_uM{GbLTIn14AD=-V zq2VYQZ-R3F=;S~+T51Tpt-V%r&vU$byXksPqeVaVYHc9PNca?qLO|>@f0~G}nv7{V z!h8QPE+a%a3d?>HmkGLAWfMpDT!7R3U8mlvx4mW^StfoQ&md5whd+CKc>KVCw=r^) zz`x8utqUvX5Lem;w%cg%K*|UY3EZ@MQ4~-#V;*sUI-O7kX`|0MtUtU4D?`ww#?e=Z zY=om81*c_Q9traPv%Hu0r8t@~A=qXfk2)oAnk|9^pfDN`aozeNmg9m9uS+s_yBgh$ z6%tN3DG$PF+*sawY0jOVCwp$RBgj#ynWYgF&V5OO8I}1<*(hHFcq~Fhu@ORs1iRuJ zP^TDGK|JgCX&ggUqqiX+Xv7{kW`CUSxyIfsd>TeqVY=xppx#i)lIL(0LpK)mt1%@> zO0i~AG*jNj1i5|aqh;s`h08p@F~PgDfhk|As>3j$42!;W+tlO{CBt;b)Z^nQGLD>4 zVv$R1EOGu0GfG%;7{mBB<}jeV3X6U2>KSeGcqPiXn={f2cuB0yiyH}Og_nd-)yYVp zm!laLB~IRlMJ1@r9LzZ_?$n}#5kvElR*++(Rs|Qgm zUUn%*-ZY^lS6nvRl^|sl&=BuVL~ukNEBbU)z~~&>>=YR3d#c?`(h88Up_q#eQCa@(NbU>a!M25Bx4jSSfq%(?kgYE69?098N;+-mw zK!Ent53z9_F*3#US4cXEZps|5eB~$OGPJB!9%;%$p9TZ0Wh3T~eBP0-=F-f4J z_e!o4{~9`P9ikLh7k_l6`gY$HW$3n|(;DH=smI9^T+$o6Y>?9}Vo0~V5YYugCo z=Jin|h}+yo5YOF45U-Wfh+AZK^;*vG+#<`X=dH;rG^hM2i&Rq2n@EFqrE4s;;pW6H z6r|D2DX>T*jb@H?A#h&VMcQk$bGKedc`xTOMZnFxsY0Syl=1ln{?NF4cntq)WS^4! z#Q!^rp69to!M-LZugmcbIld{!x8(S?9N&>+skF0GvTj+5|GH7hrACQvDRC5kH7YsO zsQ4>*tF&_q80*KF>a|r91^4XE5C44e?6>{-?Oz`J{oSwoKmCZ$pZgwv{*d24?+bqa zJ{RkM?IS*(!5rc9d-+;e1mN2X2xL&0Fi85aiO(zWz6vku6yJB{`yPy?QpqYkw=B2j yHmfzST66TQu4k^EHO0)Wx@|y~59;nVym#RJ$ar0tE4?dy!V-VfeJZ`E_5T2;=XY%Y literal 0 HcmV?d00001 diff --git a/conformance/parameter_set.glb b/conformance/parameter_set.glb index 8ac363c5849946b32177c18ca73ff4fbc1a9069f..8ea1464b5f3ed1410f751371879fc2dc3664cc63 100644 GIT binary patch delta 78 zcmZpXo+Hhho)hB6#K6EXhnImtVj`~vqtQk;4OSVQYNfoy+*BnirP93Yy!`S!B^{-* h#GF!~i1FrJRwX7zqsbH4?HP?WUtqUqoh-sz000%s6#D=G delta 65 zcmbOu-6GAKo)hB6#K6GN!V4rO@>(z&ZgkUNWsIGi&m}$Cn~8Vx7SkDpmVL1FVB*$wglGOQU` delta 51 zcmX?8y{4KkJtxGCiGhJ(jV%Mi3)zW$=8T3L-F|Uwj^(_|!e}_zn$MomaC0@^JpRox HavS6Usl5;R diff --git a/conformance/section_cells.glb b/conformance/section_cells.glb index 2272cc37a878e7301ddb5cfe7a8802078814e76f..747bfa4ba1e3d8cc01288a9a8d115dfec9ca5d04 100644 GIT binary patch delta 87 zcmeCtT%yUFo)hB6#K6F?M1+AMVx&j?Jlp>dXK;V;T>xoGy7$>s%qUvEq0OI2qe*gdg delta 75 zcmV-R0JQ(Kj{%gA0S#wtR7L^-005Ln006*|4LAWYu}0Ji16_SAlWPJSvsDZI1pzUW hQxQK6F(ND?cXDZTWpW}qdy*0!v%L{C6tnatz#&Fe7gzuQ diff --git a/conformance/wheat_shoot.glb b/conformance/wheat_shoot.glb index 6c059ce0e951942c6c43171df052db07bc096603..639cb2b0cb95f664262e5e9e7b3762cb06c252c9 100644 GIT binary patch delta 79 zcmeA%TVuwXo)hB6#K6FyBhA2|Fp<}S(P*Qa9EXffwNhSUZmN=%QfXdxUVeF=l8#bY iVooVg#CUTu$66LfqsfB2PK-vILwFBxZ5EXH%>)20A{D&= delta 54 zcmZ2u)?>z-o)hB6#K6G7BMl@b@>(z&Zgi95n7osddvg!RLKa5D$$Gp_jE0*Fc@J=H JHj((j1OQ}O4j}*l diff --git a/doc/governance.md b/doc/governance.md index 85be9af..0867f7c 100644 --- a/doc/governance.md +++ b/doc/governance.md @@ -43,5 +43,6 @@ The CI then runs every codec against the new conformance files, and shows which ## Status -Version 0.1 is a draft. It was written with metafspm as the first platform; its partners' reviews come next. Until -0.1 is released, the draft may still change in ways that make earlier files invalid. +Version 0.2 is a draft. 0.1 was written with metafspm as the first platform; 0.2 adds what the C++ (CPlantBox), Julia +(VirtualPlantLab) and Java (GroIMP) bindings needed (its "Changes from 0.1"). The partners' reviews come next. Until +0.2 is released, the draft may still change in ways that make earlier files invalid. diff --git a/implementations.md b/implementations.md index 1efb03e..321cb1d 100644 --- a/implementations.md +++ b/implementations.md @@ -2,8 +2,8 @@ | Platform | Language | Where | Reads | Writes | Specification version | |---|---|---|---|---|---| -| openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.1 | -| openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.1 | +| openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.2 | +| openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.2 (through openalea.fspm_tf) | | fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.1 | | CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.1 | | fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.1 | diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py index f133287..5343641 100644 --- a/paraview/fspm_tf_reader.py +++ b/paraview/fspm_tf_reader.py @@ -66,9 +66,10 @@ def _numeric(column: np.ndarray) -> bool: def _codes(column: np.ndarray) -> tuple: - names = sorted(set(column.tolist())) + """Each text value's code among the column's sorted values; a missing value (None): -1.""" + names = sorted({value for value in column.tolist() if value is not None}) code_of = {name: code for code, name in enumerate(names)} - return np.array([code_of[value] for value in column.tolist()], dtype=np.int64), names + return np.array([code_of.get(value, -1) for value in column.tolist()], dtype=np.int64), names LINKS = ("complex", "parent", "edge_type") @@ -106,6 +107,8 @@ def _add_columns(data, entity_set, rows: np.ndarray, field_data, settings: bool if not _numeric(column): continue values = column[taken].astype(np.float64) # float64: NaN where no entity + if np.ma.isMaskedArray(values): + values = values.filled(np.nan) # and where a value is missing values = np.where(valid.reshape((-1,) + (1,) * (values.ndim - 1)), values, np.nan) data.AddArray(_array(label, values)) diff --git a/spec/schema/fspm_scene.schema.json b/spec/schema/fspm_scene.schema.json index 018f39d..d570ca9 100644 --- a/spec/schema/fspm_scene.schema.json +++ b/spec/schema/fspm_scene.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.1/fspm_scene.schema.json", + "$id": "https://github.com/openalea/fspm-TF/schema/0.2/fspm_scene.schema.json", "title": "FSPM_scene", - "description": "The FSPM_scene root extension of a glTF 2.0 asset (fspm-TF 0.1): one frame of a scene.", + "description": "The FSPM_scene root extension of a glTF 2.0 asset (fspm-TF 0.2): one frame of a scene.", "type": "object", "required": ["version", "frame", "dataStructures"], "properties": { @@ -39,6 +39,8 @@ "entitySets": {"type": "array", "items": { "type": "object", "required": ["name", "propertyTable"], "properties": {"name": {"$ref": "#/$defs/identifier"}, "rank": {"type": ["integer", "null"]}, + "scale": {"enum": ["Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", + "Cell", "Compartment", "Connection", "mixed"]}, "propertyTable": {"$ref": "#/$defs/index"}}, "additionalProperties": false}}, "relations": {"type": "array", "items": { @@ -47,6 +49,7 @@ "targetSet": {"$ref": "#/$defs/identifier"}, "propertyTable": {"$ref": "#/$defs/index"}}, "additionalProperties": false}}, "scalars": {"$ref": "#/$defs/index"}, + "labels": {"type": "object", "additionalProperties": {"type": "string"}}, "grid": {"$ref": "#/$defs/grid"}, "extras": {"type": "object"}, "meshes": {"type": "array", "items": { @@ -94,7 +97,8 @@ "role": {"enum": ["state", "input", "parameter", "derived", "setting"]}, "minimum": {"type": "number"}, "maximum": {"type": "number"}, "references": {"type": "array", "items": {"type": "string"}}, - "doi": {"type": "array", "items": {"type": "string"}}}, + "doi": {"type": "array", "items": {"type": "string"}}, + "format": {"enum": ["date", "date-time", "time"]}}, "additionalProperties": false} } } diff --git a/spec/schema/index.schema.json b/spec/schema/index.schema.json index abac4bb..6b90d64 100644 --- a/spec/schema/index.schema.json +++ b/spec/schema/index.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.1/index.schema.json", + "$id": "https://github.com/openalea/fspm-TF/schema/0.2/index.schema.json", "title": "fspm-TF sequence index", - "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (fspm-TF 0.1).", + "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (fspm-TF 0.2).", "type": "object", "required": ["fspmIndex", "timeUnit", "frames"], "properties": { diff --git a/spec/specification.md b/spec/specification.md index a91ec32..f63e731 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -1,6 +1,6 @@ -# fspm-TF 0.1 specification +# fspm-TF 0.2 specification -Status: draft (2026-10-29). The key words MUST, SHOULD and MAY are to be read as in RFC 2119. +Status: draft (2026-11-02). 0.2 adds to 0.1 (see "Changes from 0.1" at the end); a 0.1 file is a valid 0.2 file. The key words MUST, SHOULD and MAY are to be read as in RFC 2119 (https://www.rfc-editor.org/rfc/rfc2119). fspm-TF is an exchange format for whole functional-structural plant model scenes: multiscale plant graphs, their variables and parameters, their geometry, and the environment's grids, at successive time steps. A **frame** (one @@ -15,7 +15,16 @@ The language-neutral files of this version are part of the specification: | `spec/vocabulary/standard_names.csv` | the standard names, their canonical units and kinds, and each platform's names for them (§6) | | `spec/vocabulary/labels.csv` | entity labels, their scales and Plant Ontology terms (§6) | | `spec/vocabulary/enums.json` | the values of `edge_type`, `kind`, `role`, data structure and grid kinds; the base scales | -| `conformance/` | reference files and their expected values (§9) | +| `conformance/` | reference files and their expected values (§9): the specification as test cases | + +When these sources disagree, `specification.md` prevails, then the schemas, then the conformance files; the +disagreement is a defect of the lower one, to be fixed. + +The format builds on glTF 2.0 (https://registry.khronos.org/glTF/specs/2.0/glTF-2.0.html) and two of its +extensions: `EXT_structural_metadata` for tables +(https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata) and +`EXT_mesh_features` for feature IDs +(https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features). ## 1. The data model @@ -32,11 +41,21 @@ The language-neutral files of this version are part of the specification: Its `origin` is a 4×4 matrix placing its coordinates in the scene's frame. - **The scene's frame** is right-handed, in metres, with **z up**: z = 0 at the soil surface, negative below it. - Every position (meshes, the segment ends of the vocabulary's `proximal_end_*` and `distal_end_*`, grid origins and - spacings) is in metres. A platform with other axes or units (e.g. y up, or centimetres) converts its coordinates, or - puts the conversion in the data structure's origin. + Meshes, grid origins and grid spacings are in metres. Position columns (e.g. the vocabulary's `proximal_end_*` and + `distal_end_*`) keep the platform's length unit, declared like any variable's (§5), so that values stay at the + platform's precision; readers convert them with their declared unit. A platform with other axes (e.g. y up) puts + the rotation in the data structure's origin. - An **entity set** is a table: one row per entity, one column per variable. A graph has one entity set per scale, with that scale's `rank` (coarser scales have smaller ranks). A grid's cells form one entity set, with no rank. + - An entity set MAY declare its `scale`: one of the vocabulary's base scales (`enums.json`: Plant, Axis, + GrowthUnit, Phytomer, Organ, SubOrgan, Layer, Cell, Compartment, Connection), or `mixed` when its rows belong to + several base scales (e.g. CPlantBox's organs: whole root axes and leaves). + - The `rank` SHOULD be the position of the set's base scale in that list, from 1 (Plant 1, Axis 2, Organ 5, + SubOrgan 6), when it has one. Readers MUST NOT assume consecutive ranks. + - Platforms have different scales. A reader SHOULD find the tables it needs by what they hold (standard names, and + the `complex` chain between sets), not by their names. For example, the segments are the set holding segment + ends, and their organs are the set their `complex` points to. Writers SHOULD declare the standard names of + positions, lengths and radii. - A **relation** is an edge table between two entity sets of the same data structure. Examples: a cell network, or connections between compartments. - The **scalars** are a one-row table of values for the whole data structure. @@ -51,9 +70,11 @@ A frame holding only data structures with parameters, and no geometry, is a **pa A frame MUST be a GLB file (glTF 2.0 binary): -1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.1. +1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.2. 2. Every buffer view holding table values MUST start at a multiple of 8 bytes. The JSON chunk MUST be padded with - spaces and the binary chunk with zeros to 8 bytes, as `EXT_structural_metadata` requires. + spaces and the binary chunk with zeros to 8 bytes, as `EXT_structural_metadata` requires. The buffer's + `byteLength` SHOULD be the binary chunk's padded length (glTF-Validator warns of more than 3 bytes of padding + beyond the buffer). 3. `extensionsUsed` MUST list `FSPM_scene` and `EXT_structural_metadata`, plus `EXT_mesh_features` when there are meshes. `FSPM_scene` SHOULD NOT be listed in `extensionsRequired`, so that every glTF viewer opens the file. 4. The glTF scene's root node, `fspm_scene`, turns the scene's frame (z up) into glTF's (y up), so that glTF viewers @@ -70,14 +91,16 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: ```text { - "version": "0.1", - "vocabulary": "0.1", + "version": "0.2", + "vocabulary": "0.2", "frame": {"time": 1.0, "timeUnit": "d", "step": 24}, "writer": {"platform": "...", "model": "..."}, "dataStructures": [{ "id": "plants", "kind": "graph", "platform": "...", "model": "...", "origin": [16 numbers, column-major], "node": 0, - "entitySets": [{"name": "Plant", "rank": 1, "propertyTable": 0}, {"name": "SubOrgan", "rank": 6, "propertyTable": 1}], + "entitySets": [{"name": "Plant", "rank": 1, "scale": "Plant", "propertyTable": 0}, + {"name": "SubOrgan", "rank": 6, "scale": "SubOrgan", "propertyTable": 1}], + "labels": {"Internode": "StemInternode"}, "relations": [{"name": "Connection", "sourceSet": "Cell", "targetSet": "Cell", "propertyTable": 2}], "scalars": 3, "grid": {"kind": "regular", "shape": [..], "origin": [..], "spacing": [..], "axes": [..], "periodic": [..], @@ -88,16 +111,21 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: "mappings": [{"name": "roots_in_soil", "source": {"dataStructure": "plants", "entitySet": "SubOrgan"}, "target": {"dataStructure": "soil", "entitySet": "Voxel"}, "method": "overlap", "propertyTable": 4}], "variables": {"": {"": {"unit": "m", "standardName": "length", "kind": "extensive", - "role": "state", "minimum": 0, "references": [], "doi": []}}} + "role": "state", "minimum": 0, "references": [], "doi": [], + "format": "date"}}} } ``` - **Identifiers.** Data structure ids, entity set, relation and mapping names, and column names MUST be identifiers - (`^[a-zA-Z_][a-zA-Z0-9_]*$`). Within a frame, data structure ids are unique, and so are the names of a data - structure's entity sets. + (`^[a-zA-Z_][a-zA-Z0-9_]*$`). Within a frame, data structure ids are unique. Within a data structure, entity set + names are unique, and so are relation names. +- **Labels.** A data structure MAY carry `labels`, a map from the labels it uses (e.g. a model's class names) to the + vocabulary's labels (§6). Readers looking for an entity of the vocabulary (e.g. the leaves) use it; the entities + keep the platform's labels. - **Extras.** A data structure MAY carry `extras`, a JSON object for the writing platform, e.g. what it needs to - rebuild its own data structure exactly. Its keys SHOULD be the platform's name. Other readers MUST NOT need it, - and SHOULD keep it. + rebuild its own data structure exactly. Its keys SHOULD be the platform's name as in the vocabulary's platform + columns, in lower case (`metafspm`, `cplantbox`, `groimp`, `virtualplantlab`). Other readers MUST NOT need it, and + SHOULD keep it. - **Property tables.** Every table of §1 is one `EXT_structural_metadata` property table of its own class, and no property table serves two tables. Writers SHOULD name the classes `__` (mappings: `mapping__`). Readers MUST find tables through the `propertyTable` indices, not through class names. @@ -116,11 +144,21 @@ A column is one property of the table's class: | text | `STRING` | | labels, edge types and relation labels | `ENUM` (§4.2) | -- Variable-length arrays and the other glTF metadata types are not part of 0.1. +- Variable-length arrays and the other glTF metadata types are not part of 0.2. - **NaN.** NaN and infinities are forbidden in property tables. A float column with missing values (NaN) declares `noData`: the most negative finite value of its type, −1.7976931348623157e308 for FLOAT64 or −3.4028234663852886e38 for FLOAT32, stored where the value is NaN. Readers MUST read `noData` back as NaN. A column cannot hold both NaN and that value. +- **Missing values in other columns.** A variable column whose entities do not all have a value (e.g. a field that + only some of a platform's classes have) declares `noData`, and readers MUST read the rows holding it as missing: + - an integer column: a value of its type that the column does not hold; writers SHOULD use the type's minimum + (signed) or maximum (unsigned) when it is free; + - a text (`STRING`) column: a string the column does not hold; writers SHOULD use the empty string when it is free; + - booleans: `BOOLEAN` has no `noData`, so a boolean column with missing values is written as `UINT8` (0 false, + 1 true) with `noData` 255, and its class property declares `"extras": {"FSPM_boolean": true}`. Readers read it as + booleans with missing values. + + The reserved columns' `noData` (−1 in `complex` and `parent`) is not a missing value: it is the id of no entity. - **Infinities.** A float column holding +∞ or −∞ declares, in its class property's `extras`, `FSPM_infinity`: `{"positive": P, "negative": N}`, with P the largest finite value of its type (1.7976931348623157e308 for FLOAT64, 3.4028234663852886e38 for FLOAT32) and N the next value after the most negative one @@ -140,7 +178,8 @@ These columns have fixed meanings, in the tables where they appear: | `label` | ENUM | the entity's label (§6) | | `complex` | INT64, `noData` −1 | the id of the entity containing it, in a coarser entity set of the graph (MTG decomposition `/`); usually the nearest coarser one, but e.g. a cell wall may belong to its segment directly | | `parent` | INT64, `noData` −1 | the id of its predecessor in the same entity set | -| `edge_type` | ENUM `FSPM_edge_type` | how it follows its parent: `none` (no parent), `successor` (MTG `<`), `branch` (MTG `+`) | +| `edge_type` | ENUM `FSPM_edge_type` | how it follows its parent: `none` (no parent), `successor` (MTG `<`), `branch` (MTG `+`), `unknown` (it has a parent, but the writer does not know how it follows it) | +| `order` | INT64, optional | the entity's position, from 0, among the entities that share its parent in the writer's tree (its `parent`, else its bearer, else its `complex`), all entity sets together: for platforms that keep the order of children | | `level`, `index` | INT64; INT64 vector, one per dimension | an adaptive grid's leaf cell: its refinement level, and its integer index at that level | **Relations** @@ -164,14 +203,20 @@ Rules on these columns: complex. - `parent` and `edge_type` go together. `edge_type` is `none` exactly where `parent` is −1. The parents MUST form no cycle. +- `parent` MAY be in another complex than the entity's own: the first entity of a branch (e.g. a lateral root's + first segment, whose parent is a segment of the bearing root) usually is. - Every id referred to MUST exist in the entity set referred to. - `parent` links entities of the same scale. When an axis's first entity grows on an entity of another scale (in an MTG, a branch axis's first internode, linked to its bearer only through the axes), its `parent` is −1. Its bearer MAY be written as a relation named `bearer`, from the entity to the one it is borne on. + Its `relation` labels say how: `successor`, `branch` or `decomposition` (MTG `<`, `+`, or `/` to an entity that is + not its complex). - Other links between entities are relations, one per kind of link. A pair of entities linked in several ways (e.g. a - successor edge that is also a platform's own edge) appears in each of these relations. + successor edge that is also a platform's own edge) appears in each of these relations. A kind of link between + several pairs of entity sets is one relation per pair, named ``, or `__` when + there are several (e.g. `bearer_Axis_Internode`); readers find them by the prefix. - Each label column or relation column is an `ENUM` of its own, listing the values used. The enum - `FSPM_edge_type` is fixed: `none` = 0, `successor` = 1, `branch` = 2. + `FSPM_edge_type` is fixed: `none` = 0, `successor` = 1, `branch` = 2, `unknown` = 3. - A grid's cells MAY omit `id`; their ids are then their row numbers. A regular grid's rows are its cells in C order of `shape`, the last axis varying fastest, and its ids, if written, MUST be these row numbers. @@ -188,6 +233,7 @@ Every other column is a variable. It SHOULD be declared in `FSPM_scene.variables | `role` | required: `state`, `input`, `parameter`, `derived` or `setting` (enums.json) | | `minimum`, `maximum` | the variable's plausible range, as documentation (not a constraint on values) | | `references`, `doi` | its sources | +| `format` | for a text column holding dates or times: `date`, `date-time` or `time`, as ISO 8601 (`2022-08-24`, `2022-08-24T13:00:00`, `13:00:00`) | Parameters are variables with role `parameter`. A parameter goes in the table it applies to: @@ -210,7 +256,7 @@ Units are UCUM codes, with: ## 6. Vocabulary -The vocabulary (version `0.1`) lists standard names. For each, it gives: +The vocabulary (version `0.2`) lists standard names. For each, it gives: - a canonical unit; - a kind; @@ -253,7 +299,7 @@ two platforms. A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json`, following `index.schema.json`: ```text -{"fspmIndex": "0.1", "timeUnit": "d", +{"fspmIndex": "0.2", "timeUnit": "d", "frames": [{"time": 0.0, "step": 0, "file": "frame_000000.glb"}, ...]} ``` @@ -263,7 +309,9 @@ A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json` given to another entity in a later frame of the same data structure and entity set. Labels and parents MAY change, e.g. an apex moved by growth. A platform whose own ids may come back (a counter restarted, a deleted maximum's id given again) keeps, across a sequence, a map from its entities to the ids it writes. -- **Packing.** Packing a sequence into one GLB is not part of 0.1. +- **Ids are not positions.** Ids may be sparse (e.g. offsets keeping them stable as a plant grows). Readers SHOULD + NOT size arrays by the largest id. +- **Packing.** Packing a sequence into one GLB is not part of 0.2. ## 9. Conformance @@ -278,15 +326,33 @@ sequence: | `soil_and_roots` | a root system, a regular soil grid with z pointing down, and their mapping | | `adaptive_soil` | an adaptive grid's leaf cells (`level`, `index`) | | `parameter_set` | a `table` data structure with parameters per plant and in the scalars, references and DOIs; an infinite value | +| `missing_values` | integer, text and boolean columns with missing values; a date column; `unknown` edge types and an `order` column; a `mixed` entity set; a labels map; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system and their index | -An implementation conforms to 0.1 when two things hold: +An implementation conforms to 0.2 when two things hold: -- **Reading:** what it reads from every conformance GLB equals the expected JSON. Floats are compared exactly; NaN is - null in the JSON, +∞ and −∞ the strings `"Infinity"` and `"-Infinity"`; positions are the FLOAT32 values. +- **Reading:** what it reads from every conformance GLB equals the expected JSON. Floats are compared exactly; NaN and + missing values are null in the JSON, +∞ and −∞ the strings `"Infinity"` and `"-Infinity"`; positions are the + FLOAT32 values. - **Writing:** what it writes passes `fspm-tf validate` with no error, and Khronos' glTF-Validator with no error. ## 10. Keeping what one does not read -A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.1 this covers +A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.2 this covers the glTF root's other extensions and `extras`, when they hold no buffer views. + +## Changes from 0.1 + +All additions; a 0.1 file is a valid 0.2 file. + +- Sources: `conformance/` is part of the agreement; the order in which the sources prevail. +- §1: an entity set's `scale`, ranks as base scale positions, reading by content; position columns in their declared + length unit. +- §2: the buffer's `byteLength` holds the padding. +- §3: unique relation names, a data structure's `labels`, the extras key. +- §4.1: missing values in integer, text and boolean columns. +- §4.2: `edge_type` `unknown`, the `order` column, a parent in another complex, bearer relations' labels and names. +- §5: a variable's `format`. +- §6: the vocabulary 0.2 (`Seed`, `mtg_index`). +- §8: sparse ids. +- §9: the `missing_values` scene. diff --git a/spec/vocabulary/enums.json b/spec/vocabulary/enums.json index 1135935..d9b0491 100644 --- a/spec/vocabulary/enums.json +++ b/spec/vocabulary/enums.json @@ -1,8 +1,9 @@ { - "version": "0.1", + "version": "0.2", "edge_type": {"none": "the entity's scale has no predecessor for it (a root of its scale)", "successor": "it follows its parent on the same axis (MTG '<')", - "branch": "it branches from its parent (MTG '+')"}, + "branch": "it branches from its parent (MTG '+')", + "unknown": "it has a parent, but the writer does not know how it follows it"}, "kind": {"extensive": "proportional to the amount of matter: summed when entities are merged, split by weights", "intensive": "independent of the amount: averaged (weighted by volume or mass) when entities are merged", "massic_concentration": "an amount per mass: averaged weighted by mass", @@ -20,5 +21,11 @@ "grid_kind": {"regular": "shape cells of size spacing from origin", "adaptive": "a regular base grid refined up to maxLevel; the leaf cells are listed"}, "scales": ["Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", - "Connection"] + "Connection"], + "scale": {"mixed": "an entity set whose rows belong to several base scales (e.g. CPlantBox's organs: root axes and leaves)"}, + "format": {"date": "ISO 8601 date: 2022-08-24", "date-time": "ISO 8601 date and time: 2022-08-24T13:00:00", + "time": "ISO 8601 time: 13:00:00"}, + "bearer_relation": {"successor": "MTG '<' to an entity of another scale", + "branch": "MTG '+' to an entity of another scale", + "decomposition": "MTG '/' to an entity that is not its complex"} } diff --git a/spec/vocabulary/labels.csv b/spec/vocabulary/labels.csv index 0db358b..cefdf81 100644 --- a/spec/vocabulary/labels.csv +++ b/spec/vocabulary/labels.csv @@ -9,6 +9,7 @@ ShootGrowthUnit,GrowthUnit,,,no Plant Ontology term RootGrowthUnit,GrowthUnit,,,no Plant Ontology term ShootPhytomer,Phytomer,,,no Plant Ontology term RootPhytomer,Phytomer,,,no Plant Ontology term +Seed,Organ,PO:0009010,seed,"the seed (or a grass's caryopsis), bearing the seminal roots and the shoot" Meristem,Organ,PO:0020147,root apical meristem,or a shoot apical meristem: the term depends on the axis StemInternode,Organ,PO:0020142,stem internode, RootInternode,Organ,PO:0009005,root,a root portion between two branching points: no own term diff --git a/spec/vocabulary/standard_names.csv b/spec/vocabulary/standard_names.csv index 5eaae3b..b1379ae 100644 --- a/spec/vocabulary/standard_names.csv +++ b/spec/vocabulary/standard_names.csv @@ -40,3 +40,4 @@ cell_area,m2,extensive,area,cell,cross section,,Cell,area of the cell in the cro wall_thickness,m,intensive,thickness,cell wall,,,Connection Cell,thickness of the cell wall,wall_thickness,,,,GRANAP/MECHA: wall_thickness (um) net_co2_assimilation_rate,mol/(m2.s),intensive,assimilation rate,carbon dioxide,leaf,net,SubOrgan Organ,net CO2 assimilated per unit of leaf area (photosynthesis minus respiration),,,,,"not in metafspm yet (VPL's Ecophys.jl, GroIMP's photosynthesis models)" nitrate_concentration,mol/m3,volumic_concentration,amount concentration,nitrate,entity or soil,,SubOrgan Voxel,amount of nitrate per volume,,,,,not in metafspm yet (CPlantBox/DuMux solute transport) +mtg_index,1,descriptor,index,entity,,in its MTG code,any,the index of an entity in its MTG code: 1 in Internode1,,,,mtg_index, From 92c98d5eafd484747a0b646c9344536c3f35c804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:07:27 +0100 Subject: [PATCH 11/39] FspmTF.jl to specification 0.2: missing values in integer, text and boolean columns, an entity set's scale, a data structure's labels, a variable's format, edge_type unknown, the order column; its adapters on 0.2 instead of their extras: MTG.jl's missing attributes as missing, dates with their format, the children's order as order, mtg_index's standard name, ranks and labels given, its extras under virtualplantlab (model, scales, metadata, scene); PlantGraphs' edges unknown and children's order, missing fields; reading another platform's frame unchanged --- .../ext/FspmTFMultiScaleTreeGraphExt.jl | 135 ++++++++---------- .../julia/FspmTF/ext/FspmTFPlantGraphsExt.jl | 86 +++++------ bindings/julia/FspmTF/src/FspmTF.jl | 2 +- bindings/julia/FspmTF/src/codec.jl | 79 ++++++++-- bindings/julia/FspmTF/src/dump.jl | 12 +- bindings/julia/FspmTF/src/model.jl | 58 +++++--- bindings/julia/FspmTF/src/values.jl | 48 +++++-- bindings/julia/FspmTF/test/Project.toml | 1 + .../julia/FspmTF/test/plantgraphs/runtests.jl | 9 +- bindings/julia/FspmTF/test/runtests.jl | 18 +++ bindings/julia/README.md | 2 +- doc/julia.md | 16 ++- implementations.md | 6 +- 13 files changed, 296 insertions(+), 176 deletions(-) diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl index 0c4a38c..1ea1b16 100644 --- a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -9,14 +9,13 @@ A MultiScaleTreeGraph (MTG.jl) as an fspm-TF `graph` data structure, and back: | `complex` | its first ancestor at a smaller scale (none for the scene) | | `parent`, `edge_type` | its parent, when at the same scale and linked by `<` (successor) or `+` (branch) | | relation `bearer` | its parent otherwise, when it is not its complex linked by `/` (e.g. an axis borne by an internode) | -| `mtg_index` | the node's `index` | -| variables | its attributes | +| `order` | the node's position among its parent's children | +| `mtg_index` (standard name `mtg_index`) | the node's `index` | +| variables | its attributes: missing where a node has none; dates and times as ISO 8601 text with their `format` | | scalars | the scene's (the scale 0 root's) attributes | -What fspm-TF 0.1 cannot say is kept in the data structure's `extras["MultiScaleTreeGraph"]`, so that `to_mtg` rebuilds -the same MTG: the attributes a node does not have (fspm-TF columns have a value on every row; the missing value -written is NaN, `false`, 0 or ""), the types written as text (dates), the scene's node and its MTG metadata -(`symbols`, `scales`, `description`), and the order of children when it is not the order of their ids. +MTG.jl's scene node and metadata (`symbols`, `scales`, `description`) are kept in the data structure's +`extras["virtualplantlab"]` (with `"model": "MultiScaleTreeGraph"`), so that `to_mtg` rebuilds the same MTG. """ module FspmTFMultiScaleTreeGraphExt @@ -27,6 +26,7 @@ using MultiScaleTreeGraph using OrderedCollections: OrderedDict const PLATFORM = "MultiScaleTreeGraph" +const EXTRAS_KEY = "virtualplantlab" # the extras key: VirtualPlantLab's name in the vocabulary const METADATA_ATTRIBUTES = (:symbols, :scales, :description) # MTG.jl's node ids start at 1 and size its attribute store (an id of 2^32 allocates 2^32 rows), fspm-TF's start at 0 # and may be sparse (CPlantBox's segments: 2^32 + node id). to_mtg then numbers the nodes 1, 2, ... in the order of @@ -41,8 +41,10 @@ const OWN_ATTRIBUTES = (ID_ATTRIBUTE, PARENT_ATTRIBUTE, EDGE_TYPE_ATTRIBUTE) const LINK_NAMES = Dict(:< => "successor", :+ => "branch", :/ => "decomposition") const LINKS = Dict(name => link for (link, name) in LINK_NAMES) const INDEX_COLUMN = "mtg_index" -const INDEX_VARIABLE = Variable(unit = "1", kind = "descriptor", role = "state", +const INDEX_VARIABLE = Variable(unit = "1", standard_name = "mtg_index", kind = "descriptor", role = "state", description = "the node's index in its MTG code (1 in Internode1)") +"""The node's position among its parent's children, from 0.""" +position_among_siblings(node) = isroot(node) ? 0 : findfirst(child -> child === node, children(parent(node))) - 1 parent_node(node) = isroot(node) ? nothing : parent(node) @@ -72,8 +74,9 @@ default_set_name(scale_number, symbols) = # ── Attributes as columns ─────────────────────────────────────────────── -"""Columns of `nodes`' attributes, by name in order of first appearance; the missing rows and text types of each. -`skip`: attributes not written; the MTG's root's metadata (MTG.jl's `symbols`, `scales`, `description`) neither.""" +"""Columns of `nodes`' attributes (missing where a node has none), by name in order of first appearance, and their +declarations (a date's `format` added). `skip`: attributes not written; the MTG's root's metadata (MTG.jl's `symbols`, +`scales`, `description`) neither.""" function attribute_columns(nodes, set_name, variables, skip = ()) attribute_of(node, name) = isroot(node) && name in METADATA_ATTRIBUTES ? nothing : attribute(node, name; default = nothing) @@ -83,7 +86,6 @@ function attribute_columns(nodes, set_name, variables, skip = ()) push!(names, name) end columns, declarations = Columns(), Declarations() - missing_rows, text_types = json_object(), json_object() for name in names values = Any[attribute_of(node, name) for node in nodes] converted = FspmTF.column_of_values(values) @@ -94,29 +96,34 @@ function attribute_columns(nodes, set_name, variables, skip = ()) end column_name = string(name) check_identifier(column_name, "attribute of '$set_name'") - column, missing_of_column, text_type = converted + column, format = converted columns[column_name] = column - isempty(missing_of_column) || (missing_rows[column_name] = missing_of_column) - text_type === nothing || (text_types[column_name] = text_type) - haskey(variables, column_name) && (declarations[column_name] = variables[column_name]) + declared = get(variables, column_name, nothing) + if format !== nothing # dates and times: their format declared + declared = declared === nothing ? Variable(unit = "1", kind = "descriptor") : deepcopy(declared) + declared.format = format + end + declared === nothing || (declarations[column_name] = declared) end - return columns, declarations, missing_rows, text_types + return columns, declarations end # ── MTG → data structure ──────────────────────────────────────────────── """ - from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) - -> DataStructure + from_mtg(mtg; id="plants", scales=Dict(), ranks=Dict(), variables=Dict(), labels=Dict(), + platform="MultiScaleTreeGraph", model=nothing) -> DataStructure The MTG whose root is `mtg` as a `graph` data structure `id`. `scales`: the name of each scale's entity set (scale => name, e.g. `Dict(1 => "Plant", 2 => "Axis", 3 => "Organ")`; by default the scale's symbol when it has -only one, `Scale` otherwise). `variables`: the declarations of the attributes (name => `Variable`: unit, standard -name, kind, role), MTG.jl having no units. A scale 0 root (MTG.jl's `Scene`) is the data structure itself, its +only one, `Scale` otherwise). `ranks`: each scale's rank (by default the scale), e.g. the vocabulary's base scales' +positions. `variables`: the declarations of the attributes (name => `Variable`: unit, standard name, kind, role), +MTG.jl having no units. `labels`: the symbols => the vocabulary's labels (e.g. `Dict("Internode" => "StemInternode")`). A scale 0 root (MTG.jl's `Scene`) is the data structure itself, its attributes the scalars. """ -function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,String}(), - variables = Dict{String,Variable}(), platform = PLATFORM, model = nothing) +function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,String}(), ranks = Dict{Int,Int}(), + variables = Dict{String,Variable}(), labels = Dict{String,String}(), platform = PLATFORM, + model = nothing) variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) fspm_id(node) = attribute(node, ID_ATTRIBUTE; default = node_id(node)) nodes = preorder(mtg) @@ -137,7 +144,7 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S set_names[scale_number] = name end - entity_sets, missing_rows, text_types = EntitySet[], json_object(), json_object() + entity_sets = EntitySet[] bearers = OrderedDict{Tuple{String,String},Vector{Tuple{Int64,Int64,String}}}() for (scale_number, scale_nodes) in by_scale set_name = set_names[scale_number] @@ -173,14 +180,13 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S (fspm_id(node), fspm_id(tree_parent), LINK_NAMES[link(node)])) end columns["parent"], columns["edge_type"] = parents, edge_types + columns["order"] = Int64[position_among_siblings(node) for node in scale_nodes] columns[INDEX_COLUMN] = Int64[index(node) for node in scale_nodes] - attributes, declarations, missing_of_set, text_types_of_set = - attribute_columns(scale_nodes, set_name, variables, (Symbol(INDEX_COLUMN),)) + attributes, declarations = attribute_columns(scale_nodes, set_name, variables, (Symbol(INDEX_COLUMN),)) merge!(columns, attributes) declarations = merge(Declarations(INDEX_COLUMN => INDEX_VARIABLE), declarations) - isempty(missing_of_set) || (missing_rows[set_name] = missing_of_set) - isempty(text_types_of_set) || (text_types[set_name] = text_types_of_set) - push!(entity_sets, EntitySet(set_name; columns = columns, variables = declarations, rank = scale_number)) + push!(entity_sets, EntitySet(set_name; columns = columns, variables = declarations, + rank = get(ranks, scale_number, scale_number))) end relations = Relation[] @@ -204,55 +210,31 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S if scene !== nothing extras["scene"] = json_object("id" => fspm_id(scene), "symbol" => string(symbol(scene)), "index" => index(scene), "link" => string(link(scene))) - columns, declarations, missing_of_scene, text_types_of_scene = - attribute_columns([scene], "scalars", variables) - if !isempty(columns) - scalars = Table("scalars"; columns = columns, variables = declarations) - isempty(missing_of_scene) || (missing_rows["scalars"] = missing_of_scene) - isempty(text_types_of_scene) || (text_types["scalars"] = text_types_of_scene) - end + columns, declarations = attribute_columns([scene], "scalars", variables) + isempty(columns) || (scalars = Table("scalars"; columns = columns, variables = declarations)) end - isempty(missing_rows) || (extras["missing"] = missing_rows) - isempty(text_types) || (extras["text_types"] = text_types) - order = [fspm_id(node) for node in nodes] - order == [fspm_id(node) for node in preorder_by_id(nodes)] || (extras["order"] = order) + extras = merge(json_object("model" => PLATFORM, # each set's MTG scale (its rank may differ) + "scales" => json_object(set_names[scale_number] => scale_number + for scale_number in keys(by_scale))), extras) return DataStructure(id; kind = "graph", entity_sets = entity_sets, relations = relations, scalars = scalars, - platform = platform, model = model, extras = json_object(PLATFORM => extras)) -end - -"""The depth-first order of the same tree, children sorted by id: the order to_mtg rebuilds without `order`.""" -function preorder_by_id(nodes) - ordered, stack = Any[], Any[first(nodes)] - while !isempty(stack) - node = pop!(stack) - push!(ordered, node) - append!(stack, sort(children(node); by = node_id, rev = true)) - end - return ordered + platform = platform, model = model, extras = json_object(EXTRAS_KEY => extras), + labels = labels) end # ── Data structure → MTG ──────────────────────────────────────────────── -attribute_value(column::AbstractVector, row) = column[row] -attribute_value(column::AbstractMatrix, row) = collect(column[row, :]) - -function restore(value, text_type) - text_type === nothing && return value - return FspmTF.TEXT_TYPES[text_type](value) -end - -"""The attributes of each row of `table`: Dict(column => value), without the rows' missing ones.""" -function row_attributes(table, missing_of_table, text_types_of_table, skip) - missing_rows = Dict(name => Set(rows) for (name, rows) in pairs(missing_of_table)) +"""The attributes of each row of `table`: Dict(column => value), without the rows' missing ones (`missing`, and NaN, +a float column's missing value: a node has no such attribute); dates and times read back from their text by their +declared `format`.""" +function row_attributes(table, skip) rows = [Dict{Symbol,Any}() for _ in 1:row_count(table)] for (name, column) in table.columns name in skip && continue - absent = get(missing_rows, name, Set{Int}()) - text_type = get(text_types_of_table, name, nothing) + format = haskey(table.variables, name) ? table.variables[name].format : nothing for row in 1:row_count(table) - (row - 1) in absent && continue - rows[row][Symbol(name)] = restore(attribute_value(column, row), text_type) + value = FspmTF.value_of(column, row, format) + ismissing(value) || (value isa AbstractFloat && isnan(value)) || (rows[row][Symbol(name)] = value) end end return rows @@ -269,24 +251,28 @@ variables as attributes, under a `Scene` root holding the scalars. function FspmTF.to_mtg(data_structure::DataStructure) data_structure.kind == "graph" || throw(ArgumentError("data structure '$(data_structure.id)' is a $(data_structure.kind), not a graph")) - extras = get(data_structure.extras, PLATFORM, json_object()) - missing_rows, text_types = get(extras, "missing", json_object()), get(extras, "text_types", json_object()) + extras = get(data_structure.extras, EXTRAS_KEY, json_object()) + get(extras, "model", nothing) == PLATFORM || (extras = json_object()) # another VPL model's: not MTG.jl's entity_sets = [set for set in data_structure.entity_sets if set.rank !== nothing] sort!(entity_sets; by = set -> set.rank) + mtg_scales = get(extras, "scales", json_object()) # MTG.jl's own: its scales; else the ranks + scale_of(set) = Int(get(mtg_scales, set.name, set.rank)) # Every entity: its scale, symbol, index, attributes; its parent in the tree and the link to it encoding, attributes, tree_parent = Dict{Int64,Any}(), Dict{Int64,Dict{Symbol,Any}}(), Dict{Int64,Tuple{Int64,Symbol}}() reserved = (FspmTF.ENTITY_SET_RESERVED..., INDEX_COLUMN) + order_of = Dict{Int64,Int64}() # each entity's position among its siblings for set in entity_sets ids = set.columns["id"] labels = get(set.columns, "label", fill(set.name, row_count(set))) indices = get(set.columns, INDEX_COLUMN, zeros(Int64, row_count(set))) - rows = row_attributes(set, get(missing_rows, set.name, json_object()), get(text_types, set.name, json_object()), - reserved) + rows = row_attributes(set, reserved) for row in 1:row_count(set) - encoding[ids[row]] = (symbol = labels[row], index = indices[row], scale = set.rank) + encoding[ids[row]] = (symbol = labels[row], index = indices[row], scale = scale_of(set)) attributes[ids[row]] = rows[row] end + haskey(set.columns, "order") && foreach(((id, position),) -> order_of[id] = position, + zip(ids, set.columns["order"])) complexes = get(set.columns, "complex", fill(NO_ID, row_count(set))) for row in 1:row_count(set) # complex: by decomposition, '/' complexes[row] == NO_ID || (tree_parent[ids[row]] = (complexes[row], :/)) @@ -324,8 +310,7 @@ function FspmTF.to_mtg(data_structure::DataStructure) roots = sort([id for id in keys(encoding) if !haskey(tree_parent, id)]) scene_entry = get(extras, "scene", nothing) scene_attributes = data_structure.scalars === nothing ? Dict{Symbol,Any}() : - only(row_attributes(data_structure.scalars, get(missing_rows, "scalars", json_object()), - get(text_types, "scalars", json_object()), ())) + only(row_attributes(data_structure.scalars, ())) # MTG.jl's ids: the entities' own, unless they start below 1 or are sparse; then 1, 2, ... in their order all_ids = sort(collect(keys(encoding))) renumbered = !isempty(all_ids) && (first(all_ids) < 1 || last(all_ids) > 2 * length(all_ids) + 1) @@ -353,10 +338,8 @@ function FspmTF.to_mtg(data_structure::DataStructure) push!(get!(children_of, parent_id, Int64[]), id) end isempty(roots) || (children_of[root_id] = vcat(roots, get(children_of, root_id, Int64[]))) - rank_of = haskey(extras, "order") ? Dict(Int64(id) => position for (position, id) in enumerate(extras["order"])) : - nothing - for list in values(children_of) - rank_of === nothing ? sort!(list) : sort!(list; by = id -> get(rank_of, id, typemax(Int))) + for list in values(children_of) # the `order` column, else by id + sort!(list; by = id -> (get(order_of, id, typemax(Int64)), id)) end stack = Any[root] while !isempty(stack) diff --git a/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl index a9e9ffc..656bff3 100644 --- a/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl @@ -6,13 +6,13 @@ A VirtualPlantLab graph (PlantGraphs.jl: a `Graph` or a `StaticGraph`) as an fsp | one entity set (default `Organ`, rank 5) | the graph's nodes, but the turtle's movements (PlantGeomTurtle: `T`, `RU`, `F`, `SET`, …) | | `id`, `label` | the node's id, the name of its type (`Internode`, `Leaf`, …) | | `parent` | its nearest kept ancestor | -| `edge_type` | guessed: PlantGraphs stores none; a node's last kept child follows it (`successor`), the others branch | -| variables | the fields of each node's type (missing in other types: see below) | +| `edge_type` | `unknown`: PlantGraphs stores no edge types | +| `order` | the node's position among its kept parent's kept children | +| variables | the fields of each node's type: missing where a type has no such field; dates as ISO 8601 text with their `format` | | scalars | the graph's data (`data(graph)`), its fields | -PlantGraphs has one scale. What fspm-TF 0.1 cannot say is kept in `extras["VirtualPlantLab"]`: the rows where a -node's type has no such field, the types written as text (dates), each type's field names (to rebuild nodes with -`to_graph`), and that edge types were guessed. +PlantGraphs has one scale. Each type's field names, to rebuild nodes with `to_graph`, are kept in +`extras["virtualplantlab"]` (with `"model": "PlantGraphs"`). """ module FspmTFPlantGraphsExt @@ -22,6 +22,7 @@ using OrderedCollections: OrderedDict import PlantGraphs as PG const PLATFORM = "VirtualPlantLab" +const EXTRAS_KEY = "virtualplantlab" # the extras key: VirtualPlantLab's name in the vocabulary const IDENTIFIER = r"^[a-zA-Z_][a-zA-Z0-9_]*$" """Whether a node is kept: the turtle's movements and state nodes (PlantGeomTurtle's types) are not.""" @@ -32,14 +33,15 @@ label_of(node_data) = string(nameof(typeof(node_data))) field_values(object) = object === nothing ? Pair{Symbol,Any}[] : [name => getfield(object, name) for name in fieldnames(typeof(object))] -"""Columns of the objects' fields, by name in order of first appearance; their missing rows and text types.""" +"""Columns of the objects' fields (missing where an object has none), by name in order of first appearance, and their +declarations (a date's `format` added).""" function field_columns(objects, set_name, variables) names = Symbol[] values_of = [Dict{Symbol,Any}(field_values(object)) for object in objects] for object in objects, (name, _) in field_values(object) name in names || push!(names, name) end - columns, declarations, missing_rows, text_types = Columns(), Declarations(), json_object(), json_object() + columns, declarations = Columns(), Declarations() for name in names column_name = string(name) if !occursin(IDENTIFIER, column_name) || column_name in FspmTF.ENTITY_SET_RESERVED @@ -52,13 +54,16 @@ function field_columns(objects, set_name, variables) "fixed-size numeric vectors" continue end - column, missing_of_column, text_type = converted + column, format = converted columns[column_name] = column - isempty(missing_of_column) || (missing_rows[column_name] = missing_of_column) - text_type === nothing || (text_types[column_name] = text_type) - haskey(variables, column_name) && (declarations[column_name] = variables[column_name]) + declared = get(variables, column_name, nothing) + if format !== nothing # dates and times: their format declared + declared = declared === nothing ? Variable(unit = "1", kind = "descriptor") : deepcopy(declared) + declared.format = format + end + declared === nothing || (declarations[column_name] = declared) end - return columns, declarations, missing_rows, text_types + return columns, declarations end static_of(graph::PG.Graph) = PG.static_graph(graph) @@ -85,58 +90,53 @@ function kept_nodes(static, keep) end """ - from_graph(graph; id="plant", entity_set="Organ", rank=5, variables=Dict(), keep=default_keep, - platform="VirtualPlantLab", model=nothing) -> DataStructure + from_graph(graph; id="plant", entity_set="Organ", rank=5, scale=nothing, variables=Dict(), labels=Dict(), + keep=default_keep, platform="VirtualPlantLab", model=nothing) -> DataStructure -The PlantGraphs graph as a `graph` data structure of one entity set. `variables`: the fields' declarations (name => -`Variable`). `keep`: which nodes (by their data) are entities; by default all but the turtle's movements. +The PlantGraphs graph as a `graph` data structure of one entity set. `scale`: its base scale (e.g. `"Organ"`), or +`"mixed"`. `variables`: the fields' declarations (name => `Variable`). `labels`: the node types' names => the +vocabulary's labels (e.g. `Dict("Internode" => "StemInternode")`). `keep`: which nodes (by their data) are entities; +by default all but the turtle's movements. """ function FspmTF.from_graph(graph::Union{PG.Graph,PG.StaticGraph}; id::AbstractString = "plant", - entity_set::AbstractString = "Organ", rank = 5, variables = Dict{String,Variable}(), - keep = default_keep, platform = PLATFORM, model = nothing) + entity_set::AbstractString = "Organ", rank = 5, scale = nothing, + variables = Dict{String,Variable}(), labels = Dict{String,String}(), keep = default_keep, + platform = PLATFORM, model = nothing) check_identifier(entity_set, "entity set name") variables = Dict{String,Variable}(string(name) => variable for (name, variable) in pairs(variables)) static = static_of(graph) rows = kept_nodes(static, keep) ids = Int64[PG.self_id(node) for (node, _) in rows] parents = Int64[parent for (_, parent) in rows] - last_child = Dict{Int64,Int64}() # each parent's last kept child - for (child, parent) in zip(ids, parents) - parent == NO_ID || (last_child[parent] = child) - end - edge_types = String[parent == NO_ID ? "none" : last_child[parent] == child ? "successor" : "branch" - for (child, parent) in zip(ids, parents)] + siblings_seen = Dict{Int64,Int64}() # each parent's kept children so far + order = Int64[parent == NO_ID ? 0 : (siblings_seen[parent] = get(siblings_seen, parent, -1) + 1) + for parent in parents] + edge_types = String[parent == NO_ID ? "none" : "unknown" for parent in parents] # PlantGraphs stores none objects = [PG.data(node) for (node, _) in rows] columns = Columns("id" => ids, "label" => String[label_of(object) for object in objects], "parent" => parents, - "edge_type" => edge_types) - fields, declarations, missing_rows, text_types = field_columns(objects, entity_set, variables) + "edge_type" => edge_types, "order" => order) + fields, declarations = field_columns(objects, entity_set, variables) merge!(columns, fields) - extras = json_object("edge_types" => "guessed: PlantGraphs stores none; a node's last child follows it, the others branch", + extras = json_object("model" => "PlantGraphs", "types" => json_object(label_of(object) => string.(fieldnames(typeof(object))) for object in unique(object -> typeof(object), objects))) - isempty(missing_rows) || (extras["missing"] = json_object(entity_set => missing_rows)) - isempty(text_types) || (extras["text_types"] = json_object(entity_set => text_types)) scalars = nothing data = graph_data(graph) if data !== nothing && !isempty(fieldnames(typeof(data))) - scalar_columns, scalar_declarations, missing_scalars, text_scalars = field_columns([data], "scalars", variables) + scalar_columns, scalar_declarations = field_columns([data], "scalars", variables) if !isempty(scalar_columns) scalars = Table("scalars"; columns = scalar_columns, variables = scalar_declarations) extras["data_type"] = string(nameof(typeof(data))) - isempty(text_scalars) || (extras["text_types"] = merge(get(extras, "text_types", json_object()), - json_object("scalars" => text_scalars))) end end - set = EntitySet(entity_set; columns = columns, variables = declarations, rank = rank) + set = EntitySet(entity_set; columns = columns, variables = declarations, rank = rank, scale = scale) return DataStructure(id; kind = "graph", entity_sets = [set], scalars = scalars, platform = platform, - model = model, extras = json_object(PLATFORM => extras)) + model = model, extras = json_object(EXTRAS_KEY => extras), labels = labels) end # ── Data structure → graph ────────────────────────────────────────────── -row_value(column::AbstractVector, row) = column[row] -row_value(column::AbstractMatrix, row) = collect(column[row, :]) """ to_graph(data_structure; types, entity_set=nothing) -> (StaticGraph, ids) @@ -152,9 +152,6 @@ function FspmTF.to_graph(data_structure::DataStructure; types::AbstractDict, ent (isempty(candidates) ? throw(ArgumentError("'$(data_structure.id)': no entity set with parents")) : last(sort(candidates; by = set -> something(set.rank, 0)))) : FspmTF.entity_set(data_structure, entity_set) - extras = get(data_structure.extras, PLATFORM, json_object()) - missing_rows = Dict(name => Set(rows) for (name, rows) in pairs(get(get(extras, "missing", json_object()), set.name, json_object()))) - text_types = get(get(extras, "text_types", json_object()), set.name, json_object()) labels = set.columns["label"] objects = Vector{Any}(undef, row_count(set)) for row in 1:row_count(set) @@ -162,10 +159,10 @@ function FspmTF.to_graph(data_structure::DataStructure; types::AbstractDict, ent node_type = types[labels[row]] arguments = map(fieldnames(node_type)) do field name = string(field) - haskey(set.columns, name) && !((row - 1) in get(missing_rows, name, Set{Int}())) || - throw(ArgumentError("'$(labels[row])' row $row: no value for field '$name'")) - value = row_value(set.columns[name], row) - haskey(text_types, name) ? FspmTF.TEXT_TYPES[text_types[name]](value) : value + format = haskey(set.variables, name) ? set.variables[name].format : nothing + value = haskey(set.columns, name) ? FspmTF.value_of(set.columns[name], row, format) : missing + ismissing(value) && throw(ArgumentError("'$(labels[row])' row $row: no value for field '$name'")) + value end objects[row] = node_type(arguments...) end @@ -175,6 +172,9 @@ function FspmTF.to_graph(data_structure::DataStructure; types::AbstractDict, ent for row in 1:row_count(set) parents[row] == NO_ID ? push!(roots, row) : push!(get!(children_of, parents[row], Int[]), row) end + if haskey(set.columns, "order") # children in their order, else in their rows' + foreach(list -> sort!(list; by = row -> set.columns["order"][row]), values(children_of)) + end length(roots) == 1 || throw(ArgumentError("'$(set.name)': $(length(roots)) roots, a PlantGraphs graph has one")) static = PG.StaticGraph(PG.GraphNode(objects[only(roots)])) graph_ids = Dict{Int64,Int}(ids[only(roots)] => PG.self_id(PG.getroot(static))) diff --git a/bindings/julia/FspmTF/src/FspmTF.jl b/bindings/julia/FspmTF/src/FspmTF.jl index 2620bb7..688cda1 100644 --- a/bindings/julia/FspmTF/src/FspmTF.jl +++ b/bindings/julia/FspmTF/src/FspmTF.jl @@ -21,7 +21,7 @@ using JSON using OrderedCollections: OrderedDict """The version of the specification this package reads and writes.""" -const SPECIFICATION_VERSION = "0.1" +const SPECIFICATION_VERSION = "0.2" include("glb.jl") include("model.jl") diff --git a/bindings/julia/FspmTF/src/codec.jl b/bindings/julia/FspmTF/src/codec.jl index de6de16..19d8138 100644 --- a/bindings/julia/FspmTF/src/codec.jl +++ b/bindings/julia/FspmTF/src/codec.jl @@ -64,9 +64,12 @@ function to_bytes(frame::Frame) data_structure.platform === nothing || (entry["platform"] = data_structure.platform) data_structure.model === nothing || (entry["model"] = data_structure.model) entry["origin"] = vec(data_structure.origin) # column-major, as glTF matrices - entry["entitySets"] = [json_object("name" => set.name, "rank" => set.rank, - "propertyTable" => add_table(set, "$(data_structure.id)__$(set.name)")) - for set in data_structure.entity_sets] + entry["entitySets"] = map(data_structure.entity_sets) do set + set_entry = json_object("name" => set.name, "rank" => set.rank) + set.scale === nothing || (set_entry["scale"] = set.scale) + set_entry["propertyTable"] = add_table(set, "$(data_structure.id)__$(set.name)") + set_entry + end if !isempty(data_structure.relations) entry["relations"] = [json_object("name" => relation.name, "sourceSet" => relation.source_set, "targetSet" => relation.target_set, @@ -76,6 +79,7 @@ function to_bytes(frame::Frame) data_structure.scalars === nothing || (entry["scalars"] = add_table(data_structure.scalars, "$(data_structure.id)__scalars")) data_structure.grid === nothing || (entry["grid"] = grid_json(data_structure.grid)) + isempty(data_structure.labels) || (entry["labels"] = data_structure.labels) isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) node = json_object("name" => data_structure.id) data_structure.origin == identity_matrix() || (node["matrix"] = entry["origin"]) @@ -158,6 +162,7 @@ function declaration_json(variable::Variable) variable.maximum === nothing || (declaration["maximum"] = variable.maximum) isempty(variable.references) || (declaration["references"] = variable.references) isempty(variable.doi) || (declaration["doi"] = variable.doi) + variable.format === nothing || (declaration["format"] = variable.format) return declaration end @@ -173,7 +178,8 @@ function encode_table(table::AbstractTable, class_id::AbstractString, buffers::B description = declaration(table, name).description property = json_object() description === nothing || (property["description"] = description) - element = eltype(column) + element = nonmissingtype(eltype(column)) + has_missing = Missing <: eltype(column) && any(ismissing, column) if name in ENUM_COLUMNS && name in reserved_names enum_name, codes = enum_codes(name, column, class_id, schema) property["type"], property["enumType"] = "ENUM", enum_name @@ -182,12 +188,25 @@ function encode_table(table::AbstractTable, class_id::AbstractString, buffers::B column isa AbstractVector || throw(ArgumentError("column '$name' of '$(table.name)': text vectors are not supported")) property["type"] = "STRING" + if has_missing # missing text: noData, a string the column lacks + no_data = free_string(column) + property["noData"] = no_data + column = String[ismissing(value) ? no_data : value for value in column] + end encoded = [Vector{UInt8}(codeunits(value)) for value in column] offsets = vcat(0, cumsum(Int64[length(value) for value in encoded])) offset_type = offsets[end] < 2^32 ? UInt32 : UInt64 stored[name] = json_object("values" => add!(buffers, reduce(vcat, encoded; init = UInt8[])), "stringOffsets" => add!(buffers, offset_type.(offsets)), "stringOffsetType" => uppercase(string(offset_type))) + elseif element === Bool && has_missing # booleans with missing values: UINT8 0, 1, 255 missing + column isa AbstractVector || + throw(ArgumentError("column '$name' of '$(table.name)': boolean vectors are not supported")) + property["type"], property["componentType"] = "SCALAR", "UINT8" + property["noData"] = BOOLEAN_NO_DATA + property["extras"] = json_object("FSPM_boolean" => true) + stored[name] = json_object("values" => add!(buffers, UInt8[ismissing(value) ? BOOLEAN_NO_DATA : UInt8(value) + for value in column])) elseif element === Bool column isa AbstractVector || throw(ArgumentError("column '$name' of '$(table.name)': boolean vectors are not supported")) @@ -228,6 +247,10 @@ function encode_table(table::AbstractTable, class_id::AbstractString, buffers::B end elseif name in ("complex", "parent") && name in reserved_names property["noData"] = NO_ID + elseif has_missing # missing integers: noData, a value the column lacks + no_data = free_integer(element, column) + property["noData"] = no_data + data = element[ismissing(value) ? no_data : value for value in column] end stored[name] = json_object("values" => add!(buffers, row_major(data))) end @@ -237,6 +260,32 @@ function encode_table(table::AbstractTable, class_id::AbstractString, buffers::B return properties, stored end +const BOOLEAN_NO_DATA = 0xff # a boolean column with missing values: UINT8 0, 1, and 255 missing + +"""A value of type `T` the column does not hold: its minimum (signed) or maximum (unsigned) first, then inwards.""" +function free_integer(::Type{T}, column) where {T<:Integer} + present = Set(value for value in column if !ismissing(value)) + first, second = T <: Signed ? (typemin(T), typemax(T)) : (typemax(T), typemin(T)) + for candidate in (first, second) + candidate in present || return candidate + end + candidate = first + while candidate in present # at most length(present) steps + candidate = T <: Signed ? candidate + one(T) : candidate - one(T) + end + return candidate +end + +"""A string the column does not hold: the empty string first, then NUL characters.""" +function free_string(column) + present = Set(value for value in column if !ismissing(value)) + candidate = "" + while candidate in present + candidate *= "\0" + end + return candidate +end + function enum_codes(name, column, class_id, schema) if name == "edge_type" unknown = setdiff(Set(column), Set(EDGE_TYPES)) @@ -338,12 +387,13 @@ function from_bytes(data::AbstractVector{UInt8}) origin = haskey(entry, "origin") ? reshape(Float64.(entry["origin"]), 4, 4) : identity_matrix() data_structure = DataStructure(entry["id"]; kind = get(entry, "kind", "graph"), platform = get(entry, "platform", nothing), model = get(entry, "model", nothing), - extras = get(entry, "extras", json_object()), origin = origin) + extras = get(entry, "extras", json_object()), origin = origin, + labels = get(entry, "labels", json_object())) for item in get(entry, "entitySets", Any[]) - rank = get(item, "rank", nothing) + rank, scale = get(item, "rank", nothing), get(item, "scale", nothing) push!(data_structure.entity_sets, table(item["propertyTable"], (name, columns, declarations) -> - EntitySet(name, columns, declarations, rank === nothing ? nothing : Int(rank)))) + EntitySet(name, columns, declarations, rank === nothing ? nothing : Int(rank), scale))) end for item in get(entry, "relations", Any[]) push!(data_structure.relations, @@ -403,6 +453,9 @@ function decode_table(document, binary, description, schema, variables) offset_type = TYPES_OF_COMPONENTS[get(stored, "stringOffsetType", "UINT32")] offsets = read_values(offset_type, view_bytes(document, binary, stored["stringOffsets"]), count + 1) column = String[String(raw[offsets[row]+1:offsets[row+1]]) for row in 1:count] + if haskey(property, "noData") && any(==(property["noData"]), column) # missing text + column = Union{Missing,String}[value == property["noData"] ? missing : value for value in column] + end elseif type == "BOOLEAN" column = Bool[(raw[(row-1)÷8+1] >> ((row - 1) % 8)) & 0x01 == 0x01 for row in 1:count] elseif type == "SCALAR" @@ -425,6 +478,15 @@ function decode_table(document, binary, description, schema, variables) end end column = vector ? permutedims(reshape(values, width, count)) : values + if get(get(property, "extras", json_object()), "FSPM_boolean", false) # booleans with missing values + no_data = get(property, "noData", BOOLEAN_NO_DATA) + column = any(==(no_data), column) ? + Union{Missing,Bool}[value == no_data ? missing : value == 1 for value in column] : + Bool[value == 1 for value in column] + elseif element <: Integer && !vector && haskey(property, "noData") && !(name in ("complex", "parent")) && + any(==(property["noData"]), column) # missing integers + column = Union{Missing,element}[value == property["noData"] ? missing : value for value in column] + end else throw(ArgumentError("'$class_id.$name': property type $type is not part of fspm-TF 0.1")) end @@ -439,7 +501,8 @@ function decode_table(document, binary, description, schema, variables) minimum = get(declared, "minimum", nothing), maximum = get(declared, "maximum", nothing), references = String.(get(declared, "references", String[])), - doi = String.(get(declared, "doi", String[]))) + doi = String.(get(declared, "doi", String[])), + format = get(declared, "format", nothing)) end end return columns, declarations diff --git a/bindings/julia/FspmTF/src/dump.jl b/bindings/julia/FspmTF/src/dump.jl index 0742b3a..6648ee5 100644 --- a/bindings/julia/FspmTF/src/dump.jl +++ b/bindings/julia/FspmTF/src/dump.jl @@ -3,6 +3,7 @@ json_value(value::AbstractFloat) = isnan(value) ? nothing : isinf(value) ? (value > 0 ? "Infinity" : "-Infinity") : Float64(value) +json_value(::Missing) = nothing json_value(value) = value json_column(column::AbstractVector) = [json_value(value) for value in column] @@ -10,7 +11,10 @@ json_column(column::AbstractMatrix) = [[json_value(value) for value in row] for """The name of a column's type in the expected values: numpy's (`float64`, `int64`, `uint8`, `bool`...), or `string`.""" -dtype_name(column::AbstractArray) = eltype(column) <: AbstractString ? "string" : lowercase(string(eltype(column))) +function dtype_name(column::AbstractArray) + element = nonmissingtype(eltype(column)) + return element <: AbstractString ? "string" : lowercase(string(element)) +end function table_dict(table::AbstractTable) entry = json_object("name" => table.name, "count" => row_count(table), @@ -27,6 +31,7 @@ function table_dict(table::AbstractTable) variable.maximum === nothing || (declared["maximum"] = variable.maximum) isempty(variable.references) || (declared["references"] = variable.references) isempty(variable.doi) || (declared["doi"] = variable.doi) + variable.format === nothing || (declared["format"] = variable.format) declarations[name] = declared end isempty(declarations) || (entry["variables"] = declarations) @@ -43,11 +48,14 @@ function frame_to_dict(frame::Frame) for data_structure in frame.data_structures entry = json_object("id" => data_structure.id, "kind" => data_structure.kind, "origin" => [collect(row) for row in eachrow(data_structure.origin)], - "entity_sets" => [merge!(table_dict(set), json_object("rank" => set.rank)) + "entity_sets" => [merge!(table_dict(set), json_object("rank" => set.rank), + set.scale === nothing ? json_object() : + json_object("scale" => set.scale)) for set in data_structure.entity_sets]) data_structure.platform === nothing || (entry["platform"] = data_structure.platform) data_structure.model === nothing || (entry["model"] = data_structure.model) isempty(data_structure.extras) || (entry["extras"] = data_structure.extras) + isempty(data_structure.labels) || (entry["labels"] = data_structure.labels) if !isempty(data_structure.relations) entry["relations"] = [merge!(table_dict(relation), json_object("source_set" => relation.source_set, "target_set" => relation.target_set)) diff --git a/bindings/julia/FspmTF/src/model.jl b/bindings/julia/FspmTF/src/model.jl index 3a52deb..ef465d0 100644 --- a/bindings/julia/FspmTF/src/model.jl +++ b/bindings/julia/FspmTF/src/model.jl @@ -10,14 +10,20 @@ # Mapping source entity ↔ target entity weights between two data structures # # A column is a Vector (one value per row: numbers, Bool, String) or a Matrix (one row per entity, one column per -# component: a fixed-size vector per entity). +# component: a fixed-size vector per entity). Missing values: NaN in float columns; `missing` elsewhere (a +# Vector{Union{Missing,T}} of integers, booleans or text). """The id of no entity: the `complex` or `parent` of an entity that has none, the feature row of a vertex with none.""" const NO_ID = -1 """The values of `edge_type`: the root of its scale, MTG `<`, MTG `+`.""" -const EDGE_TYPES = ("none", "successor", "branch") +const EDGE_TYPES = ("none", "successor", "branch", "unknown") const ENUM_COLUMNS = ("label", "edge_type", "relation") -const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index") +const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "order") +"""The vocabulary's base scales: an entity set's `scale` is one of them, or `"mixed"`.""" +const BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", + "Connection") +"""The formats of a text variable holding ISO 8601 dates or times.""" +const FORMATS = ("date", "date-time", "time") const DATA_STRUCTURE_KINDS = ("graph", "regular_grid", "adaptive_grid", "table") const ROLES = ("state", "input", "parameter", "derived", "setting") const COLUMN_TYPES = (Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64) @@ -27,7 +33,8 @@ const COLUMN_TYPES = (Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, minimum=nothing, maximum=nothing, references=String[], doi=String[]) The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its kind -(`extensive`, `intensive`, ...: how it maps between scales and grids), its role and its provenance. +(`extensive`, `intensive`, ...: how it maps between scales and grids), its role and its provenance; the `format` of +a text column holding dates or times (`"date"`, `"date-time"`, `"time"`). """ Base.@kwdef mutable struct Variable unit::String = "1" @@ -39,6 +46,7 @@ Base.@kwdef mutable struct Variable maximum::Union{Nothing,Real} = nothing references::Vector{String} = String[] doi::Vector{String} = String[] + format::Union{Nothing,String} = nothing end function Base.:(==)(left::Variable, right::Variable) @@ -70,13 +78,16 @@ end The entities of one scale of a graph (`rank`: the scale's rank, coarser first), or a grid's cells (`rank` nothing). Reserved columns: `id` (Int64, stable across frames, never reused), `label` (String), `complex` (the id of the containing entity at a coarser scale, `NO_ID` if none), `parent` (the predecessor's id at the same scale, `NO_ID` if -none), `edge_type` (`"none"`, `"successor"`, `"branch"`); an adaptive grid's cells: `level` and `index`. +none), `edge_type` (`"none"`, `"successor"`, `"branch"`, `"unknown"`), `order` (optional: its position among its tree +parent's children); an adaptive grid's cells: `level` and `index`. `scale`: its base scale (`BASE_SCALES`), or +`"mixed"`. """ mutable struct EntitySet <: AbstractTable name::String columns::Columns variables::Declarations rank::Union{Nothing,Int} + scale::Union{Nothing,String} end """ @@ -109,7 +120,7 @@ mutable struct Mapping <: AbstractTable end # The columns whose meaning the specification fixes, per kind of table -const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index") +const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index", "order") const RELATION_RESERVED = ("id", "source", "target", "relation") const MAPPING_RESERVED = ("source", "target", "weight") reserved(::Table) = () @@ -125,12 +136,18 @@ as_column(name::AbstractString, values::Number) = [values] as_column(name::AbstractString, values::AbstractString) = [String(values)] function as_column(name::AbstractString, values::AbstractVector) isempty(values) && return Float64[] - if all(value -> value isa AbstractString, values) - return Vector{String}(values) + present = [value for value in values if !ismissing(value)] + if all(value -> value isa AbstractString, present) + return length(present) == length(values) ? Vector{String}(values) : + Union{Missing,String}[ismissing(value) ? missing : String(value) for value in values] end - promoted = [values...] # the narrowest common type - eltype(promoted) <: Union{Number,Bool} && return promoted - throw(ArgumentError("column '$name': numbers, booleans or strings per row (got $(eltype(promoted)))")) + promoted = isempty(present) ? Float64[] : [present...] # the narrowest common type + element = eltype(promoted) + element <: Union{Number,Bool} || + throw(ArgumentError("column '$name': numbers, booleans or strings per row (got $element)")) + length(present) == length(values) && return promoted + element <: AbstractFloat && return element[ismissing(value) ? NaN : value for value in values] # missing: NaN + return Union{Missing,element}[value for value in values] end as_column(name::AbstractString, values) = throw(ArgumentError("column '$name': a vector, or a matrix of one row per entity (got $(typeof(values)))")) @@ -143,6 +160,8 @@ function prepare_columns(table_name::AbstractString, columns, reserved_names) if name in ID_COLUMNS && name in reserved_names && eltype(column) <: Integer && !(eltype(column) <: Bool) column = convert(Array{Int64}, column) # ids are Int64, whatever the platform's integers end + name in reserved_names && Missing <: eltype(column) && + throw(ArgumentError("table '$table_name': the reserved column '$name' has no missing values")) prepared[name] = column end counts = Dict(name => size(column, 1) for (name, column) in prepared) @@ -155,8 +174,9 @@ prepare_variables(variables) = Declarations(String(name) => variable for (name, Table(name::AbstractString; columns = Columns(), variables = Declarations()) = Table(String(name), prepare_columns(name, columns, ()), prepare_variables(variables)) -EntitySet(name::AbstractString; columns = Columns(), variables = Declarations(), rank = nothing) = - EntitySet(String(name), prepare_columns(name, columns, ENTITY_SET_RESERVED), prepare_variables(variables), rank) +EntitySet(name::AbstractString; columns = Columns(), variables = Declarations(), rank = nothing, scale = nothing) = + EntitySet(String(name), prepare_columns(name, columns, ENTITY_SET_RESERVED), prepare_variables(variables), rank, + scale) Relation(name::AbstractString; columns = Columns(), variables = Declarations(), source_set = "", target_set = "") = Relation(String(name), prepare_columns(name, columns, RELATION_RESERVED), prepare_variables(variables), String(source_set), String(target_set)) @@ -236,8 +256,9 @@ end meshes=Mesh[], origin=identity, platform=nothing, model=nothing, extras=OrderedDict()) One plant population's graph, or one environment model's grid. `origin`: its frame in the scene (a 4×4 matrix, -`origin[row, column]`, or 16 numbers row by row). `extras`: JSON values for the writing platform (e.g. what it needs to rebuild its own data -structure); other readers keep them without interpreting them. +`origin[row, column]`, or 16 numbers row by row). `extras`: JSON values for the writing platform (e.g. what it needs +to rebuild its own data structure); other readers keep them without interpreting them. `labels`: the platform's +labels => the vocabulary's. """ mutable struct DataStructure id::String @@ -251,17 +272,20 @@ mutable struct DataStructure platform::Union{Nothing,String} model::Union{Nothing,String} extras::AbstractDict{String,Any} + labels::AbstractDict{String,String} end identity_matrix() = Matrix{Float64}([i == j for i in 1:4, j in 1:4]) function DataStructure(id::AbstractString; kind = "graph", entity_sets = EntitySet[], relations = Relation[], scalars = nothing, grid = nothing, meshes = Mesh[], origin = identity_matrix(), - platform = nothing, model = nothing, extras = OrderedDict{String,Any}()) + platform = nothing, model = nothing, extras = OrderedDict{String,Any}(), + labels = OrderedDict{String,String}()) origin = Matrix{Float64}(origin isa AbstractVector ? permutedims(reshape(origin, 4, 4)) : origin) size(origin) == (4, 4) || throw(ArgumentError("data structure '$id': the origin is a 4×4 matrix")) DataStructure(String(id), String(kind), collect(EntitySet, entity_sets), collect(Relation, relations), scalars, - grid, collect(Mesh, meshes), origin, platform, model, extras) + grid, collect(Mesh, meshes), origin, platform, model, extras, + OrderedDict{String,String}(String(key) => String(value) for (key, value) in pairs(labels))) end """The entity set `name` of a data structure.""" diff --git a/bindings/julia/FspmTF/src/values.jl b/bindings/julia/FspmTF/src/values.jl index 02f84f3..1213373 100644 --- a/bindings/julia/FspmTF/src/values.jl +++ b/bindings/julia/FspmTF/src/values.jl @@ -1,36 +1,56 @@ # Values of a platform's objects (attributes, fields) as columns: what the adapters share. const TextType = Union{Date,DateTime,Time} -"""The types written as ISO 8601 text, by name (an adapter records the name to read them back).""" -const TEXT_TYPES = Dict("Date" => Date, "DateTime" => DateTime, "Time" => Time) +"""The format (a variable's `format`) of each type written as ISO 8601 text.""" +const FORMAT_OF = Dict{DataType,String}(Date => "date", DateTime => "date-time", Time => "time") +"""Each format's type, to read its text back.""" +const TYPE_OF_FORMAT = Dict{String,DataType}(format => type for (type, format) in FORMAT_OF) -"""A column of `values` (`nothing` where a node has no such attribute): the column, its missing rows (0-based), -the type written as text (or nothing); nothing for values fspm-TF cannot hold.""" +""" + column_of_values(values) -> (column, format) or nothing + +A column of `values`, `nothing` where an object has no such value: missing (NaN in a float column, `missing` +otherwise). `format`: the variable's format when the values are dates or times written as ISO 8601 text, else +nothing. Nothing for values fspm-TF cannot hold. +""" function column_of_values(values::AbstractVector) present = [value for value in values if value !== nothing] - missing_rows = [row - 1 for (row, value) in enumerate(values) if value === nothing] - text_type = nothing + complete = length(present) == length(values) + column_of(type) = complete ? type[value for value in values] : + Union{Missing,type}[value === nothing ? missing : value for value in values] + format = nothing if all(value -> value isa Bool, present) - column = Bool[value === nothing ? false : value for value in values] + column = column_of(Bool) elseif all(value -> value isa Integer && !(value isa Bool), present) - column = Int64[value === nothing ? 0 : value for value in values] + column = column_of(Int64) elseif all(value -> value isa Real && !(value isa Bool), present) column = Float64[value === nothing ? NaN : value for value in values] elseif all(value -> value isa Union{AbstractString,Symbol}, present) - column = String[value === nothing ? "" : string(value) for value in values] + column = complete ? String[string(value) for value in values] : + Union{Missing,String}[value === nothing ? missing : string(value) for value in values] elseif !isempty(present) && all(value -> value isa TextType, present) && length(Set(typeof.(present))) == 1 - text_type = string(nameof(typeof(first(present)))) - column = String[value === nothing ? "" : string(value) for value in values] + format = FORMAT_OF[typeof(first(present))] + column = complete ? String[string(value) for value in values] : + Union{Missing,String}[value === nothing ? missing : string(value) for value in values] elseif all(value -> value isa AbstractVector{<:Real}, present) && length(Set(length.(present))) == 1 && length(first(present)) >= 2 width = length(first(present)) - integer = all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) + integer = complete && all(value -> eltype(value) <: Integer && !(eltype(value) <: Bool), present) column = integer ? zeros(Int64, length(values), width) : fill(NaN, length(values), width) for (row, value) in enumerate(values) - value === nothing || (column[row, :] .= value) + value === nothing || (column[row, :] .= value) # a missing vector: NaN end else return nothing end - return column, missing_rows, text_type + return column, format +end + +"""A column's value at `row` as a platform reads it back: `missing` stays missing, ISO 8601 text of a `format` its +date or time, a vector row a Vector.""" +function value_of(column::AbstractVector, row, format) + value = column[row] + ismissing(value) && return missing + return format === nothing ? value : TYPE_OF_FORMAT[format](value) end +value_of(column::AbstractMatrix, row, format) = collect(column[row, :]) diff --git a/bindings/julia/FspmTF/test/Project.toml b/bindings/julia/FspmTF/test/Project.toml index 5e2d591..d7fc633 100644 --- a/bindings/julia/FspmTF/test/Project.toml +++ b/bindings/julia/FspmTF/test/Project.toml @@ -1,4 +1,5 @@ [deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/bindings/julia/FspmTF/test/plantgraphs/runtests.jl b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl index c4a369f..00b7135 100644 --- a/bindings/julia/FspmTF/test/plantgraphs/runtests.jl +++ b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl @@ -42,10 +42,11 @@ plant() = Graph(axiom = Internode(0.1, 0.01) + (RU(30.0) + Leaf(0.05, 0.02, 3), @test organs.columns["label"] == ["Internode", "Leaf", "Leaf", "Internode", "Meristem"] # no turtle moves ids = organs.columns["id"] @test organs.columns["parent"] == [-1, ids[1], ids[1], ids[1], ids[4]] # the leaves on the first internode - @test organs.columns["edge_type"] == ["none", "branch", "branch", "successor", "successor"] + @test organs.columns["edge_type"] == ["none", "unknown", "unknown", "unknown", "unknown"] # none stored + @test organs.columns["order"] == [0, 0, 1, 2, 0] # the leaves, then the internode @test isequal(organs.columns["length"], [0.1, 0.05, 0.04, 0.08, NaN]) # NaN: the meristem has none - @test organs.columns["age"] == [0, 3, 1, 0, 0] # missing where no field - @test written.extras["VirtualPlantLab"]["missing"]["Organ"]["age"] == [0, 3, 4] + @test isequal(organs.columns["age"], [missing, 3, 1, missing, missing]) # missing where no field + @test written.extras["virtualplantlab"]["model"] == "PlantGraphs" @test written.scalars.columns["temperature"] == [20.5] frame = from_bytes(to_bytes(Frame(data_structures = [written]))) @@ -56,7 +57,7 @@ plant() = Graph(axiom = Internode(0.1, 0.01) + (RU(30.0) + Leaf(0.05, 0.02, 3), again_organs = entity_set(again, "Organ") @test again_organs.columns["label"] == organs.columns["label"] @test isequal(again_organs.columns["length"], organs.columns["length"]) - @test again_organs.columns["age"] == organs.columns["age"] + @test isequal(again_organs.columns["age"], organs.columns["age"]) @test again_organs.columns["edge_type"] == organs.columns["edge_type"] @test again_organs.columns["parent"] == [-1; [graph_ids[parent] for parent in organs.columns["parent"][2:end]]] end diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl index b03aece..e254a47 100644 --- a/bindings/julia/FspmTF/test/runtests.jl +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -1,3 +1,4 @@ +using Dates using FspmTF using JSON using MultiScaleTreeGraph @@ -171,6 +172,23 @@ expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] end end + @testset "MultiScaleTreeGraph: missing attributes, dates, the order of children, MTG.jl's extras" begin + mtg = read_mtg(joinpath(MTG_FILES, "simple_plant.mtg")) + written = from_mtg(mtg; scales = Dict(3 => "Organ"), ranks = Dict(3 => 5)) + organs = entity_set(written, "Organ") + @test organs.rank == 5 # the vocabulary's position + @test isequal(organs.columns["isAlive"], [missing, false, true, true]) # Internode, Leaf, Internode, Leaf + @test organs.variables["dateDeath"].format == "date" + @test isequal(organs.columns["dateDeath"], [missing, "2022-08-24", missing, missing]) + @test organs.columns["order"] == [0, 0, 1, 0] # the leaf before the next internode + @test organs.variables["mtg_index"].standard_name == "mtg_index" + @test Set(keys(written.extras["virtualplantlab"])) == Set(["model", "scales", "metadata", "scene"]) + rebuilt = to_mtg(from_bytes(to_bytes(Frame(data_structures = [written]))).data_structures[1]) + @test scale(get_node(rebuilt, 4)) == 3 # its MTG scale, not its rank + @test attribute(get_node(rebuilt, 5), :dateDeath) == Date(2022, 8, 24) + @test !(:isAlive in attribute_names(get_node(rebuilt, 4))) # missing: no attribute + end + @testset "MultiScaleTreeGraph: the bearer of a branch at another scale" begin scene = Node(1, MutableNodeMTG(:/, :Scene, 0, 0), Dict{Symbol,Any}()) plant = Node(2, scene, MutableNodeMTG(:/, :Plant, 1, 1), Dict{Symbol,Any}()) diff --git a/bindings/julia/README.md b/bindings/julia/README.md index 13381a7..c7edda4 100644 --- a/bindings/julia/README.md +++ b/bindings/julia/README.md @@ -1,6 +1,6 @@ # FspmTF.jl -The Julia codec of fspm-TF: reading and writing frames and sequences (spec 0.1), and, as a package extension, a +The Julia codec of fspm-TF: reading and writing frames and sequences (spec 0.2), and, as a package extension, a MultiScaleTreeGraph.jl MTG to and from a `graph` data structure (`from_mtg`, `to_mtg`). - `FspmTF/src/`: the GLB container (`glb.jl`), the data model (`model.jl`), reading and writing (`codec.jl`), diff --git a/doc/julia.md b/doc/julia.md index 163634d..65e9939 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -50,13 +50,15 @@ Two things differ from the other platform's graph, and are undone by `from_mtg`: | `complex` | the first ancestor at a smaller scale | | `parent`, `edge_type` | the parent, at the same scale, by `<` (successor) or `+` (branch) | | relation `bearer` | the parent otherwise, e.g. an axis borne by an internode | -| `mtg_index` | the node's `index` | -| variables | the attributes, declared by `variables` (MTG.jl has no units) | +| `order` | the node's position among its parent's children | +| `mtg_index` (standard name `mtg_index`) | the node's `index` | +| variables | the attributes, declared by `variables` (MTG.jl has no units); missing where a node has no such attribute; dates and times as ISO 8601 text with their `format` | | scalars | the attributes of the scale 0 root (`Scene`) | -Some MTG content has no place in fspm-TF 0.1: missing attributes, dates, the MTG's classes and description, and -the order of children. It is kept in the data structure's `extras["MultiScaleTreeGraph"]`, so that `to_mtg` -rebuilds the MTG exactly. Other readers see the missing values as NaN, `false`, 0 or "". +`ranks` gives a scale another rank than its number, e.g. the vocabulary's (Organ 5), and `labels` maps the +symbols to the vocabulary's labels. MTG.jl's own metadata (its classes and description, the scene node, each set's +MTG scale) is kept in the data structure's `extras["virtualplantlab"]`, so that `to_mtg` rebuilds the MTG exactly. +A float attribute a node lacks is NaN in its column, so `to_mtg` reads a NaN as no attribute. ## A VirtualPlantLab graph @@ -67,8 +69,8 @@ With PlantGraphs loaded (Julia 1.12 or later), `from_graph` writes one entity se - the graph's data as scalars; - its parent as the nearest kept node. -The turtle's movements (`RU`, `T`, `F`, …) are left out. PlantGraphs stores no edge type, so the last child of a -node is written as its successor and the others as branches; `extras` says the edge types were guessed. With +The turtle's movements (`RU`, `T`, `F`, …) are left out. PlantGraphs stores no edge type, so the edges are written as +`unknown`, and the children's order as the `order` column. A field a node's type lacks is missing. With PlantGeomTurtle loaded too, `add_geometry!` adds the graph's turtle mesh, each triangle carrying the row of the node that made it. diff --git a/implementations.md b/implementations.md index 321cb1d..199172d 100644 --- a/implementations.md +++ b/implementations.md @@ -8,9 +8,9 @@ | CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.1 | | fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.1 | | GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.1 | -| FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.1 | -| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.1 | -| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.1 | +| FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.2 | +| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.2 | +| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.2 | Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters (a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional From b6603577da63f999055abcf70b7d94ccdecd33dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:11:16 +0100 Subject: [PATCH 12/39] The C++ codec to specification 0.2: a column's missing mask (integers with a free noData, text with a free noData string, booleans as UINT8 0/1/255 with FSPM_boolean), an entity set's scale, a data structure's labels, a variable's format, edge_type unknown, the order column; CPlantBox's adapter on 0.2: its Organ set mixed, Plant and SubOrgan with their scales, segment ends in cm as declared (a rebuilt root system bit for bit), its order as branching_order (order is reserved) --- bindings/cpp/README.md | 2 +- .../cplantbox/include/fspm_tf/cplantbox.hpp | 7 +- bindings/cpp/cplantbox/src/cplantbox.cpp | 32 +++--- .../cpp/cplantbox/test/test_cplantbox.cpp | 27 +++-- bindings/cpp/include/fspm_tf/model.hpp | 25 ++++- bindings/cpp/src/codec.cpp | 106 +++++++++++++++++- bindings/cpp/src/dump.cpp | 10 +- bindings/cpp/src/model.cpp | 16 ++- bindings/cpp/test/test_fspm_tf.cpp | 2 +- doc/cpp.md | 6 +- implementations.md | 4 +- 11 files changed, 196 insertions(+), 41 deletions(-) diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 2cc9b8b..6396685 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -1,6 +1,6 @@ # fspm_tf (C++) -The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.1). In `cplantbox/` is +The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.2). In `cplantbox/` is CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, and turns a graph into a static CPlantBox root system. diff --git a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp index 5d34508..30bbc9e 100644 --- a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp +++ b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp @@ -5,12 +5,13 @@ // | fspm-TF | CPlantBox | // |----------------------------------|---------------------------------------------------------------------------| // | Plant (rank 1), one row, id 0 | the organism | -// | Organ (rank 5), id 1 + organ id | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | +// | Organ (rank 5, scale "mixed"), | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | // | | plant, parent its parent organ (branch) | // | SubOrgan (rank 6), id | its segments: complex its organ, parent the segment ending at its | // | 2^32 + distal node id | proximal node (successor along an organ, branch at a lateral's base) | -// | positions (proximal_end_*, ...) | node coordinates, cm converted to metres | -// | other variables | CPlantBox's names and units (cm, d), declared with their standard names | +// | positions (proximal_end_*, ...) | node coordinates, in cm (declared; readers convert) | +// | other variables | CPlantBox's names and units (cm, d), declared with their standard names; | +// | | its `order` as `branching_order` (`order` is fspm-TF's reserved column) | // | extras["cplantbox"] | the id offsets, the organism's parameters (its XML) | // // CPlantBox has two scales below the plant: organs and segments. A frame with more scales (e.g. metafspm's Plant, diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp index d1b3821..020eda4 100644 --- a/bindings/cpp/cplantbox/src/cplantbox.cpp +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -95,10 +95,12 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio // Plant: the organism EntitySet plant("Plant", 1); + plant.scale = "Plant"; plant.set("id", std::vector{0}); // Organ: one row per organ EntitySet organ_set("Organ", 5); + organ_set.scale = "mixed"; // whole root and stem axes, leaves, the seed std::vector organ_ids, complexes, parents; std::vector labels, edge_types; std::vector organ_types, sub_types, orders, parent_node_indices, node_counts; @@ -125,12 +127,13 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio } organ_set.set("id", organ_ids).set("label", Column(labels)).set("complex", complexes).set("parent", parents) .set("edge_type", Column(edge_types)).set("organType", organ_types).set("subType", sub_types) - .set("order", orders).set("age", ages).set("length", lengths).set("radius", radii) + .set("branching_order", orders).set("age", ages).set("length", lengths).set("radius", radii) .set("creationTime", creation_times).set("parentNI", parent_node_indices).set("numberOfNodes", node_counts) .set("alive", alive).set("active", active); organ_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type: 1 seed, 2 root, 3 stem, 4 leaf", {}, "setting")) .declare("subType", declared("1", "descriptor", "CPlantBox's sub type: the organ's parameter set", {}, "setting")) - .declare("order", declared("1", "descriptor", "branching order (0: a base organ)", "branching_order")) + .declare("branching_order", declared("1", "descriptor", "CPlantBox's order: the branching order (0: a base organ)", + "branching_order")) .declare("age", declared("d", "intensive", "the organ's age", "age")) .declare("length", declared("cm", "extensive", "the organ's realized length", "length")) .declare("radius", declared("cm", "intensive", "the organ's radius", "radius", "parameter")) @@ -146,6 +149,7 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio for (int node = 1; node < organ->getNumberOfNodes(); ++node) segment_ending_at[organ->getNodeId(node)] = SEGMENT_ID_OFFSET + organ->getNodeId(node); EntitySet segment_set("SubOrgan", 6); + segment_set.scale = "SubOrgan"; std::vector segment_ids, segment_complexes, segment_parents; std::vector segment_labels, segment_edge_types; std::vector segment_organ_types, segment_sub_types; @@ -163,12 +167,12 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio segment_labels.push_back(segment_label(organ->organType())); segment_organ_types.push_back(organ->organType()); segment_sub_types.push_back(sub_types[index]); - x1.push_back(proximal.x * METRES_PER_CENTIMETRE); - y1.push_back(proximal.y * METRES_PER_CENTIMETRE); - z1.push_back(proximal.z * METRES_PER_CENTIMETRE); - x2.push_back(distal.x * METRES_PER_CENTIMETRE); - y2.push_back(distal.y * METRES_PER_CENTIMETRE); - z2.push_back(distal.z * METRES_PER_CENTIMETRE); + x1.push_back(proximal.x); + y1.push_back(proximal.y); + z1.push_back(proximal.z); + x2.push_back(distal.x); + y2.push_back(distal.y); + z2.push_back(distal.z); segment_radii.push_back(radius); segment_lengths.push_back(distal.minus(proximal).length()); segment_creation_times.push_back(organ->getNodeCT(node)); @@ -181,12 +185,12 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio .set("radius", segment_radii).set("length", segment_lengths).set("creationTime", segment_creation_times); segment_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type of its organ", {}, "setting")) .declare("subType", declared("1", "descriptor", "CPlantBox's sub type of its organ", {}, "setting")) - .declare("x1", declared("m", "descriptor", "proximal node, x", "proximal_end_x")) - .declare("y1", declared("m", "descriptor", "proximal node, y", "proximal_end_y")) - .declare("z1", declared("m", "descriptor", "proximal node, z", "proximal_end_z")) - .declare("x2", declared("m", "descriptor", "distal node, x", "distal_end_x")) - .declare("y2", declared("m", "descriptor", "distal node, y", "distal_end_y")) - .declare("z2", declared("m", "descriptor", "distal node, z", "distal_end_z")) + .declare("x1", declared("cm", "descriptor", "proximal node, x", "proximal_end_x")) + .declare("y1", declared("cm", "descriptor", "proximal node, y", "proximal_end_y")) + .declare("z1", declared("cm", "descriptor", "proximal node, z", "proximal_end_z")) + .declare("x2", declared("cm", "descriptor", "distal node, x", "distal_end_x")) + .declare("y2", declared("cm", "descriptor", "distal node, y", "distal_end_y")) + .declare("z2", declared("cm", "descriptor", "distal node, z", "distal_end_z")) .declare("radius", declared("cm", "intensive", "the segment's radius (its organ's)", "radius", "parameter")) .declare("length", declared("cm", "extensive", "the segment's length", "length", "derived")) .declare("creationTime", declared("d", "descriptor", "the creation time of its distal node")); diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp index 23b5082..c5a28f7 100644 --- a/bindings/cpp/cplantbox/test/test_cplantbox.cpp +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -23,6 +23,16 @@ namespace { int failures = 0, checks = 0; +void check(bool condition, const std::string &what); + +/// Checks that two JSON values are equal, printing their first differences when they are not. +void check_same(const json &actual, const json &expected, const std::string &what) { + const auto differences = compare(actual, expected); + for (std::size_t item = 0; item < std::min(differences.size(), 3); ++item) + std::cerr << " " << differences[item] << "\n"; + check(differences.empty(), what); +} + void check(bool condition, const std::string &what) { ++checks; if (!condition) { @@ -96,10 +106,10 @@ void test_root_system(const filesystem::path ¶meters, const filesystem::path write_frame((out / "anagallis.glb").string(), frame); // written, read back the same const auto read_back = read_frame((out / "anagallis.glb").string()); - check(compare(frame_to_json(read_back), frame_to_json(frame)).empty(), "the root system read back as written"); + check_same(frame_to_json(read_back), frame_to_json(frame), "the root system read back as written"); // Rebuilt as a static root system (CPlantBox has static roots only: the stem is left out), and exported again: - // the same root segments (ends to the nanometre) + // the same root segments, their ends bit for bit const auto rebuilt = cplantbox::to_root_system(read_back.data_structures[0]); const auto organ_types = values(segments, "organType"); const auto root_segments = std::count(organ_types.begin(), organ_types.end(), double(CPlantBox::Organism::ot_root)); @@ -114,14 +124,16 @@ void test_root_system(const filesystem::path ¶meters, const filesystem::path if (organ_types[row] == CPlantBox::Organism::ot_root) root_total += lengths[row]; for (const auto length : values(rebuilt_segments, "length")) rebuilt_total += length; check(std::abs(rebuilt_total - root_total) < 1e-9 * root_total, "the static root system has the same root length"); - std::multiset ends, rebuilt_ends; + std::multiset ends, rebuilt_ends; // in cm both ways: bit for bit for (const char *name : {"x2", "y2", "z2"}) { const auto coordinates = values(segments, name); for (std::size_t row = 0; row < coordinates.size(); ++row) - if (organ_types[row] == CPlantBox::Organism::ot_root) ends.insert(std::llround(coordinates[row] * 1e9)); - for (const auto value : values(rebuilt_segments, name)) rebuilt_ends.insert(std::llround(value * 1e9)); + if (organ_types[row] == CPlantBox::Organism::ot_root) ends.insert(coordinates[row]); + for (const auto value : values(rebuilt_segments, name)) rebuilt_ends.insert(value); } check(ends == rebuilt_ends, "the static root system's segment ends are the same"); + check(data_structure.entity_set("Organ").scale == std::optional("mixed"), "the Organ set is mixed"); + check(segments.variables.at("x1").unit == "cm", "segment ends in CPlantBox's unit, cm"); } void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const filesystem::path &out) { @@ -156,7 +168,8 @@ void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const const auto &targets = mapping.column("target").as(); for (std::size_t item = 0; item < sources.size(); ++item) { const auto row = row_of.at(sources[item]); - const double middle[3] = {(x1[row] + x2[row]) / 2, (y1[row] + y2[row]) / 2, (z1[row] + z2[row]) / 2}; + const double middle[3] = {(x1[row] + x2[row]) / 200, (y1[row] + y2[row]) / 200, // cm -> m + (z1[row] + z2[row]) / 200}; std::int64_t index[3]; for (int axis = 0; axis < 3; ++axis) index[axis] = std::int64_t(std::floor((middle[axis] - grid.origin[axis]) / grid.spacing[axis])); @@ -170,7 +183,7 @@ void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const frame.mappings = {mapping}; write_frame((out / "fspm2023.glb").string(), frame); const auto read_back = read_frame((out / "fspm2023.glb").string()); - check(compare(frame_to_json(read_back), frame_to_json(frame)).empty(), "the plant, soil and mapping read back"); + check_same(frame_to_json(read_back), frame_to_json(frame), "the plant, soil and mapping read back"); } void test_another_platform(const filesystem::path &conformance) { diff --git a/bindings/cpp/include/fspm_tf/model.hpp b/bindings/cpp/include/fspm_tf/model.hpp index b5617f8..df0e12f 100644 --- a/bindings/cpp/include/fspm_tf/model.hpp +++ b/bindings/cpp/include/fspm_tf/model.hpp @@ -10,7 +10,8 @@ // Mapping source entity <-> target entity weights between two data structures // // A column holds one value per row (numbers, booleans, text), or a fixed-size vector per row (`width` values per -// row, row after row). +// row, row after row). Missing values: NaN in float columns; elsewhere the column's `missing` mask (one flag per row, +// empty when none is missing). #pragma once #include @@ -31,11 +32,14 @@ namespace fspm_tf { using json = nlohmann::ordered_json; /// The version of the specification this library reads and writes. -inline constexpr const char *SPECIFICATION_VERSION = "0.1"; +inline constexpr const char *SPECIFICATION_VERSION = "0.2"; /// The id of no entity: the complex or parent of an entity that has none, the feature row of a vertex with none. inline constexpr std::int64_t NO_ID = -1; /// The values of edge_type: the root of its scale, MTG '<', MTG '+'. -inline const std::vector EDGE_TYPES = {"none", "successor", "branch"}; +inline const std::vector EDGE_TYPES = {"none", "successor", "branch", "unknown"}; +/// The vocabulary's base scales: an entity set's scale is one of them, or "mixed". +inline const std::vector BASE_SCALES = {"Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", + "SubOrgan", "Layer", "Cell", "Compartment", "Connection"}; /// What the specification forbids, or a file that is not fspm-TF. struct Error : std::runtime_error { @@ -49,10 +53,12 @@ using ColumnValues = std::variant, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector>; -/// A column: its values, `width` per row (1: one value per row; >= 2: a fixed-size vector per row). +/// A column: its values, `width` per row (1: one value per row; >= 2: a fixed-size vector per row), and which rows are +/// missing (`missing`, one flag per row, empty when none is; NaN in a float column). struct Column { ColumnValues values; std::size_t width = 1; + std::vector missing; Column() : values(std::vector{}) {} template Column(std::vector column_values, std::size_t width = 1) @@ -73,6 +79,14 @@ struct Column { throw Error("a column of another type than the one asked for"); } template bool holds() const { return std::holds_alternative>(values); } + /// Whether row `row` is missing (by the mask; a float's NaN is read as a value). + bool is_missing(std::size_t row) const { return !missing.empty() && missing[row]; } + bool has_missing() const; + /// Marks rows as missing (one flag per row); returns the column, for chaining. + Column &with_missing(std::vector rows) { + missing = std::move(rows); + return *this; + } /// The column's type name in the conformance files' expected values: numpy's (float64, int64, bool...), or /// "string". std::string dtype() const; @@ -90,6 +104,7 @@ struct Variable { std::optional maximum; std::vector references; std::vector doi; + std::optional format; ///< a text column of dates or times: "date", "date-time" or "time" bool operator==(const Variable &other) const; }; @@ -130,6 +145,7 @@ struct Table { /// edge_type ("none", "successor", "branch"); an adaptive grid's cells: level and index. struct EntitySet : Table { std::optional rank; + std::optional scale; ///< a base scale (BASE_SCALES), or "mixed" EntitySet() : Table("", TableKind::entity_set) {} explicit EntitySet(std::string name, std::optional rank = std::nullopt) @@ -209,6 +225,7 @@ struct DataStructure { std::optional platform; std::optional model; json extras = json::object(); + std::vector> labels; ///< the platform's labels -> the vocabulary's DataStructure() = default; explicit DataStructure(std::string id, std::string kind = "graph") : id(std::move(id)), kind(std::move(kind)) {} diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp index 2e767a4..0ae63b5 100644 --- a/bindings/cpp/src/codec.cpp +++ b/bindings/cpp/src/codec.cpp @@ -25,7 +25,8 @@ const std::string METADATA = "EXT_structural_metadata"; const std::string FEATURES = "EXT_mesh_features"; const std::string EDGE_TYPE_ENUM = "FSPM_edge_type"; const std::vector ENUM_COLUMNS = {"label", "edge_type", "relation"}; -const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index"}; +const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index", "order"}; +constexpr std::uint8_t BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing constexpr int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; constexpr int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; constexpr std::int64_t MAX_FLOAT_FEATURE = std::int64_t(1) << 24; // feature rows are float vertex attributes @@ -66,6 +67,30 @@ template T negative_stand_in() { return std::nextafter(std::numeric_limits::lowest(), T(0)); } +/// A value of T the column does not hold: its minimum (signed) or maximum (unsigned) first, then inwards. +template T free_integer(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + const T first = std::is_signed_v ? std::numeric_limits::min() : std::numeric_limits::max(); + const T second = std::is_signed_v ? std::numeric_limits::max() : std::numeric_limits::min(); + if (!present.count(first)) return first; + if (!present.count(second)) return second; + T candidate = first; + while (present.count(candidate)) candidate = std::is_signed_v ? T(candidate + 1) : T(candidate - 1); + return candidate; +} + +/// A string the column does not hold: the empty string first, then NUL characters. +std::string free_string(const std::vector &values, const std::vector &missing) { + std::set present; + for (std::size_t row = 0; row < values.size(); ++row) + if (missing.empty() || !missing[row]) present.insert(values[row]); + std::string candidate; + while (present.count(candidate)) candidate.push_back('\0'); + return candidate; +} + json get(const json &object, const std::string &key, json fallback = nullptr) { if (object.is_object()) { auto found = object.find(key); @@ -102,6 +127,7 @@ json declaration_json(const Variable &variable) { if (variable.maximum) declaration["maximum"] = *variable.maximum; if (!variable.references.empty()) declaration["references"] = variable.references; if (!variable.doi.empty()) declaration["doi"] = variable.doi; + if (variable.format) declaration["format"] = *variable.format; return declaration; } @@ -167,9 +193,16 @@ std::pair encode_table(const Table &table, const std::string &class_ } else if (column.holds()) { if (column.width != 1) throw Error(where + ": text vectors are not supported"); property["type"] = "STRING"; + std::vector strings = column.as(); + if (column.has_missing()) { // missing text: noData, a string the column lacks + const std::string no_data = free_string(strings, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < strings.size(); ++row) + if (column.missing[row]) strings[row] = no_data; + } std::vector text; std::vector offsets = {0}; - for (const auto &value : column.as()) { + for (const auto &value : strings) { text.insert(text.end(), value.begin(), value.end()); offsets.push_back(text.size()); } @@ -182,6 +215,16 @@ std::pair encode_table(const Table &table, const std::string &class_ item["stringOffsetType"] = "UINT64"; } stored[name] = item; + } else if (column.holds() && column.has_missing()) { // booleans with missing values: UINT8 0, 1, 255 + property["type"] = "SCALAR"; + property["componentType"] = "UINT8"; + property["noData"] = BOOLEAN_NO_DATA; + property["extras"] = {{"FSPM_boolean", true}}; + const auto &values = column.as(); + std::vector codes(values.size()); + for (std::size_t row = 0; row < values.size(); ++row) + codes[row] = column.missing[row] ? BOOLEAN_NO_DATA : std::uint8_t(values[row]); + stored[name] = {{"values", buffers.add(codes)}}; } else if (column.holds()) { if (column.width != 1) throw Error(where + ": boolean vectors are not supported"); property["type"] = "BOOLEAN"; @@ -216,6 +259,8 @@ std::pair encode_table(const Table &table, const std::string &class_ } std::vector data = values; if constexpr (std::is_floating_point_v) { + for (std::size_t row = 0; row < column.missing.size(); ++row) // a missing float: NaN + if (column.missing[row]) data[row] = std::numeric_limits::quiet_NaN(); const bool infinite = std::any_of(data.begin(), data.end(), [](T value) { return std::isinf(value); }); if (infinite) { const T positive = positive_stand_in(), negative = negative_stand_in(); @@ -237,6 +282,11 @@ std::pair encode_table(const Table &table, const std::string &class_ } } else if (reserved && (name == "complex" || name == "parent")) { property["noData"] = NO_ID; + } else if (column.has_missing()) { // missing integers: noData, a value the column lacks + const T no_data = free_integer(data, column.missing); + property["noData"] = no_data; + for (std::size_t row = 0; row < data.size(); ++row) + if (column.missing[row]) data[row] = no_data; } stored[name] = {{"values", buffers.add(data)}}; } @@ -386,6 +436,7 @@ std::vector to_bytes(const Frame &frame) { for (const auto &set : data_structure.entity_sets) { json item = {{"name", set.name}, {"rank", nullptr}}; if (set.rank) item["rank"] = *set.rank; + if (set.scale) item["scale"] = *set.scale; item["propertyTable"] = add_table(set, data_structure.id + "__" + set.name); sets.push_back(item); } @@ -400,6 +451,11 @@ std::vector to_bytes(const Frame &frame) { } if (data_structure.scalars) entry["scalars"] = add_table(*data_structure.scalars, data_structure.id + "__scalars"); if (data_structure.grid) entry["grid"] = grid_json(*data_structure.grid); + if (!data_structure.labels.empty()) { + json labels = json::object(); + for (const auto &[label, standard] : data_structure.labels) labels[label] = standard; + entry["labels"] = labels; + } if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; json node = {{"name", data_structure.id}}; if (!is_identity(data_structure.origin)) node["matrix"] = entry["origin"]; @@ -583,7 +639,19 @@ void decode_table(const json &document, const std::vector &binary, throw Error("'" + class_id + "." + name + "': string offsets outside its values"); text.emplace_back(reinterpret_cast(raw.data + offsets[row]), offsets[row + 1] - offsets[row]); } + std::vector missing; + if (property.contains("noData")) { // missing text + const std::string no_data = property["noData"].get(); + missing.assign(count, false); + for (std::size_t row = 0; row < count; ++row) + if (text[row] == no_data) { + missing[row] = true; + text[row].clear(); + } + if (std::none_of(missing.begin(), missing.end(), [](bool flag) { return flag; })) missing.clear(); + } column = Column(std::move(text)); + column.missing = std::move(missing); } else if (type == "BOOLEAN") { if (raw.size * 8 < count) throw Error("'" + class_id + "." + name + "': too few bits"); std::vector values(count); @@ -594,7 +662,34 @@ void decode_table(const json &document, const std::vector &binary, if (vector && !property.contains("count")) throw Error("'" + class_id + "." + name + "': variable-length arrays are not part of fspm-TF 0.1"); const std::size_t width = vector ? property["count"].get() : 1; - column = decode_scalar_of(property.at("componentType").get(), raw, property, count, width); + const std::string component = property.at("componentType").get(); + column = decode_scalar_of(component, raw, property, count, width); + if (get(get(property, "extras", json::object()), "FSPM_boolean", false).get()) { + const auto codes = column.as(); // booleans with missing values + const auto no_data = get(property, "noData", BOOLEAN_NO_DATA).get(); + std::vector values(count), missing(count); + for (std::size_t row = 0; row < count; ++row) { + missing[row] = codes[row] == no_data; + values[row] = !missing[row] && codes[row] == 1; + } + column = Column(std::move(values)); + if (std::any_of(missing.begin(), missing.end(), [](bool flag) { return flag; })) column.missing = missing; + } else if (width == 1 && component.rfind("FLOAT", 0) != 0 && property.contains("noData") && + name != "complex" && name != "parent") { // missing integers + const double no_data = property["noData"].get(); + std::vector missing(count); + bool any = false; + for (std::size_t row = 0; row < count; ++row) { + const double value = std::visit([&](const auto &values) -> double { + using V = typename std::decay_t::value_type; + if constexpr (std::is_arithmetic_v) return static_cast(values[row]); + else return 0.0; + }, column.values); + missing[row] = value == no_data; + any = any || missing[row]; + } + if (any) column.missing = missing; + } } else { throw Error("'" + class_id + "." + name + "': property type " + type + " is not part of fspm-TF 0.1"); } @@ -609,6 +704,7 @@ void decode_table(const json &document, const std::vector &binary, if (property.contains("description")) variable.description = property["description"].get(); if (!get(declared, "minimum").is_null()) variable.minimum = declared["minimum"].get(); if (!get(declared, "maximum").is_null()) variable.maximum = declared["maximum"].get(); + if (!get(declared, "format").is_null()) variable.format = declared["format"].get(); for (const auto &reference : get(declared, "references", json::array())) variable.references.push_back(reference.get()); for (const auto &doi : get(declared, "doi", json::array())) variable.doi.push_back(doi.get()); @@ -705,6 +801,9 @@ Frame from_bytes(const std::vector &data) { if (!get(entry, "platform").is_null()) data_structure.platform = entry["platform"].get(); if (!get(entry, "model").is_null()) data_structure.model = entry["model"].get(); data_structure.extras = get(entry, "extras", json::object()); + const json labels = get(entry, "labels", json::object()); + for (const auto &item : labels.items()) + data_structure.labels.emplace_back(item.key(), item.value().get()); if (entry.contains("origin")) { const auto values = entry["origin"].get>(); if (values.size() != 16) throw Error("data structure '" + data_structure.id + "': an origin of 16 numbers"); @@ -716,6 +815,7 @@ Frame from_bytes(const std::vector &data) { for (const auto &item : get(entry, "entitySets", json::array())) { EntitySet set; if (!get(item, "rank").is_null()) set.rank = item["rank"].get(); + if (!get(item, "scale").is_null()) set.scale = item["scale"].get(); table(item.at("propertyTable").get(), set); tables[item["propertyTable"].get()] = item.at("name").get(); data_structure.entity_sets.push_back(std::move(set)); diff --git a/bindings/cpp/src/dump.cpp b/bindings/cpp/src/dump.cpp index fd40c8d..9f55bb7 100644 --- a/bindings/cpp/src/dump.cpp +++ b/bindings/cpp/src/dump.cpp @@ -23,7 +23,8 @@ json column_json(const Column &column) { return std::visit([&](const auto &values) { json items = json::array(); if (column.width == 1) { - for (std::size_t item = 0; item < values.size(); ++item) items.push_back(value_json(values[item])); + for (std::size_t item = 0; item < values.size(); ++item) + items.push_back(column.is_missing(item) ? json(nullptr) : value_json(values[item])); // missing: null return items; } for (std::size_t row = 0; row < column.rows(); ++row) { @@ -51,6 +52,7 @@ json table_json(const Table &table) { if (variable.maximum) declared["maximum"] = *variable.maximum; if (!variable.references.empty()) declared["references"] = variable.references; if (!variable.doi.empty()) declared["doi"] = variable.doi; + if (variable.format) declared["format"] = *variable.format; declarations[name] = declared; } if (!declarations.empty()) entry["variables"] = declarations; @@ -72,6 +74,7 @@ json frame_to_json(const Frame &frame) { for (const auto &set : data_structure.entity_sets) { json item = table_json(set); item["rank"] = set.rank ? json(*set.rank) : json(nullptr); + if (set.scale) item["scale"] = *set.scale; sets.push_back(item); } json entry = {{"id", data_structure.id}, {"kind", data_structure.kind}, {"origin", origin}, @@ -79,6 +82,11 @@ json frame_to_json(const Frame &frame) { if (data_structure.platform) entry["platform"] = *data_structure.platform; if (data_structure.model) entry["model"] = *data_structure.model; if (!data_structure.extras.empty()) entry["extras"] = data_structure.extras; + if (!data_structure.labels.empty()) { + json labels = json::object(); + for (const auto &[label, standard] : data_structure.labels) labels[label] = standard; + entry["labels"] = labels; + } if (!data_structure.relations.empty()) { json relations = json::array(); for (const auto &relation : data_structure.relations) { diff --git a/bindings/cpp/src/model.cpp b/bindings/cpp/src/model.cpp index b267685..ec7a0a2 100644 --- a/bindings/cpp/src/model.cpp +++ b/bindings/cpp/src/model.cpp @@ -26,12 +26,19 @@ template <> struct TypeName { static constexpr const char *value = "bool"; template <> struct TypeName { static constexpr const char *value = "string"; }; const std::vector NO_RESERVED = {}; -const std::vector ENTITY_SET_RESERVED = {"id", "label", "complex", "parent", "edge_type", "level", "index"}; +const std::vector ENTITY_SET_RESERVED = {"id", "label", "complex", "parent", "edge_type", + "level", "index", "order"}; const std::vector RELATION_RESERVED = {"id", "source", "target", "relation"}; const std::vector MAPPING_RESERVED = {"source", "target", "weight"}; } // namespace +bool Column::has_missing() const { + for (const bool flag : missing) + if (flag) return true; + return false; +} + std::string Column::dtype() const { return std::visit([](const auto &column_values) -> std::string { using T = typename std::decay_t::value_type; @@ -42,7 +49,7 @@ std::string Column::dtype() const { bool Variable::operator==(const Variable &other) const { return unit == other.unit && standard_name == other.standard_name && kind == other.kind && role == other.role && description == other.description && minimum == other.minimum && maximum == other.maximum && - references == other.references && doi == other.doi; + references == other.references && doi == other.doi && format == other.format; } std::size_t Table::row_count() const { return columns.empty() ? 0 : columns.front().second.rows(); } @@ -103,6 +110,11 @@ void Table::check_lengths() const { throw Error("column '" + item.first + "' of '" + name + "': its values are not whole rows"); if (item.second.rows() != row_count()) throw Error("table '" + name + "': columns of different lengths"); + if (!item.second.missing.empty() && item.second.missing.size() != item.second.rows()) + throw Error("column '" + item.first + "' of '" + name + "': one missing flag per row"); + if (item.second.has_missing() && (is_reserved(item.first) || item.second.width != 1)) + throw Error("column '" + item.first + "' of '" + name + "': missing values only in variables of one value " + "per row"); } } diff --git a/bindings/cpp/test/test_fspm_tf.cpp b/bindings/cpp/test/test_fspm_tf.cpp index 7b4482a..4229202 100644 --- a/bindings/cpp/test/test_fspm_tf.cpp +++ b/bindings/cpp/test/test_fspm_tf.cpp @@ -67,7 +67,7 @@ void report(const std::vector &differences, const std::string &what void test_conformance(const filesystem::path &conformance) { const auto files = conformance_files(conformance); - check(files.size() == 9, "nine conformance files"); + check(files.size() == 10, "ten conformance files"); for (const auto &path : files) { const auto name = filesystem::relative(path, conformance).string(); const auto data = file_bytes(path); diff --git a/doc/cpp.md b/doc/cpp.md index bb5dd14..905c2a6 100644 --- a/doc/cpp.md +++ b/doc/cpp.md @@ -78,10 +78,10 @@ auto roots = fspm_tf::cplantbox::to_root_system(fspm_tf::read_frame("roots.glb") | fspm-TF | CPlantBox | |---|---| | Plant (rank 1), id 0 | the organism | -| Organ (rank 5), id 1 + organ id | a seed, a root or a stem (labelled `RootAxis`, `ShootAxis`: a whole axis), a leaf | +| Organ (rank 5, scale `mixed`), id 1 + organ id | a seed, a root or a stem (labelled `RootAxis`, `ShootAxis`: a whole axis), a leaf | | SubOrgan (rank 6), id 2^32 + distal node id | a segment; its parent is the segment ending at its proximal node | -| `x1` … `z2` (`proximal_end_*`, `distal_end_*`) | the segment's nodes, cm converted to metres | -| other variables | CPlantBox's names and units (`length` in cm, `age` in d, `order`, `subType`, …), with their standard names | +| `x1` … `z2` (`proximal_end_*`, `distal_end_*`) | the segment's nodes, in cm (declared; readers convert, so a rebuilt root system has them bit for bit) | +| other variables | CPlantBox's names and units (`length` in cm, `age` in d, `subType`, …), with their standard names; its `order` as `branching_order` (`order` is fspm-TF's reserved column) | | `extras["cplantbox"]` | the id offsets, the organism's parameters as CPlantBox's XML | CPlantBox has two scales below the plant. `to_root_system` reads frames with more scales by what they hold: the diff --git a/implementations.md b/implementations.md index 199172d..0ab7149 100644 --- a/implementations.md +++ b/implementations.md @@ -4,8 +4,8 @@ |---|---|---|---|---|---| | openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.2 | | openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.2 (through openalea.fspm_tf) | -| fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.1 | -| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.1 | +| fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.2 | +| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.2 | | fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.1 | | GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.1 | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.2 | From f969cc39e68d5f99143e454e3bfedf884e58c518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:12:04 +0100 Subject: [PATCH 13/39] FspmTF.jl, MTG attributes named as a reserved column (e.g. order) are not written, with a warning, rather than overwriting it --- bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl index 1ea1b16..803e44f 100644 --- a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -87,6 +87,10 @@ function attribute_columns(nodes, set_name, variables, skip = ()) end columns, declarations = Columns(), Declarations() for name in names + if string(name) in FspmTF.ENTITY_SET_RESERVED # e.g. an `order` attribute (a branching order) + @warn "attribute '$name' of '$set_name' not written: '$name' is a reserved column of fspm-TF" + continue + end values = Any[attribute_of(node, name) for node in nodes] converted = FspmTF.column_of_values(values) if converted === nothing From 596adefefe667c9ca3b0317f54a94879816d3e11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:14:33 +0100 Subject: [PATCH 14/39] The Java codec to specification 0.2: a column's missing mask (integers with a free noData, text with a free noData string, booleans as UINT8 0/1/255 with FSPM_boolean), an entity set's scale, a data structure's labels, a variable's format, edge_type unknown, the order column; GroIMP's adapter on 0.2: missing fields as missing rather than in extras, each scale's base scale (scaleScales) and the classes' vocabulary labels (labels) given --- CHANGELOG.md | 4 + bindings/java/README.md | 2 +- .../org/openalea/fspmtf/groimp/GroImp.java | 37 +++-- .../openalea/fspmtf/groimp/GroImpTest.java | 10 +- .../main/java/org/openalea/fspmtf/Codec.java | 151 +++++++++++++++++- .../main/java/org/openalea/fspmtf/Column.java | 23 ++- .../org/openalea/fspmtf/DataStructure.java | 2 + .../main/java/org/openalea/fspmtf/Dump.java | 10 ++ .../java/org/openalea/fspmtf/EntitySet.java | 2 + .../main/java/org/openalea/fspmtf/Table.java | 5 +- .../java/org/openalea/fspmtf/Variable.java | 4 +- .../java/org/openalea/fspmtf/CodecTest.java | 4 +- doc/java.md | 9 +- implementations.md | 4 +- 14 files changed, 228 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b64b61b..170a9a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ dates (`format`); sparse ids; the conformance files' status and the sources' precedence. - Vocabulary 0.2: `Seed`, `mtg_index`, the new enumerations. - The reference codec, the validator, the ParaView reader and the conformance files (a new scene, `missing_values`). +- The Julia, C++ and Java codecs on 0.2, and their adapters using it instead of their extras: MTG.jl (missing + attributes, dates, children's order, `mtg_index`), PlantGraphs (`unknown` edges, children's order), CPlantBox (a + `mixed` Organ set, segment ends in cm, `branching_order`), GroIMP (missing fields, scales, labels); extras keys in + lower case (`virtualplantlab`, `cplantbox`, `groimp`). ## Specification 0.1 (draft) diff --git a/bindings/java/README.md b/bindings/java/README.md index 6546d9b..4e1e2e4 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -1,6 +1,6 @@ # fspm-tf (Java) -The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.1), with one dependency, +The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.2), with one dependency, Jackson. GroIMP's adapter is in `groimp/`. - `src/main/java/org/openalea/fspmtf/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB diff --git a/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java b/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java index 443c57b..281452b 100644 --- a/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java +++ b/bindings/java/groimp/src/main/java/org/openalea/fspmtf/groimp/GroImp.java @@ -47,7 +47,8 @@ *
* *

The graph's root (e.g. an RGG root) is the data structure itself, not an entity. What fspm-TF 0.1 cannot say is - * kept in {@code extras["groimp"]}: the rows where a node's class has no such field, and the classes' names.

+ * kept in {@code extras["groimp"]}: the classes' names. A field that other classes of the same scale lack is missing in + * their rows.

*/ public final class GroImp { /** The name of GroIMP's entry in a data structure's extras. */ @@ -66,6 +67,10 @@ public static final class Options { /** The rank of each scale's entity set (by default its refinement depth); e.g. the vocabulary's base scales' * positions (Plant 1, Axis 2, Organ 5, SubOrgan 6). */ public final Map scaleRanks = new HashMap<>(); + /** The base scale of each scale's entity set (e.g. "Organ"), or "mixed". */ + public final Map scaleScales = new HashMap<>(); + /** The model's class names to the vocabulary's labels (e.g. Internode to StemInternode). */ + public final Map labels = new LinkedHashMap<>(); /** The declarations of the fields (by name): unit, standard name, kind, role. */ public final Map variables = new HashMap<>(); public String model; @@ -149,13 +154,13 @@ else if ((bits & (Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE)) != 0 dataStructure.platform = "GroIMP"; dataStructure.model = options.model; ObjectNode extras = JsonNodeFactory.instance.objectNode(); - ObjectNode missing = extras.putObject("missing"); ObjectNode classes = extras.putObject("classes"); int coarsest = byScale.isEmpty() ? 0 : byScale.keySet().iterator().next(); for (Map.Entry> item : byScale.entrySet()) { String setName = setNames.get(item.getKey()); List rows = item.getValue(); EntitySet set = new EntitySet(setName, options.scaleRanks.getOrDefault(item.getKey(), (long) item.getKey())); + set.scale = options.scaleScales.get(item.getKey()); long[] ids = new long[rows.size()], complexes = new long[rows.size()], parents = new long[rows.size()]; String[] labels = new String[rows.size()], edgeTypes = new String[rows.size()]; for (int row = 0; row < rows.size(); row++) { @@ -170,14 +175,12 @@ else if ((bits & (Graph.SUCCESSOR_EDGE | Graph.BRANCH_EDGE)) != 0 set.set("id", Column.of(ids)).set("label", Column.of(labels)); if (item.getKey() != coarsest) set.set("complex", Column.of(complexes)); set.set("parent", Column.of(parents)).set("edge_type", Column.of(edgeTypes)); - ObjectNode missingOfSet = JsonNodeFactory.instance.objectNode(); - addFields(set, rows, options, missingOfSet); - if (!missingOfSet.isEmpty()) missing.set(setName, missingOfSet); + addFields(set, rows, options); dataStructure.entitySets.add(set); } - if (missing.isEmpty()) extras.remove("missing"); addRelations(dataStructure, order, entities, setNames); dataStructure.extras.set(PLATFORM, extras); + dataStructure.labels.putAll(options.labels); return dataStructure; } @@ -243,8 +246,8 @@ private static Object objectValue(Object value) { return null; } - /** The rows' managed fields as columns (in order of first appearance), the missing rows of each in {@code missing}. */ - private static void addFields(EntitySet set, List rows, Options options, ObjectNode missing) { + /** The rows' managed fields as columns (in order of first appearance), missing where a row's class has none. */ + private static void addFields(EntitySet set, List rows, Options options) { Map values = new LinkedHashMap<>(); for (int row = 0; row < rows.size(); row++) { Node node = rows.get(row).node; @@ -260,10 +263,11 @@ private static void addFields(EntitySet set, List rows, Options options, for (Map.Entry item : values.entrySet()) { Column column = column(item.getValue()); if (column == null) continue; // values of several kinds: not written + boolean[] absent = new boolean[item.getValue().length]; + boolean any = false; + for (int row = 0; row < absent.length; row++) any |= absent[row] = item.getValue()[row] == null; + if (any && column.width == 1) column.withMissing(absent); // a missing vector: NaN set.set(item.getKey(), column); - var absent = JsonNodeFactory.instance.arrayNode(); - for (int row = 0; row < item.getValue().length; row++) if (item.getValue()[row] == null) absent.add(row); - if (!absent.isEmpty()) missing.set(item.getKey(), absent); Variable declared = options.variables.get(item.getKey()); if (declared != null) set.declare(item.getKey(), declared); } @@ -371,7 +375,6 @@ public static Node toGraph(DataStructure dataStructure, Function n List sets = new ArrayList<>(); for (EntitySet set : dataStructure.entitySets) if (set.rank != null && set.has("id")) sets.add(set); sets.sort((left, right) -> Long.compare(left.rank, right.rank)); - ObjectNode missing = dataStructure.extras.path(PLATFORM).path("missing") instanceof ObjectNode found ? found : null; Node root = new Node(); for (EntitySet set : sets) { long[] ids = set.column("id").longs(); @@ -379,7 +382,7 @@ public static Node toGraph(DataStructure dataStructure, Function n for (int row = 0; row < ids.length; row++) { Node node = nodes.apply(labels == null ? set.name : labels[row]); if (node == null) throw new FspmTfException("no node made for label '" + labels[row] + "'"); - setFields(node, set, row, missing != null && missing.get(set.name) instanceof ObjectNode ofSet ? ofSet : null); + setFields(node, set, row); idsOf.put(ids[row], node); } } @@ -433,17 +436,13 @@ private static Long complexOf(List sets, long id) { return null; } - private static void setFields(Node node, EntitySet set, int row, ObjectNode missing) { + private static void setFields(Node node, EntitySet set, int row) { ManageableType type = node.getNType(); for (String name : set.variableNames()) { ManageableType.Field field = type.getManagedField(name); if (field == null || !isOwnField(field)) continue; - if (missing != null && missing.has(name)) { - boolean absent = false; - for (var absentRow : missing.get(name)) absent |= absentRow.asInt() == row; - if (absent) continue; - } Column column = set.column(name); + if (column.isMissing(row)) continue; // the field keeps its class's default switch (field.getType().getTypeId()) { case TypeId.BOOLEAN -> field.setBoolean(node, column.booleans()[row]); case TypeId.BYTE -> field.setByte(node, (byte) column.doubleAt(row)); diff --git a/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java index 691c584..4caa323 100644 --- a/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java +++ b/bindings/java/groimp/src/test/java/org/openalea/fspmtf/groimp/GroImpTest.java @@ -58,6 +58,8 @@ void exportedWrittenAndRebuilt() throws IOException { GroImp.Options options = new GroImp.Options(); options.scaleNames.put(2, "Organ"); options.scaleRanks.put(2, 5L); // the vocabulary's Organ + options.scaleScales.put(2, "Organ"); + options.labels.put("Internode", "StemInternode"); options.variables.put("length", new Variable("m", "length", "extensive", "state")); DataStructure written = GroImp.toDataStructure(root, options); @@ -77,8 +79,11 @@ void exportedWrittenAndRebuilt() throws IOException { assertArrayEquals(new boolean[] {false, false, false, true}, organs.column("alive").booleans()); assertEquals("mark", written.relations.get(0).name); assertArrayEquals(new long[] {ids[3]}, written.relations.get(0).column("source").longs()); - assertEquals(List.of(0, 1), List.of(written.extras.get("groimp").get("missing").get("Organ").get("alive").get(0).asInt(), - written.extras.get("groimp").get("missing").get("Organ").get("alive").get(1).asInt())); + assertArrayEquals(new boolean[] {true, true, false, false}, organs.column("alive").missing); // internodes + assertArrayEquals(new boolean[] {false, false, true, true}, organs.column("rank").missing); // leaves + assertEquals("Organ", organs.scale); + assertEquals("StemInternode", written.labels.get("Internode")); + assertTrue(!written.extras.get("groimp").has("missing")); Files.createDirectories(OUT); Frame frame = new Frame(); @@ -97,6 +102,7 @@ void exportedWrittenAndRebuilt() throws IOException { assertArrayEquals(lengths, againOrgans.column("length").doubles()); assertArrayEquals(organs.column("edge_type").strings(), againOrgans.column("edge_type").strings()); assertArrayEquals(organs.column("alive").booleans(), againOrgans.column("alive").booleans()); + assertArrayEquals(organs.column("alive").missing, againOrgans.column("alive").missing); assertEquals(0.004f, ((Organs.Internode) nodes.get(ids[0])).radius); assertEquals("Wheat", ((Organs.Plant) nodes.get(plantId)).species); assertEquals(1, again.relations.size()); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java index ce93a4f..2f7bd29 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java @@ -26,13 +26,15 @@ */ public final class Codec { /** The version of the specification this library reads and writes. */ - public static final String SPECIFICATION_VERSION = "0.1"; - public static final List EDGE_TYPES = List.of("none", "successor", "branch"); + public static final String SPECIFICATION_VERSION = "0.2"; + public static final List EDGE_TYPES = List.of("none", "successor", "branch", "unknown"); private static final String EXTENSION = "FSPM_scene", METADATA = "EXT_structural_metadata", FEATURES = "EXT_mesh_features", EDGE_TYPE_ENUM = "FSPM_edge_type"; private static final List ENUM_COLUMNS = List.of("label", "edge_type", "relation"); - private static final List ID_COLUMNS = List.of("id", "complex", "parent", "source", "target", "level", "index"); + private static final List ID_COLUMNS = List.of("id", "complex", "parent", "source", "target", "level", "index", + "order"); + private static final int BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing private static final int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; private static final int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; private static final long MAX_FLOAT_FEATURE = 1L << 24; // feature rows are float vertex attributes @@ -78,6 +80,7 @@ public static byte[] toBytes(Frame frame) { for (EntitySet set : dataStructure.entitySets) { ObjectNode item = sets.addObject().put("name", set.name); if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + if (set.scale != null) item.put("scale", set.scale); item.put("propertyTable", addTable(set, dataStructure.id + "__" + set.name, buffers, schema, tables, variables)); } if (!dataStructure.relations.isEmpty()) { @@ -90,6 +93,10 @@ public static byte[] toBytes(Frame frame) { if (dataStructure.scalars != null) entry.put("scalars", addTable(dataStructure.scalars, dataStructure.id + "__scalars", buffers, schema, tables, variables)); if (dataStructure.grid != null) entry.set("grid", gridJson(dataStructure.grid)); + if (!dataStructure.labels.isEmpty()) { + ObjectNode labels = entry.putObject("labels"); + dataStructure.labels.forEach(labels::put); + } if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); ObjectNode node = nodes.addObject().put("name", dataStructure.id); if (!isIdentity(dataStructure.origin)) putNumbers(node.putArray("matrix"), columnMajor); @@ -201,6 +208,7 @@ private static ObjectNode declarationJson(Variable variable) { if (variable.maximum != null) declaration.put("maximum", variable.maximum); if (!variable.references.isEmpty()) variable.references.forEach(declaration.putArray("references")::add); if (!variable.doi.isEmpty()) variable.doi.forEach(declaration.putArray("doi")::add); + if (variable.format != null) declaration.put("format", variable.format); return declaration; } @@ -237,6 +245,12 @@ private static void encodeColumn(Table table, String classId, String name, Colum if (column.width != 1) throw new FspmTfException(where + ": text vectors are not supported"); property.put("type", "STRING"); String[] values = column.strings(); + if (column.hasMissing()) { // missing text: noData, a string the column lacks + String noData = freeString(column); + property.put("noData", noData); + values = values.clone(); + for (int row = 0; row < values.length; row++) if (column.missing[row]) values[row] = noData; + } byte[][] encoded = new byte[values.length][]; long[] offsets = new long[values.length + 1]; for (int row = 0; row < values.length; row++) { @@ -250,6 +264,14 @@ private static void encodeColumn(Table table, String classId, String name, Colum ByteBuffer offsetBytes = little(offsets.length * (narrow ? 4 : 8)); for (long offset : offsets) if (narrow) offsetBytes.putInt((int) offset); else offsetBytes.putLong(offset); item.put("stringOffsets", buffers.add(offsetBytes.array())).put("stringOffsetType", narrow ? "UINT32" : "UINT64"); + } else if (column.type == ComponentType.BOOLEAN && column.hasMissing()) { // UINT8 0, 1, 255 missing + property.put("type", "SCALAR").put("componentType", "UINT8").put("noData", BOOLEAN_NO_DATA); + property.putObject("extras").put("FSPM_boolean", true); + boolean[] values = column.booleans(); + byte[] codes = new byte[values.length]; + for (int row = 0; row < values.length; row++) + codes[row] = column.missing[row] ? (byte) BOOLEAN_NO_DATA : (byte) (values[row] ? 1 : 0); + stored.putObject(name).put("values", buffers.add(codes)); } else if (column.type == ComponentType.BOOLEAN) { if (column.width != 1) throw new FspmTfException(where + ": boolean vectors are not supported"); property.put("type", "BOOLEAN"); @@ -272,15 +294,92 @@ private static void encodeColumn(Table table, String classId, String name, Colum if (column.width < 2) throw new FspmTfException(where + ": vectors have at least 2 components"); property.put("array", true).put("count", column.width); } - if (type == ComponentType.FLOAT64) data = standIns(data.doubles().clone(), property, where); - else if (type == ComponentType.FLOAT32) data = standIns(data.floats().clone(), property, where); - else if (reserved && (name.equals("complex") || name.equals("parent"))) property.put("noData", EntitySet.NO_ID); + if (type == ComponentType.FLOAT64) { + double[] values = data.doubles().clone(); + for (int row = 0; column.missing != null && row < values.length; row++) // a missing float: NaN + if (column.missing[row]) values[row] = Double.NaN; + data = standIns(values, property, where); + } else if (type == ComponentType.FLOAT32) { + float[] values = data.floats().clone(); + for (int row = 0; column.missing != null && row < values.length; row++) + if (column.missing[row]) values[row] = Float.NaN; + data = standIns(values, property, where); + } else if (reserved && (name.equals("complex") || name.equals("parent"))) { + property.put("noData", EntitySet.NO_ID); + } else if (column.hasMissing()) { // missing integers: noData, a value the column lacks + long noData = freeInteger(column); + property.put("noData", noData); + data = withValueAt(column, noData); + } if (data.width != column.width) data = new Column(data.type, data.values, column.width); stored.putObject(name).put("values", buffers.add(bytesOf(data))); } if (name.equals("id") && table.kind() == Table.Kind.ENTITY_SET) property.put("required", true); } + /** A value of the column's integer type that it does not hold: its minimum (signed) or maximum (unsigned) first. */ + private static long freeInteger(Column column) { + java.util.Set present = new java.util.HashSet<>(); + for (int row = 0; row < column.size(); row++) + if (!column.isMissing(row)) present.add(rawLong(column, row)); + long minimum, maximum; + switch (column.type) { + case INT8 -> { minimum = Byte.MIN_VALUE; maximum = Byte.MAX_VALUE; } + case INT16 -> { minimum = Short.MIN_VALUE; maximum = Short.MAX_VALUE; } + case INT32 -> { minimum = Integer.MIN_VALUE; maximum = Integer.MAX_VALUE; } + case INT64 -> { minimum = Long.MIN_VALUE; maximum = Long.MAX_VALUE; } + case UINT8 -> { minimum = 0; maximum = 0xFFL; } + case UINT16 -> { minimum = 0; maximum = 0xFFFFL; } + case UINT32 -> { minimum = 0; maximum = 0xFFFFFFFFL; } + default -> { minimum = 0; maximum = -1; } // UINT64: its maximum's bits + } + boolean signed = !column.type.isUnsigned(); + long first = signed ? minimum : maximum, second = signed ? maximum : minimum; + if (!present.contains(first)) return first; + if (!present.contains(second)) return second; + long candidate = first; + while (present.contains(candidate)) candidate = signed ? candidate + 1 : candidate - 1; + return candidate; + } + + /** A value's bits as a long, unsigned values as stored (UINT8 255 as 255). */ + private static long rawLong(Column column, int index) { + return switch (column.type) { + case INT8 -> ((byte[]) column.values)[index]; + case UINT8 -> Byte.toUnsignedLong(((byte[]) column.values)[index]); + case INT16 -> ((short[]) column.values)[index]; + case UINT16 -> Short.toUnsignedLong(((short[]) column.values)[index]); + case INT32 -> ((int[]) column.values)[index]; + case UINT32 -> Integer.toUnsignedLong(((int[]) column.values)[index]); + case INT64, UINT64 -> ((long[]) column.values)[index]; + default -> throw new FspmTfException("a " + column.type + " column has no integers"); + }; + } + + /** The column's values with {@code value} at its missing rows. */ + private static Column withValueAt(Column column, long value) { + Object copy = java.lang.reflect.Array.newInstance(column.values.getClass().getComponentType(), column.size()); + System.arraycopy(column.values, 0, copy, 0, column.size()); + for (int row = 0; row < column.size(); row++) + if (column.isMissing(row)) + switch (column.type) { + case INT8, UINT8 -> ((byte[]) copy)[row] = (byte) value; + case INT16, UINT16 -> ((short[]) copy)[row] = (short) value; + case INT32, UINT32 -> ((int[]) copy)[row] = (int) value; + default -> ((long[]) copy)[row] = value; + } + return new Column(column.type, copy, column.width); + } + + /** A string the column does not hold: the empty string first, then NUL characters. */ + private static String freeString(Column column) { + java.util.Set present = new java.util.HashSet<>(); + for (int row = 0; row < column.size(); row++) if (!column.isMissing(row)) present.add(column.strings()[row]); + String candidate = ""; + while (present.contains(candidate)) candidate += "\0"; + return candidate; + } + // EXT_structural_metadata forbids NaN and infinities: NaN is written as noData, the most negative finite value; // +inf and -inf as the largest finite value and the next one after the most negative, declared in the property's // extras (FSPM_infinity). @@ -507,6 +606,10 @@ public static Frame fromBytes(byte[] data) { if (entry.hasNonNull("platform")) dataStructure.platform = entry.get("platform").asText(); if (entry.hasNonNull("model")) dataStructure.model = entry.get("model").asText(); if (entry.get("extras") instanceof ObjectNode extras) dataStructure.extras = extras; + for (Iterator> labels = entry.path("labels").fields(); labels.hasNext(); ) { + Map.Entry label = labels.next(); + dataStructure.labels.put(label.getKey(), label.getValue().asText()); + } if (entry.has("origin")) { JsonNode origin = entry.get("origin"); if (origin.size() != 16) throw new FspmTfException("data structure '" + dataStructure.id + "': an origin of 16 numbers"); @@ -517,6 +620,7 @@ public static Frame fromBytes(byte[] data) { Map tableNames = new HashMap<>(); for (JsonNode item : entry.path("entitySets")) { EntitySet set = new EntitySet("", item.hasNonNull("rank") ? item.get("rank").asLong() : null); + if (item.hasNonNull("scale")) set.scale = item.get("scale").asText(); decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), set); tableNames.put(item.get("propertyTable").asInt(), item.get("name").asText()); dataStructure.entitySets.add(set); @@ -630,6 +734,17 @@ private static void decodeTable(JsonNode document, byte[] binary, JsonNode metad text[row] = new String(bytes, (int) offsets[row], (int) (offsets[row + 1] - offsets[row]), StandardCharsets.UTF_8); } column = new Column(ComponentType.STRING, text); + if (property.has("noData")) { // missing text + String noData = property.get("noData").asText(); + boolean[] missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) + if (text[row].equals(noData)) { + missing[row] = any = true; + text[row] = ""; + } + if (any) column.missing = missing; + } } case "BOOLEAN" -> { if (raw.remaining() * 8L < count) throw new FspmTfException("'" + classId + "." + name + "': too few bits"); @@ -650,6 +765,29 @@ private static void decodeTable(JsonNode document, byte[] binary, JsonNode metad } column = readScalars(componentType, raw, count, width); restoreStandIns(column, property); + if (property.path("extras").path("FSPM_boolean").asBoolean(false)) { // booleans with missing values + long noData = property.path("noData").asLong(BOOLEAN_NO_DATA); + boolean[] values = new boolean[count], missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) { + long code = (long) column.doubleAt(row); + missing[row] = code == noData; + any |= missing[row]; + values[row] = !missing[row] && code == 1; + } + column = new Column(ComponentType.BOOLEAN, values); + if (any) column.missing = missing; + } else if (width == 1 && !componentType.isFloat() && property.has("noData") + && !name.equals("complex") && !name.equals("parent")) { // missing integers + long noData = property.get("noData").asLong(); + boolean[] missing = new boolean[count]; + boolean any = false; + for (int row = 0; row < count; row++) { + missing[row] = rawLong(column, row) == noData; + any |= missing[row]; + } + if (any) column.missing = missing; + } } default -> throw new FspmTfException("'" + classId + "." + name + "': property type " + type + " is not part of fspm-TF 0.1"); } @@ -666,6 +804,7 @@ private static void decodeTable(JsonNode document, byte[] binary, JsonNode metad if (declared.hasNonNull("maximum")) variable.maximum = declared.get("maximum").asDouble(); for (JsonNode reference : declared.path("references")) variable.references.add(reference.asText()); for (JsonNode doi : declared.path("doi")) variable.doi.add(doi.asText()); + if (declared.hasNonNull("format")) variable.format = declared.get("format").asText(); } if (property.has("description")) variable.description = property.get("description").asText(); table.variables.put(name, variable); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Column.java b/bindings/java/src/main/java/org/openalea/fspmtf/Column.java index de3c7cc..b3cb02a 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Column.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Column.java @@ -5,12 +5,14 @@ /** * A column: its values (a Java array, see {@link ComponentType}), {@code width} per row (1: one value per row; 2 or more: - * a fixed-size vector per row, row after row). + * a fixed-size vector per row, row after row), and which rows are missing ({@code missing}, one flag per row, null when + * none is; NaN in a float column). */ public final class Column { public final ComponentType type; public final Object values; public final int width; + public boolean[] missing; public Column(ComponentType type, Object values, int width) { if (!values.getClass().isArray()) throw new FspmTfException("a column's values are an array"); @@ -45,6 +47,22 @@ public Column(ComponentType type, Object values) { /** A vector column: {@code width} values per row, row after row. */ public static Column vectors(int width, double... values) { return new Column(ComponentType.FLOAT64, values, width); } + /** Marks rows as missing (one flag per row); returns the column, for chaining. */ + public Column withMissing(boolean... rows) { + if (rows.length != rows()) throw new FspmTfException("one missing flag per row"); + missing = rows; + return this; + } + + public boolean isMissing(int row) { + return missing != null && missing[row]; + } + + public boolean hasMissing() { + if (missing != null) for (boolean flag : missing) if (flag) return true; + return false; + } + /** The number of values (rows × width). */ public int size() { return Array.getLength(values); @@ -81,7 +99,8 @@ public double doubleAt(int index) { @Override public boolean equals(Object other) { if (!(other instanceof Column column)) return false; - return type == column.type && width == column.width && Arrays.deepEquals(new Object[] {values}, new Object[] {column.values}); + return type == column.type && width == column.width && Arrays.deepEquals(new Object[] {values}, new Object[] {column.values}) + && hasMissing() == column.hasMissing() && (!hasMissing() || Arrays.equals(missing, column.missing)); } @Override diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java b/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java index 72dab03..47a4067 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/DataStructure.java @@ -23,6 +23,8 @@ public final class DataStructure { public String platform; public String model; public ObjectNode extras = JsonNodeFactory.instance.objectNode(); + /** The platform's labels to the vocabulary's. */ + public final java.util.Map labels = new java.util.LinkedHashMap<>(); public DataStructure(String id) { this.id = id; diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java index 0b24fd3..41d270c 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java @@ -18,6 +18,10 @@ public final class Dump { private Dump() {} private static void addValue(ArrayNode array, Column column, int index) { + if (column.width == 1 && column.isMissing(index)) { // missing values: null + array.addNull(); + return; + } switch (column.type) { case FLOAT32, FLOAT64 -> { double value = column.type == ComponentType.FLOAT32 ? column.floats()[index] : column.doubles()[index]; @@ -64,6 +68,7 @@ private static ObjectNode tableJson(Table table) { if (variable.maximum != null) declared.put("maximum", variable.maximum); if (!variable.references.isEmpty()) variable.references.forEach(declared.putArray("references")::add); if (!variable.doi.isEmpty()) variable.doi.forEach(declared.putArray("doi")::add); + if (variable.format != null) declared.put("format", variable.format); } if (!declarations.isEmpty()) entry.set("variables", declarations); return entry; @@ -90,11 +95,16 @@ public static ObjectNode frameToJson(Frame frame) { for (EntitySet set : dataStructure.entitySets) { ObjectNode item = tableJson(set); if (set.rank == null) item.putNull("rank"); else item.put("rank", set.rank); + if (set.scale != null) item.put("scale", set.scale); sets.add(item); } if (dataStructure.platform != null) entry.put("platform", dataStructure.platform); if (dataStructure.model != null) entry.put("model", dataStructure.model); if (!dataStructure.extras.isEmpty()) entry.set("extras", dataStructure.extras); + if (!dataStructure.labels.isEmpty()) { + ObjectNode labels = entry.putObject("labels"); + dataStructure.labels.forEach(labels::put); + } if (!dataStructure.relations.isEmpty()) { ArrayNode relations = entry.putArray("relations"); for (Relation relation : dataStructure.relations) diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java b/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java index 7424789..2ad661b 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/EntitySet.java @@ -11,6 +11,8 @@ public class EntitySet extends Table { public static final long NO_ID = -1; public Long rank; + /** Its base scale (Plant, Axis, ..., Connection), or "mixed". */ + public String scale; public EntitySet(String name, Long rank) { super(name); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Table.java b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java index f56dc20..b08b17e 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Table.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java @@ -10,7 +10,7 @@ public class Table { /** The kinds of tables, each with its reserved columns. */ public enum Kind { SCALARS(List.of()), - ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index")), + ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index", "order")), RELATION(List.of("id", "source", "target", "relation")), MAPPING(List.of("source", "target", "weight")); @@ -73,6 +73,9 @@ public List variableNames() { public void checkLengths() { for (Map.Entry entry : columns.entrySet()) { Column column = entry.getValue(); + if (column.hasMissing() && (isReserved(entry.getKey()) || column.width != 1)) + throw new FspmTfException("column '" + entry.getKey() + "' of '" + name + + "': missing values only in variables of one value per row"); if (column.size() % column.width != 0) throw new FspmTfException("column '" + entry.getKey() + "' of '" + name + "': its values are not whole rows"); if (column.rows() != rowCount()) throw new FspmTfException("table '" + name + "': columns of different lengths"); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java b/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java index b7386f9..a0d56ac 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Variable.java @@ -18,6 +18,8 @@ public final class Variable { public Double maximum; public List references = new ArrayList<>(); public List doi = new ArrayList<>(); + /** A text column of dates or times: "date", "date-time" or "time" (ISO 8601). */ + public String format; public Variable() {} @@ -35,7 +37,7 @@ public boolean equals(Object other) { && Objects.equals(kind, variable.kind) && Objects.equals(role, variable.role) && Objects.equals(description, variable.description) && Objects.equals(minimum, variable.minimum) && Objects.equals(maximum, variable.maximum) && references.equals(variable.references) - && doi.equals(variable.doi); + && doi.equals(variable.doi) && Objects.equals(format, variable.format); } @Override diff --git a/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java index 7e1f28a..48b476d 100644 --- a/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java +++ b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java @@ -40,8 +40,8 @@ static JsonNode asJson(Frame frame) throws IOException { } @Test - void nineConformanceFiles() throws IOException { - assertEquals(9, conformanceFiles().count()); + void tenConformanceFiles() throws IOException { + assertEquals(10, conformanceFiles().count()); } @ParameterizedTest diff --git a/doc/java.md b/doc/java.md index 1936d29..3bfde44 100644 --- a/doc/java.md +++ b/doc/java.md @@ -28,7 +28,8 @@ Codec.write(Path.of("plants.glb"), written); ``` A `Column` holds a Java array of its type (`long[]`, `double[]`, `boolean[]`, `String[]`, …; unsigned integers in -the array of their width), with a `width` of 2 or more for a fixed-size vector per row. NaN and infinities are written +the array of their width), with a `width` of 2 or more for a fixed-size vector per row, and its missing rows +(`withMissing(...)`, `isMissing(row)`; NaN in a float column). NaN and infinities are written with the specification's stand-ins and read back as they were. ## GroIMP @@ -44,8 +45,10 @@ with the specification's stand-ins and read back as they were. | relations | the other edge bits (containment, mark, notifies, user edges), one relation per bit and pair of sets | | variables | the managed fields of the model's classes (numbers, booleans, text; vecmath tuples as vectors) | -`options.scaleNames` and `options.scaleRanks` give each scale its entity set's name and rank, e.g. the vocabulary's -`Organ`, rank 5. `options.variables` declares the fields' units and standard names. GroIMP's fields have no units. +`options.scaleNames`, `options.scaleRanks` and `options.scaleScales` give each scale its entity set's name, rank and +base scale, e.g. the vocabulary's `Organ`, rank 5. `options.labels` maps the model's class names to the vocabulary's +labels, and `options.variables` declares the fields' units and standard names (GroIMP's fields have no units). A field +that other classes of the same scale lack is missing in their rows. `GroImp.toGraph(dataStructure, label -> new Internode(), ids)` builds the nodes back: their fields, refinement, successor and branch edges, and edge bits. It does this for any graph, e.g. metafspm's Plant, Axis and SubOrgan become three scales linked by refinement edges. diff --git a/implementations.md b/implementations.md index 0ab7149..c9560b9 100644 --- a/implementations.md +++ b/implementations.md @@ -6,8 +6,8 @@ | openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.2 (through openalea.fspm_tf) | | fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.2 | | CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.2 | -| fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.1 | -| GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.1 | +| fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.2 | +| GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.2 | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.2 | | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.2 | | VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.2 | From 5f5a2a782a864e9006e467e7967ca29946cae2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:15:41 +0100 Subject: [PATCH 15/39] Reading the specification (doc/reading_the_specification.md): what spec/ holds and how each file is checked, which source prevails, a reading path, the three layers of a file with conformance/root_system.glb annotated, a glossary, how a codec proves itself, the standards it builds on; the conformance files' README in the built docs --- doc/conformance.md | 2 + doc/index.md | 2 + doc/reading_the_specification.md | 132 +++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 doc/conformance.md create mode 100644 doc/reading_the_specification.md diff --git a/doc/conformance.md b/doc/conformance.md new file mode 100644 index 0000000..5da5ab7 --- /dev/null +++ b/doc/conformance.md @@ -0,0 +1,2 @@ +```{include} ../conformance/README.md +``` diff --git a/doc/index.md b/doc/index.md index 5cd94e6..144890c 100644 --- a/doc/index.md +++ b/doc/index.md @@ -45,9 +45,11 @@ For platform developers: how a platform reads and writes the format, and how the ```{toctree} :maxdepth: 1 +Reading the specification Implementing a platform Changing the format Implementations +The conformance files ``` ## Reference diff --git a/doc/reading_the_specification.md b/doc/reading_the_specification.md new file mode 100644 index 0000000..c673706 --- /dev/null +++ b/doc/reading_the_specification.md @@ -0,0 +1,132 @@ +# Reading the specification + +This page is for a platform's developers, before they read the specification itself. It explains: + +- what the `spec/` folder holds; +- in which order to read it; +- how a file is built; +- what each word means; +- how a codec proves that it follows the format. + +## What is agreed, and where + +Everything the platforms agree on is in the repository's `spec/` folder, together with the conformance files: + +| File | What it fixes | How it is checked | +|---|---|---| +| `spec/specification.md` | the rules, in words (MUST, SHOULD, MAY as in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)) | by the validator, `fspm-tf validate` | +| `spec/schema/fspm_scene.schema.json` | the shape of the `FSPM_scene` JSON (its fields and their types) | by any JSON Schema validator | +| `spec/schema/index.schema.json` | the shape of a sequence's index, `scene.fspm.json` | by any JSON Schema validator | +| `spec/vocabulary/standard_names.csv` | the shared quantities: canonical unit, kind, definition, and each platform's names for them | by the validator (units, kinds) | +| `spec/vocabulary/labels.csv` | the entity labels, with their scale and Plant Ontology term | by the validator (warnings) | +| `spec/vocabulary/enums.json` | the allowed values: edge types, kinds, roles, base scales, formats… | by the schemas and the validator | +| `conformance/` | the specification as test cases: reference files and what a reader must find in them | `fspm-tf compare-all` | + +When they disagree, `specification.md` prevails, then the schemas, then the conformance files. A disagreement is a +defect of the lower one. A change of the format is a pull request to `spec/`, with its conformance scene +([Changing the format](governance.md)). + +## A reading path + +1. **§1, the data model**, all of it: frames, data structures, entity sets, relations, mappings, meshes, variables. + Everything else describes how these are written. +2. **§4.2, the reserved columns.** `id`, `label`, `complex`, `parent`, `edge_type` and `order` carry the plant's + topology (an MTG's decomposition `/`, successor `<` and branch `+`). +3. **§5 and §6, variables and the vocabulary**: units, kinds, roles, standard names. This is what lets two platforms + recognise the same quantity. +4. **§2, §3, §4.1, §7, the encoding.** These come with the example below; a codec's writer needs them in full. +5. **§8 and §9, sequences and conformance.** + +## The three layers of a file + +A frame is a standard glTF 2.0 binary file (`.glb`): a JSON part and a binary part. Any glTF viewer opens it. Three +layers hold the scene: + +| Layer | Defined by | Holds | +|---|---|---| +| Property tables | the glTF extension [`EXT_structural_metadata`](https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_structural_metadata) | every table: one column per variable, with its values in the binary part | +| Feature IDs | the glTF extension [`EXT_mesh_features`](https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_mesh_features) | for each vertex of a mesh, the row of its entity in one or several tables | +| `FSPM_scene` | fspm-TF (`spec/schema/fspm_scene.schema.json`) | what glTF cannot say: the time, the data structures, which table is which scale, relations, grids, mappings, the variables' units and standard names | + +Here is the JSON part of `conformance/root_system.glb`, abridged: + +```text +"extensions": { + "FSPM_scene": { + "version": "0.2", "vocabulary": "0.2", + "frame": {"time": 0.0, "timeUnit": "d", "step": 0}, + "dataStructures": [{ + "id": "root_system", "kind": "graph", "origin": [1, 0, 0, 0, ...], a 4×4 matrix, column by column + "entitySets": [{"name": "Plant", "rank": 1, "propertyTable": 0}, one table per scale + {"name": "Axis", "rank": 2, "propertyTable": 1}, + {"name": "SubOrgan", "rank": 6, "propertyTable": 2}], + "scalars": 3, a one-row table for the whole plant + "meshes": [{"name": "roots", "mesh": 0}]}], + "variables": {"root_system__SubOrgan": { + "length": {"unit": "m", "standardName": "length", "kind": "extensive", "role": "state"}, ...}}}, + "EXT_structural_metadata": { + "schema": {"classes": {"root_system__SubOrgan": {"properties": { + "id": {"type": "SCALAR", "componentType": "INT64", "required": true}, + "label": {"type": "ENUM", "enumType": "root_system__SubOrgan__label"}, + "complex": {"type": "SCALAR", "componentType": "INT64", "noData": -1}, the axis it belongs to + "parent": {"type": "SCALAR", "componentType": "INT64", "noData": -1}, the segment it follows + "edge_type": {"type": "ENUM", "enumType": "FSPM_edge_type"}, none, successor, branch, unknown + "length": {"type": "SCALAR", "componentType": "FLOAT64"}, ...}}}}, + "propertyTables": [..., {"class": "root_system__SubOrgan", "count": 10, + "properties": {"id": {"values": 9}, "length": {"values": 14}, ...}}]}} buffer views +"meshes": [{"primitives": [{"attributes": {"POSITION": 0, "_FEATURE_ID_0": 1, "_FEATURE_ID_1": 2, "_FEATURE_ID_2": 3}, + "extensions": {"EXT_mesh_features": {"featureIds": [ + {"attribute": 0, "propertyTable": 2, "label": "SubOrgan"}, each vertex's segment, + {"attribute": 1, "propertyTable": 1, "label": "Axis"}, its axis, + {"attribute": 2, "propertyTable": 0, "label": "Plant"}]}}}]}] and its plant +``` + +To read the segments' lengths, a reader: + +1. finds the `SubOrgan` entity set in `FSPM_scene`, and its `propertyTable` (2); +2. finds that table's `length` property and its buffer view (14); +3. reads `count` (10) FLOAT64 values from the binary part; +4. finds their unit (`m`) and standard name in `variables`. + +`fspm-tf info file.glb` lists what a frame holds: its data structures, tables, columns and their declarations. + +## Words + +| Word | Meaning | +|---|---| +| **frame** | the scene at one time step: one `.glb` file | +| **sequence** | a folder of frames and their index, `scene.fspm.json` | +| **data structure** | one plant population's graph, or one environment model's grid (metafspm's word). Not glTF's *node*: each data structure has a glTF node, which only places it for viewers. | +| **entity set** | a table of entities of one kind: the entities of one scale of a graph, or a grid's cells | +| **scale** | a level of description: plant, axis, organ, segment, cell… The vocabulary lists base scales (`enums.json`); an entity set may declare its own (`scale`). | +| **rank** | an entity set's order among the scales, coarser first. It SHOULD be the base scale's position: Plant 1, Axis 2, Organ 5, SubOrgan 6. | +| **entity** | one row of an entity set: a plant, a root axis, a segment, a cell | +| **relation** | an edge table between two entity sets: a cell network, a branch's bearer | +| **mapping** | weights between the entities of two data structures, e.g. root segments and soil voxels | +| **class** | `EXT_structural_metadata`'s word for a table's type: its columns. fspm-TF names them `__`. | +| **variable** | a column that is not reserved, with its declaration: unit, standard name, kind, role | +| **standard name** | the vocabulary's name for a quantity, the same on every platform (`length`, `water_potential`) | +| **codec** | a language's reading and writing of the format (`bindings//`) | +| **adapter** | a platform's model to and from the codec's tables | + +## How a codec proves it follows the format + +`conformance/` holds reference scenes, each a GLB with what a reader must find in it (`*.expected.json`, in the layout +of [the conformance files](conformance.md)). A codec: + +1. reads each GLB and writes what it read as JSON, in that layout; +2. writes the same scenes back as GLB files. + +`fspm-tf compare-all` then compares its JSON with the expected values, exactly. `fspm-tf validate` and Khronos' +[glTF-Validator](https://github.com/KhronosGroup/glTF-Validator) check what it wrote. In GitHub Actions, the +composite action `.github/actions/conformance` runs these checks. The Python, Julia, C++ and Java codecs all pass them +([Implementations](implementations.md)). + +## Standards it builds on + +- glTF 2.0: +- `EXT_structural_metadata` and `EXT_mesh_features`: the links above (Cesium's 3D Tiles vendor extensions) +- UCUM, the units: +- I-ADOPT, the decomposition of the vocabulary's names: +- the Plant Ontology, the labels' terms: +- RFC 2119, MUST and SHOULD: From 2142d9871ebd0cbf2065da1ed7d63d3e6fd989bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 19:17:28 +0100 Subject: [PATCH 16/39] CPlantBox's adapter: a soil solver's values per cell (add_cell_values, reordered to the cells' C order) and CPlantBox's hydraulics per segment and per node (add_segment_values, add_node_values: a segment takes its distal node's value), declared with their units and standard names; tests place each value where it belongs; doc/cpp.md --- .../cplantbox/include/fspm_tf/cplantbox.hpp | 22 ++++++ bindings/cpp/cplantbox/src/cplantbox.cpp | 71 +++++++++++++++++++ .../cpp/cplantbox/test/test_cplantbox.cpp | 34 ++++++++- doc/cpp.md | 17 +++++ doc/reading_the_specification.md | 2 +- 5 files changed, 144 insertions(+), 2 deletions(-) diff --git a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp index 30bbc9e..eca6657 100644 --- a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp +++ b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp @@ -71,6 +71,28 @@ DataStructure soil_grid(const CPlantBox::MappedSegments &segments, const std::st Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataStructure &plant, const DataStructure &soil, const std::string &name = "roots_in_soil"); +/// Values to write with their declaration: e.g. a soil solver's water potential per cell, or CPlantBox's hydraulic +/// results per segment or per node (PlantHydraulicModel), declared with their units and standard names. +struct Values { + std::string name; ///< the column's name + std::vector values; ///< one per cell, segment or node, in CPlantBox's order + Variable variable; ///< its unit, standard name, kind, role +}; + +/// Adds values per soil cell (in CPlantBox's cell order: x fastest, then y, then z) to the soil grid written by +/// soil_grid, as columns of its cells (reordered to fspm-TF's C order of x, y, z). +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values); + +/// Adds values per segment of the MappedSegments (e.g. PlantHydraulicModel's radial fluxes) to the plant's SubOrgan +/// set, each in the row of its segment. The segments must keep CPlantBox's numbering (a grid set without cutting). +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + +/// Adds values per node of the MappedSegments (e.g. PlantHydraulicModel's xylem potentials) to the plant's SubOrgan +/// set: each segment takes its distal node's value. +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values); + /// A graph as a static CPlantBox root system (StaticRoot organs on a seed, as CPlantBox's StaticPlant builds from /// RSML): its segments' ends as the organs' nodes, a lateral at its parent's node, each organ's radius the mean of its /// segments'. The segments: the entity set holding the segment ends (proximal_end_* and distal_end_*, by standard diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp index 020eda4..c5f6d78 100644 --- a/bindings/cpp/cplantbox/src/cplantbox.cpp +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -304,6 +304,77 @@ Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataSt return mapping; } +void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values) { + if (!soil.grid) throw Error("'" + soil.id + "' has no grid: write it with soil_grid"); + const auto &resolution = segments.resolution; + const auto nx = std::size_t(resolution.x), ny = std::size_t(resolution.y), nz = std::size_t(resolution.z); + auto &cells = soil.entity_set(soil.grid->cells); + for (const auto &item : values) { + if (item.values.size() != nx * ny * nz) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, the grid has " + + std::to_string(nx * ny * nz) + " cells"); + std::vector column(item.values.size()); + for (std::size_t cell = 0; cell < item.values.size(); ++cell) { // CPlantBox: x fastest; fspm-TF: C order + const std::size_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); + column[(i * ny + j) * nz + k] = item.values[cell]; + } + cells.set(item.name, column).declare(item.name, item.variable); + } +} + +namespace { + +/// The SubOrgan row of each segment of the MappedSegments (by its distal node's id), or throws. +std::vector segment_rows(const DataStructure &plant, const CPlantBox::MappedSegments &segments) { + const auto &set = plant.entity_set("SubOrgan"); + std::map row_of; + const auto &ids = set.column("id").as(); + for (std::size_t row = 0; row < ids.size(); ++row) row_of[ids[row]] = row; + std::vector rows; + for (std::size_t segment = 0; segment < segments.segments.size(); ++segment) { + const auto found = row_of.find(SEGMENT_ID_OFFSET + segments.segments[segment].y); + if (found == row_of.end()) + throw Error("segment " + std::to_string(segment) + " is not in '" + plant.id + + "': set the grid without cutting segments"); + rows.push_back(found->second); + } + return rows; +} + +void add_per_segment(DataStructure &plant, const std::vector &rows, const Values &item, + const std::vector &per_segment) { + auto &set = plant.entity_set("SubOrgan"); + std::vector column(set.row_count(), std::nan("")); // segments outside the mapped ones: NaN + for (std::size_t segment = 0; segment < rows.size(); ++segment) column[rows[segment]] = per_segment[segment]; + set.set(item.name, column).declare(item.name, item.variable); +} + +} // namespace + +void add_segment_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != rows.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(rows.size()) + " segments"); + add_per_segment(plant, rows, item, item.values); + } +} + +void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segments, + const std::vector &values) { + const auto rows = segment_rows(plant, segments); + for (const auto &item : values) { + if (item.values.size() != segments.nodes.size()) + throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, " + + std::to_string(segments.nodes.size()) + " nodes"); + std::vector per_segment; // each segment: its distal node's value + for (const auto &segment : segments.segments) per_segment.push_back(item.values.at(std::size_t(segment.y))); + add_per_segment(plant, rows, item, per_segment); + } +} + // ── A graph as a static root system ────────────────────────────────────── namespace { diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp index c5a28f7..22be38f 100644 --- a/bindings/cpp/cplantbox/test/test_cplantbox.cpp +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -152,7 +152,7 @@ void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const plant->setRectangularGrid(CPlantBox::Vector3d(-10, -10, -40), CPlantBox::Vector3d(10, 10, 0), CPlantBox::Vector3d(4, 4, 8), false); - const auto soil = cplantbox::soil_grid(*plant); + auto soil = cplantbox::soil_grid(*plant); const auto mapping = cplantbox::segments_in_soil(*plant, data_structure, soil); check(soil.entity_set("Voxel").row_count() == 4 * 4 * 8, "the soil grid's cells"); check(mapping.row_count() > 0, "segments mapped to soil cells"); @@ -177,6 +177,38 @@ void test_plant_with_geometry_and_soil(const filesystem::path ¶meters, const } check(inside, "each mapped segment's midpoint lies in its cell (C order of x, y, z)"); + // A soil solver's values per cell and CPlantBox's hydraulics per node and segment, each where it belongs + std::vector soil_potential, xylem_potential, radial_flux; + for (std::size_t cell = 0; cell < 4 * 4 * 8; ++cell) soil_potential.push_back(-100. - double(cell)); + for (const auto &node : plant->nodes) xylem_potential.push_back(node.z); // the node's own z + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) radial_flux.push_back(double(segment)); + Variable potential; + potential.unit = "cm[H2O]"; + potential.standard_name = "soil_water_potential"; + cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", soil_potential, potential}}); + potential.standard_name = "water_potential"; + Variable flux; + flux.unit = "cm3/d"; + flux.standard_name = "root_water_uptake"; + cplantbox::add_node_values(data_structure, *plant, {{"xylem_potential", xylem_potential, potential}}); + cplantbox::add_segment_values(data_structure, *plant, {{"radial_flux", radial_flux, flux}}); + const auto cell_values = values(soil.entity_set("Voxel"), "soil_water_potential"); + bool cells_right = true; + for (std::size_t item = 0; item < sources.size(); ++item) { // each mapped segment's cell: its own value + std::size_t segment = 0; + while (cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y != sources[item]) ++segment; + cells_right = cells_right && cell_values[std::size_t(targets[item])] == soil_potential[std::size_t(plant->seg2cell.at(int(segment)))]; + } + check(cells_right, "soil values reordered to the cells' C order"); + const auto xylem = values(data_structure.entity_set("SubOrgan"), "xylem_potential"); + const auto distal_z = values(data_structure.entity_set("SubOrgan"), "z2"); + bool nodes_right = true; + for (std::size_t segment = 0; segment < plant->segments.size(); ++segment) { + const auto row = row_of.at(cplantbox::SEGMENT_ID_OFFSET + plant->segments[segment].y); + nodes_right = nodes_right && xylem[row] == distal_z[row]; + } + check(nodes_right, "each segment takes its distal node's value"); + Frame frame; frame.time = plant->getSimTime(); frame.data_structures = {data_structure, soil}; diff --git a/doc/cpp.md b/doc/cpp.md index 905c2a6..d47be02 100644 --- a/doc/cpp.md +++ b/doc/cpp.md @@ -71,6 +71,19 @@ frame.mappings.push_back(fspm_tf::cplantbox::segments_in_soil(*plant, frame.data frame.data_structures.push_back(soil); fspm_tf::write_frame("plant.glb", frame); +// A soil solver's values per cell (DuMux, or Python), CPlantBox's hydraulics per node and per segment +// (PlantHydraulicModel: rx, radial fluxes), each declared with its unit and standard name +fspm_tf::Variable potential; +potential.unit = "cm[H2O]"; +potential.standard_name = "soil_water_potential"; +fspm_tf::cplantbox::add_cell_values(soil, *plant, {{"soil_water_potential", sx, potential}}); +potential.standard_name = "water_potential"; +fspm_tf::cplantbox::add_node_values(frame.data_structures[0], *plant, {{"xylem_potential", rx, potential}}); +fspm_tf::Variable uptake; +uptake.unit = "cm3/d"; +uptake.standard_name = "root_water_uptake"; +fspm_tf::cplantbox::add_segment_values(frame.data_structures[0], *plant, {{"radial_flux", fluxes, uptake}}); + // A frame (CPlantBox's, or another platform's) as a static CPlantBox root system auto roots = fspm_tf::cplantbox::to_root_system(fspm_tf::read_frame("roots.glb").data_structures[0]); ``` @@ -84,6 +97,10 @@ auto roots = fspm_tf::cplantbox::to_root_system(fspm_tf::read_frame("roots.glb") | other variables | CPlantBox's names and units (`length` in cm, `age` in d, `subType`, …), with their standard names; its `order` as `branching_order` (`order` is fspm-TF's reserved column) | | `extras["cplantbox"]` | the id offsets, the organism's parameters as CPlantBox's XML | +CPlantBox's core has no soil model: the soil's values come from a solver outside it, which hands them to CPlantBox +per cell. `add_cell_values` writes them in the soil grid's cells, reordered from CPlantBox's cell order (x fastest) to +fspm-TF's (C order of x, y, z). `add_node_values` gives each segment its distal node's value. + CPlantBox has two scales below the plant. `to_root_system` reads frames with more scales by what they hold: the segments are the entity set with segment ends, and the organs are the set their `complex` points to. For example, metafspm's Plant, Axis and SubOrgan become CPlantBox root organs. CPlantBox has static roots only, so stems and diff --git a/doc/reading_the_specification.md b/doc/reading_the_specification.md index c673706..15d08fb 100644 --- a/doc/reading_the_specification.md +++ b/doc/reading_the_specification.md @@ -88,7 +88,7 @@ To read the segments' lengths, a reader: 3. reads `count` (10) FLOAT64 values from the binary part; 4. finds their unit (`m`) and standard name in `variables`. -`fspm-tf info file.glb` lists what a frame holds: its data structures, tables, columns and their declarations. +`fspm-tf info file.glb` lists what a frame holds: its data structures, their tables and their variables. ## Words From 9333facdb3d1167aded4cf939c9539f140c83f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Mon, 2 Nov 2026 20:08:12 +0100 Subject: [PATCH 17/39] The reserved column order renamed sibling_order (D-14, user): order usually means the branching order, which platforms name order (CPlantBox's organs, OpenAlea MTGs); every codec, the validator, the adapters, the conformance files and the docs; CPlantBox's organs keep their own order --- CHANGELOG.md | 4 ++-- .../cplantbox/include/fspm_tf/cplantbox.hpp | 3 +-- bindings/cpp/cplantbox/src/cplantbox.cpp | 5 ++--- bindings/cpp/src/codec.cpp | 3 ++- bindings/cpp/src/model.cpp | 2 +- .../main/java/org/openalea/fspmtf/Codec.java | 2 +- .../main/java/org/openalea/fspmtf/Table.java | 2 +- .../ext/FspmTFMultiScaleTreeGraphExt.jl | 12 ++++++------ .../julia/FspmTF/ext/FspmTFPlantGraphsExt.jl | 8 ++++---- bindings/julia/FspmTF/src/model.jl | 6 +++--- .../julia/FspmTF/test/plantgraphs/runtests.jl | 2 +- bindings/julia/FspmTF/test/runtests.jl | 2 +- bindings/python/src/openalea/fspm_tf/model.py | 7 ++++--- .../python/src/openalea/fspm_tf/validate.py | 4 ++-- bindings/python/test/test_validate.py | 2 +- conformance/README.md | 2 +- conformance/generate.py | 2 +- conformance/missing_values.expected.json | 2 +- conformance/missing_values.glb | Bin 6972 -> 6988 bytes doc/cpp.md | 2 +- doc/julia.md | 4 ++-- doc/reading_the_specification.md | 2 +- spec/specification.md | 6 +++--- 23 files changed, 42 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 170a9a0..745229f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,13 +5,13 @@ - Specification 0.2, additive (0.1 files stay valid), from the bindings' points (`spec/specification.md`, "Changes from 0.1"): missing values in integer, text and boolean columns; an entity set's `scale` and ranks as base scale positions; reading by content; position columns in their declared unit; a data structure's `labels`; the extras - key; `edge_type` `unknown`; the `order` column; parents in another complex; bearer relations' labels and names; + key; `edge_type` `unknown`; the `sibling_order` column; parents in another complex; bearer relations' labels and names; dates (`format`); sparse ids; the conformance files' status and the sources' precedence. - Vocabulary 0.2: `Seed`, `mtg_index`, the new enumerations. - The reference codec, the validator, the ParaView reader and the conformance files (a new scene, `missing_values`). - The Julia, C++ and Java codecs on 0.2, and their adapters using it instead of their extras: MTG.jl (missing attributes, dates, children's order, `mtg_index`), PlantGraphs (`unknown` edges, children's order), CPlantBox (a - `mixed` Organ set, segment ends in cm, `branching_order`), GroIMP (missing fields, scales, labels); extras keys in + `mixed` Organ set, segment ends in cm), GroIMP (missing fields, scales, labels); extras keys in lower case (`virtualplantlab`, `cplantbox`, `groimp`). ## Specification 0.1 (draft) diff --git a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp index eca6657..556f83c 100644 --- a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp +++ b/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp @@ -10,8 +10,7 @@ // | SubOrgan (rank 6), id | its segments: complex its organ, parent the segment ending at its | // | 2^32 + distal node id | proximal node (successor along an organ, branch at a lateral's base) | // | positions (proximal_end_*, ...) | node coordinates, in cm (declared; readers convert) | -// | other variables | CPlantBox's names and units (cm, d), declared with their standard names; | -// | | its `order` as `branching_order` (`order` is fspm-TF's reserved column) | +// | other variables | CPlantBox's names and units (cm, d), declared with their standard names | // | extras["cplantbox"] | the id offsets, the organism's parameters (its XML) | // // CPlantBox has two scales below the plant: organs and segments. A frame with more scales (e.g. metafspm's Plant, diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp index c5f6d78..f4489f6 100644 --- a/bindings/cpp/cplantbox/src/cplantbox.cpp +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -127,13 +127,12 @@ DataStructure to_data_structure(const CPlantBox::Organism &organism, const Optio } organ_set.set("id", organ_ids).set("label", Column(labels)).set("complex", complexes).set("parent", parents) .set("edge_type", Column(edge_types)).set("organType", organ_types).set("subType", sub_types) - .set("branching_order", orders).set("age", ages).set("length", lengths).set("radius", radii) + .set("order", orders).set("age", ages).set("length", lengths).set("radius", radii) .set("creationTime", creation_times).set("parentNI", parent_node_indices).set("numberOfNodes", node_counts) .set("alive", alive).set("active", active); organ_set.declare("organType", declared("1", "descriptor", "CPlantBox's organ type: 1 seed, 2 root, 3 stem, 4 leaf", {}, "setting")) .declare("subType", declared("1", "descriptor", "CPlantBox's sub type: the organ's parameter set", {}, "setting")) - .declare("branching_order", declared("1", "descriptor", "CPlantBox's order: the branching order (0: a base organ)", - "branching_order")) + .declare("order", declared("1", "descriptor", "branching order (0: a base organ)", "branching_order")) .declare("age", declared("d", "intensive", "the organ's age", "age")) .declare("length", declared("cm", "extensive", "the organ's realized length", "length")) .declare("radius", declared("cm", "intensive", "the organ's radius", "radius", "parameter")) diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp index 0ae63b5..fe62682 100644 --- a/bindings/cpp/src/codec.cpp +++ b/bindings/cpp/src/codec.cpp @@ -25,7 +25,8 @@ const std::string METADATA = "EXT_structural_metadata"; const std::string FEATURES = "EXT_mesh_features"; const std::string EDGE_TYPE_ENUM = "FSPM_edge_type"; const std::vector ENUM_COLUMNS = {"label", "edge_type", "relation"}; -const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index", "order"}; +const std::vector ID_COLUMNS = {"id", "complex", "parent", "source", "target", "level", "index", + "sibling_order"}; constexpr std::uint8_t BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing constexpr int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; constexpr int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; diff --git a/bindings/cpp/src/model.cpp b/bindings/cpp/src/model.cpp index ec7a0a2..192899e 100644 --- a/bindings/cpp/src/model.cpp +++ b/bindings/cpp/src/model.cpp @@ -27,7 +27,7 @@ template <> struct TypeName { static constexpr const char *value = const std::vector NO_RESERVED = {}; const std::vector ENTITY_SET_RESERVED = {"id", "label", "complex", "parent", "edge_type", - "level", "index", "order"}; + "level", "index", "sibling_order"}; const std::vector RELATION_RESERVED = {"id", "source", "target", "relation"}; const std::vector MAPPING_RESERVED = {"source", "target", "weight"}; diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java index 2f7bd29..c10393a 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java @@ -33,7 +33,7 @@ public final class Codec { FEATURES = "EXT_mesh_features", EDGE_TYPE_ENUM = "FSPM_edge_type"; private static final List ENUM_COLUMNS = List.of("label", "edge_type", "relation"); private static final List ID_COLUMNS = List.of("id", "complex", "parent", "source", "target", "level", "index", - "order"); + "sibling_order"); private static final int BOOLEAN_NO_DATA = 255; // a boolean column with missing values: UINT8 0, 1, and 255 missing private static final int ARRAY_BUFFER = 34962, ELEMENT_ARRAY_BUFFER = 34963; private static final int FLOAT = 5126, UNSIGNED_INT = 5125, UNSIGNED_SHORT = 5123, UNSIGNED_BYTE = 5121; diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Table.java b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java index b08b17e..bd241c0 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Table.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Table.java @@ -10,7 +10,7 @@ public class Table { /** The kinds of tables, each with its reserved columns. */ public enum Kind { SCALARS(List.of()), - ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index", "order")), + ENTITY_SET(List.of("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order")), RELATION(List.of("id", "source", "target", "relation")), MAPPING(List.of("source", "target", "weight")); diff --git a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl index 803e44f..412c423 100644 --- a/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl @@ -9,7 +9,7 @@ A MultiScaleTreeGraph (MTG.jl) as an fspm-TF `graph` data structure, and back: | `complex` | its first ancestor at a smaller scale (none for the scene) | | `parent`, `edge_type` | its parent, when at the same scale and linked by `<` (successor) or `+` (branch) | | relation `bearer` | its parent otherwise, when it is not its complex linked by `/` (e.g. an axis borne by an internode) | -| `order` | the node's position among its parent's children | +| `sibling_order` | the node's position among its parent's children | | `mtg_index` (standard name `mtg_index`) | the node's `index` | | variables | its attributes: missing where a node has none; dates and times as ISO 8601 text with their `format` | | scalars | the scene's (the scale 0 root's) attributes | @@ -87,7 +87,7 @@ function attribute_columns(nodes, set_name, variables, skip = ()) end columns, declarations = Columns(), Declarations() for name in names - if string(name) in FspmTF.ENTITY_SET_RESERVED # e.g. an `order` attribute (a branching order) + if string(name) in FspmTF.ENTITY_SET_RESERVED # a reserved name, e.g. `label` @warn "attribute '$name' of '$set_name' not written: '$name' is a reserved column of fspm-TF" continue end @@ -184,7 +184,7 @@ function FspmTF.from_mtg(mtg; id::AbstractString = "plants", scales = Dict{Int,S (fspm_id(node), fspm_id(tree_parent), LINK_NAMES[link(node)])) end columns["parent"], columns["edge_type"] = parents, edge_types - columns["order"] = Int64[position_among_siblings(node) for node in scale_nodes] + columns["sibling_order"] = Int64[position_among_siblings(node) for node in scale_nodes] columns[INDEX_COLUMN] = Int64[index(node) for node in scale_nodes] attributes, declarations = attribute_columns(scale_nodes, set_name, variables, (Symbol(INDEX_COLUMN),)) merge!(columns, attributes) @@ -275,8 +275,8 @@ function FspmTF.to_mtg(data_structure::DataStructure) encoding[ids[row]] = (symbol = labels[row], index = indices[row], scale = scale_of(set)) attributes[ids[row]] = rows[row] end - haskey(set.columns, "order") && foreach(((id, position),) -> order_of[id] = position, - zip(ids, set.columns["order"])) + haskey(set.columns, "sibling_order") && foreach(((id, position),) -> order_of[id] = position, + zip(ids, set.columns["sibling_order"])) complexes = get(set.columns, "complex", fill(NO_ID, row_count(set))) for row in 1:row_count(set) # complex: by decomposition, '/' complexes[row] == NO_ID || (tree_parent[ids[row]] = (complexes[row], :/)) @@ -342,7 +342,7 @@ function FspmTF.to_mtg(data_structure::DataStructure) push!(get!(children_of, parent_id, Int64[]), id) end isempty(roots) || (children_of[root_id] = vcat(roots, get(children_of, root_id, Int64[]))) - for list in values(children_of) # the `order` column, else by id + for list in values(children_of) # the `sibling_order` column, else by id sort!(list; by = id -> (get(order_of, id, typemax(Int64)), id)) end stack = Any[root] diff --git a/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl index 656bff3..a2e89c9 100644 --- a/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl +++ b/bindings/julia/FspmTF/ext/FspmTFPlantGraphsExt.jl @@ -7,7 +7,7 @@ A VirtualPlantLab graph (PlantGraphs.jl: a `Graph` or a `StaticGraph`) as an fsp | `id`, `label` | the node's id, the name of its type (`Internode`, `Leaf`, …) | | `parent` | its nearest kept ancestor | | `edge_type` | `unknown`: PlantGraphs stores no edge types | -| `order` | the node's position among its kept parent's kept children | +| `sibling_order` | the node's position among its kept parent's kept children | | variables | the fields of each node's type: missing where a type has no such field; dates as ISO 8601 text with their `format` | | scalars | the graph's data (`data(graph)`), its fields | @@ -114,7 +114,7 @@ function FspmTF.from_graph(graph::Union{PG.Graph,PG.StaticGraph}; id::AbstractSt edge_types = String[parent == NO_ID ? "none" : "unknown" for parent in parents] # PlantGraphs stores none objects = [PG.data(node) for (node, _) in rows] columns = Columns("id" => ids, "label" => String[label_of(object) for object in objects], "parent" => parents, - "edge_type" => edge_types, "order" => order) + "edge_type" => edge_types, "sibling_order" => order) fields, declarations = field_columns(objects, entity_set, variables) merge!(columns, fields) extras = json_object("model" => "PlantGraphs", @@ -172,8 +172,8 @@ function FspmTF.to_graph(data_structure::DataStructure; types::AbstractDict, ent for row in 1:row_count(set) parents[row] == NO_ID ? push!(roots, row) : push!(get!(children_of, parents[row], Int[]), row) end - if haskey(set.columns, "order") # children in their order, else in their rows' - foreach(list -> sort!(list; by = row -> set.columns["order"][row]), values(children_of)) + if haskey(set.columns, "sibling_order") # children in their order, else in their rows' + foreach(list -> sort!(list; by = row -> set.columns["sibling_order"][row]), values(children_of)) end length(roots) == 1 || throw(ArgumentError("'$(set.name)': $(length(roots)) roots, a PlantGraphs graph has one")) static = PG.StaticGraph(PG.GraphNode(objects[only(roots)])) diff --git a/bindings/julia/FspmTF/src/model.jl b/bindings/julia/FspmTF/src/model.jl index ef465d0..53fa2f0 100644 --- a/bindings/julia/FspmTF/src/model.jl +++ b/bindings/julia/FspmTF/src/model.jl @@ -18,7 +18,7 @@ const NO_ID = -1 """The values of `edge_type`: the root of its scale, MTG `<`, MTG `+`.""" const EDGE_TYPES = ("none", "successor", "branch", "unknown") const ENUM_COLUMNS = ("label", "edge_type", "relation") -const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "order") +const ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "sibling_order") """The vocabulary's base scales: an entity set's `scale` is one of them, or `"mixed"`.""" const BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", "Connection") @@ -78,7 +78,7 @@ end The entities of one scale of a graph (`rank`: the scale's rank, coarser first), or a grid's cells (`rank` nothing). Reserved columns: `id` (Int64, stable across frames, never reused), `label` (String), `complex` (the id of the containing entity at a coarser scale, `NO_ID` if none), `parent` (the predecessor's id at the same scale, `NO_ID` if -none), `edge_type` (`"none"`, `"successor"`, `"branch"`, `"unknown"`), `order` (optional: its position among its tree +none), `edge_type` (`"none"`, `"successor"`, `"branch"`, `"unknown"`), `sibling_order` (optional: its position among its tree parent's children); an adaptive grid's cells: `level` and `index`. `scale`: its base scale (`BASE_SCALES`), or `"mixed"`. """ @@ -120,7 +120,7 @@ mutable struct Mapping <: AbstractTable end # The columns whose meaning the specification fixes, per kind of table -const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index", "order") +const ENTITY_SET_RESERVED = ("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order") const RELATION_RESERVED = ("id", "source", "target", "relation") const MAPPING_RESERVED = ("source", "target", "weight") reserved(::Table) = () diff --git a/bindings/julia/FspmTF/test/plantgraphs/runtests.jl b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl index 00b7135..0d2b9c2 100644 --- a/bindings/julia/FspmTF/test/plantgraphs/runtests.jl +++ b/bindings/julia/FspmTF/test/plantgraphs/runtests.jl @@ -43,7 +43,7 @@ plant() = Graph(axiom = Internode(0.1, 0.01) + (RU(30.0) + Leaf(0.05, 0.02, 3), ids = organs.columns["id"] @test organs.columns["parent"] == [-1, ids[1], ids[1], ids[1], ids[4]] # the leaves on the first internode @test organs.columns["edge_type"] == ["none", "unknown", "unknown", "unknown", "unknown"] # none stored - @test organs.columns["order"] == [0, 0, 1, 2, 0] # the leaves, then the internode + @test organs.columns["sibling_order"] == [0, 0, 1, 2, 0] # the leaves, then the internode @test isequal(organs.columns["length"], [0.1, 0.05, 0.04, 0.08, NaN]) # NaN: the meristem has none @test isequal(organs.columns["age"], [missing, 3, 1, missing, missing]) # missing where no field @test written.extras["virtualplantlab"]["model"] == "PlantGraphs" diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl index e254a47..5f6e1c2 100644 --- a/bindings/julia/FspmTF/test/runtests.jl +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -180,7 +180,7 @@ expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] @test isequal(organs.columns["isAlive"], [missing, false, true, true]) # Internode, Leaf, Internode, Leaf @test organs.variables["dateDeath"].format == "date" @test isequal(organs.columns["dateDeath"], [missing, "2022-08-24", missing, missing]) - @test organs.columns["order"] == [0, 0, 1, 0] # the leaf before the next internode + @test organs.columns["sibling_order"] == [0, 0, 1, 0] # the leaf before the next internode @test organs.variables["mtg_index"].standard_name == "mtg_index" @test Set(keys(written.extras["virtualplantlab"])) == Set(["model", "scales", "metadata", "scene"]) rebuilt = to_mtg(from_bytes(to_bytes(Frame(data_structures = [written]))).data_structures[1]) diff --git a/bindings/python/src/openalea/fspm_tf/model.py b/bindings/python/src/openalea/fspm_tf/model.py index c5a78d6..b0a4b43 100644 --- a/bindings/python/src/openalea/fspm_tf/model.py +++ b/bindings/python/src/openalea/fspm_tf/model.py @@ -25,12 +25,13 @@ # Columns whose meaning the specification fixes, per kind of table RESERVED = { - "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index", "order"), + "entity_set": ("id", "label", "complex", "parent", "edge_type", "level", "index", "sibling_order"), "relation": ("id", "source", "target", "relation"), "mapping": ("source", "target", "weight"), "scalars": (), } -ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", "order") # reserved, int64 +ID_COLUMNS = ("id", "complex", "parent", "source", "target", "level", "index", + "sibling_order") # reserved, int64 EDGE_TYPES = ("none", "successor", "branch", "unknown") # MTG: root of its scale, '<', '+'; a link not known BASE_SCALES = ("Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", "Connection") # an entity set's scale: one of these, or "mixed" @@ -98,7 +99,7 @@ class EntitySet(Table): """The entities of one scale of a graph (rank: the scale's rank, coarser first), or a grid's cells (rank None). Columns: id (int64, stable across frames, never reused), label (str), complex (the id of the containing entity at the coarser scale, NO_ID if none), parent (the predecessor's id at the same scale, NO_ID if none), edge_type - (str: 'none', 'successor', 'branch', 'unknown'); order (int64, optional: its position among its tree parent's + (str: 'none', 'successor', 'branch', 'unknown'); sibling_order (int64, optional: its position among its tree parent's children); an adaptive grid's cells: level and index; then the variables. *scale*: its base scale (BASE_SCALES), or "mixed".""" rank: Optional[int] = None diff --git a/bindings/python/src/openalea/fspm_tf/validate.py b/bindings/python/src/openalea/fspm_tf/validate.py index 8036694..d99c666 100644 --- a/bindings/python/src/openalea/fspm_tf/validate.py +++ b/bindings/python/src/openalea/fspm_tf/validate.py @@ -349,8 +349,8 @@ def _check_scales_and_labels(data_structure, report) -> None: and entity_set.rank != BASE_SCALES.index(entity_set.scale) + 1: report.warning(where, f"rank {entity_set.rank}: the base scale {entity_set.scale}'s position is " f"{BASE_SCALES.index(entity_set.scale) + 1}") - if "order" in entity_set.columns and (np.asarray(entity_set.columns["order"]) < 0).any(): - report.error(f"{where}/order", "positions among siblings are non-negative") + if "sibling_order" in entity_set.columns and (np.asarray(entity_set.columns["sibling_order"]) < 0).any(): + report.error(f"{where}/sibling_order", "positions among siblings are non-negative") if "label" in entity_set.columns: used |= {label for label in entity_set.columns["label"].tolist() if label is not None} vocabulary_labels = vocabulary.labels() diff --git a/bindings/python/test/test_validate.py b/bindings/python/test/test_validate.py index d05a8f0..c223820 100644 --- a/bindings/python/test/test_validate.py +++ b/bindings/python/test/test_validate.py @@ -128,7 +128,7 @@ def test_rules_of_0_2(): organs_of(frame).columns["edge_type"][0] = "unknown" # an unknown link needs a parent assert "'none' exactly for the entities without a parent" in messages(frame) frame = generate.missing_values() - organs_of(frame).columns["order"][0] = -1 + organs_of(frame).columns["sibling_order"][0] = -1 assert "positions among siblings are non-negative" in messages(frame) frame = generate.missing_values() frame.data_structures[0].relations[1].name = "bearer_Axis_Organ" diff --git a/conformance/README.md b/conformance/README.md index 95bb42a..8588750 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -12,7 +12,7 @@ numbers, the same bytes on every run; the CI checks that the committed files are | `soil_and_roots` | a root system, a regular soil grid with z pointing down, and their mapping | | `adaptive_soil` | an adaptive grid's leaf cells (`level`, `index`) | | `parameter_set` | a `table` data structure: parameters per plant and in the scalars, references, an infinite value | -| `missing_values` | integer, text, boolean and float columns with missing values; a date column (`format`); `unknown` edge types and an `order` column; a `mixed` entity set; a labels map; a parent in another axis; bearer relations between two pairs of sets | +| `missing_values` | integer, text, boolean and float columns with missing values; a date column (`format`); `unknown` edge types and a `sibling_order` column; a `mixed` entity set; a labels map; a parent in another axis; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system, ids kept, and their index | ## The expected values diff --git a/conformance/generate.py b/conformance/generate.py index 6864292..85779bc 100644 --- a/conformance/generate.py +++ b/conformance/generate.py @@ -284,7 +284,7 @@ def missing_values() -> Frame: complex=np.array([1, 1, 1, 1, 2, 2, 0], dtype=np.int64), parent=np.array([-1, 10, 10, 11, 11, 14, -1], dtype=np.int64), edge_type=np.array(["none", "successor", "branch", "unknown", "branch", "successor", "none"], dtype=object), - order=np.array([0, 1, 0, 0, 0, 0, 1], dtype=np.int64), + sibling_order=np.array([0, 1, 0, 0, 0, 0, 1], dtype=np.int64), length=np.array([0.12, 0.08, 0.2, 0.15, 0.05, np.nan, 0.3]), rank=np.ma.MaskedArray(np.array([1, 2, 1, 2, 1, 0, 0], dtype=np.int32), mask=[False, False, False, False, False, True, True]), diff --git a/conformance/missing_values.expected.json b/conformance/missing_values.expected.json index d5564c7..183f0c0 100644 --- a/conformance/missing_values.expected.json +++ b/conformance/missing_values.expected.json @@ -160,7 +160,7 @@ "none" ] }, - "order": { + "sibling_order": { "dtype": "int64", "values": [ 0, diff --git a/conformance/missing_values.glb b/conformance/missing_values.glb index 20059a9f2f8ade44d7d8c052a482efafeac87d0d..a5350d741bbc657cde0b957bea5f0655731ead55 100644 GIT binary patch delta 50 xcmdmEcE*f1JtxGCiGhK^N1B1*!baXJEF8s|NjaH$>G7K%vczy8c(%O%c>$9n5a9p- delta 42 ycmX?Ow#SS&JtxGCiGhK^Mw)@)z((FHEUfuODXB%14Oz4{OR(y4Z1&~-&kFzs Date: Mon, 2 Nov 2026 20:24:51 +0100 Subject: [PATCH 18/39] GroIMP's fspm-TF plugin (bindings/java/groimp-plugin, GroIMP 3.0, D-8): File > Export of the 3D view to .glb, or the CLI's export3d, writes the project as an fspm-TF frame: the graph by GroImp.toDataStructure, the scene's geometry as one mesh meshed by GroIMP's glTF plugin, each triangle with its node's and its complexes' rows (an instance's geometry: the node instantiating it); test/run_in_groimp.sh grows simpleRapeseed headless in GroIMP 3.0.0, exports it and checks the frame (456,900 vertices, all with their node; both validators clean); CI job groimp-plugin; GroIMP's modules from its package registry --- .github/workflows/java-codec.yml | 48 ++++- CHANGELOG.md | 3 + bindings/java/groimp-plugin/README.md | 22 +++ bindings/java/groimp-plugin/pom.xml | 138 +++++++++++++ .../fspmtf/groimp/plugin/FspmTfExport.java | 182 ++++++++++++++++++ .../src/main/resources/plugin.properties | 4 + .../src/main/resources/plugin.xml | 57 ++++++ .../java/groimp-plugin/test/run_in_groimp.sh | 34 ++++ bindings/java/groimp/pom.xml | 9 +- doc/java.md | 19 +- implementations.md | 2 +- 11 files changed, 507 insertions(+), 11 deletions(-) create mode 100644 bindings/java/groimp-plugin/README.md create mode 100644 bindings/java/groimp-plugin/pom.xml create mode 100644 bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java create mode 100644 bindings/java/groimp-plugin/src/main/resources/plugin.properties create mode 100644 bindings/java/groimp-plugin/src/main/resources/plugin.xml create mode 100755 bindings/java/groimp-plugin/test/run_in_groimp.sh diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml index fc9e194..02f2100 100644 --- a/.github/workflows/java-codec.yml +++ b/.github/workflows/java-codec.yml @@ -10,7 +10,8 @@ on: pull_request: env: - GROIMP_COMMIT: 6c5bbdef616f799c2eadec1097e1a312e43f308d + GROIMP_DEB_SHA256: b9b35811ff82482dfa676347639e40fe9601c92105f54095361a9a89f7e85dcf + GLTF_COMMIT: d093aaebe5197317dfb81fadbe0ef67268bdaf1c jobs: test: @@ -57,12 +58,7 @@ jobs: distribution: temurin java-version: "21" cache: maven - - name: GroIMP's Graph module - run: | - git clone --quiet https://gitlab.com/grogra/groimp.git groimp - git -C groimp checkout --quiet "$GROIMP_COMMIT" - mvn -B -q -f groimp/pom.xml -pl Graph -am install -DskipTests -Dmaven.javadoc.skip=true - - name: The codec and the adapter + - name: The codec and the adapter (GroIMP's modules from its package registry) run: | mvn -B -q -f bindings/java/pom.xml install -DskipTests mvn -B -f bindings/java/groimp/pom.xml verify @@ -70,3 +66,41 @@ jobs: run: | python -m pip install numpy jsonschema . fspm-tf validate bindings/java/groimp/target/groimp_out/*.glb + + # The plugin in GroIMP itself, headless: GroIMP 3.0.0's release, its glTF plugin (a pinned commit, which meshes the + # shapes), an example grown and its 3D view exported as an fspm-TF frame + groimp-plugin: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + cache: maven + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: GroIMP 3.0.0 + run: | + curl -sSL -o groimp.deb https://download.grogra.de/3_0_0/GroIMP-3.0.0-all.deb + echo "$GROIMP_DEB_SHA256 groimp.deb" | sha256sum -c + dpkg-deb -x groimp.deb groimp-install + - name: GroIMP's glTF plugin + run: | + git clone --quiet https://gitlab.com/grogra/groimp-plugins/gltf.git gltf + git -C gltf checkout --quiet "$GLTF_COMMIT" + mvn -B -q -f gltf/pom.xml install -DskipTests -Dmaven.javadoc.skip=true -Dgroimp.core.version=3.0.0 + mkdir -p gltf-plugin/ext + cp gltf/target/gltf.jar gltf/src/main/resources/plugin.xml gltf/src/main/resources/plugin*.properties gltf-plugin/ + cp gltf/target/lib/ext/*.jar gltf-plugin/ext/ + - name: The codec, the adapter and the plugin + run: | + mvn -B -q -f bindings/java/pom.xml install -DskipTests + mvn -B -q -f bindings/java/groimp/pom.xml install -DskipTests + mvn -B -f bindings/java/groimp-plugin/pom.xml package + - name: Grown and exported in GroIMP, checked + run: | + python -m pip install numpy jsonschema . + bash bindings/java/groimp-plugin/test/run_in_groimp.sh groimp-install/usr/share/GroIMP gltf-plugin \ + bindings/java/groimp-plugin/target/plugin/fspmTF out diff --git a/CHANGELOG.md b/CHANGELOG.md index 745229f..6ae4f9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ - The Java codec (`bindings/java/`): conformant reading and writing, sequences; GroIMP's adapter (`bindings/java/groimp/`: scales from refinement edges, topology, edge bits as relations, managed fields; graphs back into the model's node classes); its CI, `.github/workflows/java-codec.yml`. +- GroIMP's plugin (`bindings/java/groimp-plugin/`): the 3D view exported as an fspm-TF frame from GroIMP 3.0 (File > + Export, CLI `export3d`), its geometry meshed by GroIMP's glTF plugin, each triangle with its node; tested headless in + GroIMP 3.0.0 in CI. - FspmTF.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry (PlantGeomTurtle: `add_geometry!`), as package extensions. - FspmTF.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `fspm_tf_id`), and parents diff --git a/bindings/java/groimp-plugin/README.md b/bindings/java/groimp-plugin/README.md new file mode 100644 index 0000000..bda19e3 --- /dev/null +++ b/bindings/java/groimp-plugin/README.md @@ -0,0 +1,22 @@ +# GroIMP's fspm-TF plugin + +A GroIMP plugin (`org.openalea.fspmtf`, GroIMP 3.0): **File > Export** of the 3D view to a `.glb` file, or the CLI's +`export3d file.glb`, writes the project as an fspm-TF frame: + +- the graph below the project's root: its scales, topology and the nodes' fields (`GroImp.toDataStructure`, + `../groimp`); +- the scene's geometry as one mesh, each triangle carrying the rows of its node and of the node's complexes. + +The shapes are meshed by GroIMP's glTF plugin (`de.grogra.gltf`, from GroIMP's plugin manager or +gitlab.com/grogra/groimp-plugins/gltf), which it requires. + +```bash +mvn -f bindings/java/pom.xml install && mvn -f bindings/java/groimp/pom.xml install +mvn -f bindings/java/groimp-plugin/pom.xml package # the plugin's folder: target/plugin/fspmTF +``` + +To install it, copy `target/plugin/fspmTF` into GroIMP's `plugins/` folder, next to the glTF plugin's. +`test/run_in_groimp.sh` does this in a GroIMP install, grows an example headless, exports it and checks the frame. It +is run by the `groimp-plugin` job of `.github/workflows/java-codec.yml`. + +Licence: GPL-3.0-or-later, as GroIMP. diff --git a/bindings/java/groimp-plugin/pom.xml b/bindings/java/groimp-plugin/pom.xml new file mode 100644 index 0000000..47ec604 --- /dev/null +++ b/bindings/java/groimp-plugin/pom.xml @@ -0,0 +1,138 @@ + + + + 4.0.0 + + org.openalea + fspm-tf-groimp-plugin + 0.1.0 + jar + fspm-TF, GroIMP's plugin + + + GPL-3.0-or-later + https://www.gnu.org/licenses/gpl-3.0.txt + + + + + 21 + UTF-8 + 3.0.0 + + + + + gitlab-maven + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + fspm-tf-groimp + 0.1.0 + + + de.grogra + gltf + 0.5 + provided + + + de.javagl + jgltf-model + 2.0.4 + provided + + + de.grogra + imp3d + ${groimp.version} + provided + + + de.grogra + platform.core + ${groimp.version} + provided + + + de.grogra + platform + ${groimp.version} + provided + + + de.grogra + graph + ${groimp.version} + provided + + + de.grogra + vecmath + ${groimp.version} + provided + + + + + fspmTF + + + org.apache.maven.plugins + maven-compiler-plugin + 3.13.0 + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + libraries + package + copy-dependencies + + ${project.build.directory}/plugin/fspmTF/ext + runtime + com.fasterxml.jackson.core + + + + + + org.apache.maven.plugins + maven-antrun-plugin + 3.1.0 + + + folder + package + run + + + + + + + + + + + + + + diff --git a/bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java b/bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java new file mode 100644 index 0000000..93411fa --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java @@ -0,0 +1,182 @@ +package org.openalea.fspmtf.groimp.plugin; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openalea.fspmtf.Codec; +import org.openalea.fspmtf.DataStructure; +import org.openalea.fspmtf.EntitySet; +import org.openalea.fspmtf.Frame; +import org.openalea.fspmtf.Mesh; +import org.openalea.fspmtf.groimp.GroImp; + +import de.grogra.gltf.exporter.GLTFExport; +import de.grogra.gltf.exporter.GLTFNodeWrapper; +import de.grogra.graph.Graph; +import de.grogra.graph.impl.Node; +import de.grogra.pf.io.FilterItem; +import de.grogra.pf.io.FilterSource; +import de.javagl.jgltf.model.AccessorData; +import de.javagl.jgltf.model.AccessorFloatData; +import de.javagl.jgltf.model.AccessorIntData; +import de.javagl.jgltf.model.AccessorShortData; +import de.javagl.jgltf.model.AccessorByteData; +import de.javagl.jgltf.model.MeshModel; +import de.javagl.jgltf.model.MeshPrimitiveModel; +import de.javagl.jgltf.model.NodeModel; +import de.javagl.jgltf.model.impl.DefaultNodeModel; + +/** + * The 3D view of a GroIMP project as an fspm-TF frame (a .glb file): the graph below the project's root (its scales, + * topology and fields, by {@link GroImp#toDataStructure}) and the scene's geometry as one mesh, each triangle carrying + * the rows of its node and of the node's complexes. The shapes are meshed by GroIMP's glTF plugin (its per-shape + * exporters, as for a .gltf file); this class collects their triangles in the scene's frame (GroIMP's: z up), node by + * node, instead of writing glTF. + */ +public class FspmTfExport extends GLTFExport { + /** The id of no node, for meshes whose node is not an entity (e.g. the scene's ground). */ + private static final long NO_NODE = -1; + + public FspmTfExport(FilterItem item, FilterSource source) { + super(item, source); + } + + @Override + public void write(File file) throws IOException { + write(); // the glTF plugin's scene walk: one wrapper per node + DefaultNodeModel root = assembleTree(); + Node graphRoot = (Node) getGraphState().getGraph().getRoot(Graph.MAIN_GRAPH); + GroImp.Options options = new GroImp.Options(); + options.id = "plants"; + DataStructure plants = GroImp.toDataStructure(graphRoot, options); + + java.util.Set entities = new java.util.HashSet<>(); + for (EntitySet set : plants.entitySets) for (long id : set.column("id").longs()) entities.add(id); + List positions = new ArrayList<>(); + List nodes = new ArrayList<>(); + collect(root, identity(), NO_NODE, entities, positions, nodes); + if (!positions.isEmpty()) plants.meshes.add(mesh(plants, positions, nodes)); + + Frame frame = new Frame(); + frame.dataStructures.add(plants); + frame.writer.put("platform", "GroIMP").put("adapter", "org.openalea.fspmtf.groimp.plugin"); + frame.vocabularyVersion = "0.2"; + Codec.write(file.toPath(), frame); + } + + /** The glTF plugin's node tree, its wrappers linked as GLTFExport links them (a child under its parent's node). */ + private DefaultNodeModel assembleTree() { + DefaultNodeModel root = new DefaultNodeModel(); + for (String key : nodeRegistry.getKeys()) { + GLTFNodeWrapper wrapper = (GLTFNodeWrapper) nodeRegistry.get(key); + GLTFNodeWrapper parent = wrapper.getParentId() == -1 ? null + : (GLTFNodeWrapper) nodeRegistry.get("" + wrapper.getParentId()); + if (parent != null && parent.getOutNode() != null) parent.addChild(wrapper); + else root.addChild(wrapper.getInNode()); + } + return root; + } + + private static float[] identity() { + return new float[] {1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + } + + /** Column-major 4×4 product a·b. */ + private static float[] multiply(float[] a, float[] b) { + float[] product = new float[16]; + for (int column = 0; column < 4; column++) + for (int row = 0; row < 4; row++) { + float sum = 0; + for (int k = 0; k < 4; k++) sum += a[k * 4 + row] * b[column * 4 + k]; + product[column * 4 + row] = sum; + } + return product; + } + + /** The GroIMP node id a glTF node stands for ("12" or "12_b"), else NO_NODE. */ + private static long nodeId(NodeModel node) { + String name = node.getName(); + if (name == null || name.isEmpty()) return NO_NODE; + try { + return Long.parseLong(name.endsWith("_b") ? name.substring(0, name.length() - 2) : name); + } catch (NumberFormatException error) { + return NO_NODE; + } + } + + /** Each triangle of the tree below {@code node}, its vertices in the scene's frame, with the id of its node. A glTF + * node that is no entity (unnamed, or an instance's, whose id the glTF plugin makes up) takes its nearest ancestor's + * entity: an instance's geometry, the node instantiating it. */ + private void collect(NodeModel node, float[] parentTransform, long inheritedId, java.util.Set entities, + List positions, List nodes) { + float[] transform = multiply(parentTransform, node.computeLocalTransform(null)); + long own = nodeId(node); + long id = entities.contains(own) ? own : inheritedId; + for (MeshModel mesh : node.getMeshModels()) + for (MeshPrimitiveModel primitive : mesh.getMeshPrimitiveModels()) { + if (primitive.getMode() != 4 || !primitive.getAttributes().containsKey("POSITION")) continue; + AccessorFloatData points = (AccessorFloatData) primitive.getAttributes().get("POSITION").getAccessorData(); + int[] indices = primitive.getIndices() == null ? null : indices(primitive.getIndices().getAccessorData()); + int count = indices == null ? points.getNumElements() : indices.length; + for (int corner = 0; corner < count; corner++) { + int vertex = indices == null ? corner : indices[corner]; + float x = points.get(vertex, 0), y = points.get(vertex, 1), z = points.get(vertex, 2); + for (int axis = 0; axis < 3; axis++) + positions.add((double) (transform[axis] * x + transform[4 + axis] * y + transform[8 + axis] * z + + transform[12 + axis])); + nodes.add(id); + } + } + for (NodeModel child : node.getChildren()) collect(child, transform, id, entities, positions, nodes); + } + + private static int[] indices(AccessorData data) { + int[] values = new int[data.getNumElements()]; + for (int index = 0; index < values.length; index++) { + if (data instanceof AccessorShortData shorts) values[index] = shorts.getInt(index); + else if (data instanceof AccessorIntData ints) values[index] = ints.get(index); + else if (data instanceof AccessorByteData bytes) values[index] = bytes.getInt(index); + else throw new IllegalArgumentException("indices of type " + data.getComponentType()); + } + return values; + } + + /** The triangles as an fspm-TF mesh, three vertices each, with each vertex's row in every entity set: its node's, + * and its node's complexes' up the scales (NO_ID where it has none). */ + private static Mesh mesh(DataStructure plants, List positions, List nodes) { + Map where = new HashMap<>(); // id -> (entity set, row) + Map complexOf = new HashMap<>(); + for (int setIndex = 0; setIndex < plants.entitySets.size(); setIndex++) { + EntitySet set = plants.entitySets.get(setIndex); + long[] ids = set.column("id").longs(); + long[] complexes = set.has("complex") ? set.column("complex").longs() : null; + for (int row = 0; row < ids.length; row++) { + where.put(ids[row], new int[] {setIndex, row}); + if (complexes != null && complexes[row] != EntitySet.NO_ID) complexOf.put(ids[row], complexes[row]); + } + } + double[] points = new double[positions.size()]; + for (int index = 0; index < points.length; index++) points[index] = positions.get(index); + Mesh mesh = new Mesh("scene", points); + long[][] rows = new long[plants.entitySets.size()][nodes.size()]; + for (long[] setRows : rows) java.util.Arrays.fill(setRows, EntitySet.NO_ID); + for (int vertex = 0; vertex < nodes.size(); vertex++) { + Long id = nodes.get(vertex); + while (id != null && id != NO_NODE && where.containsKey(id)) { // the node, then its complexes + int[] place = where.get(id); + rows[place[0]][vertex] = place[1]; + id = complexOf.get(id); + } + } + for (int setIndex = 0; setIndex < rows.length; setIndex++) { + boolean any = false; + for (long row : rows[setIndex]) any |= row != EntitySet.NO_ID; + if (any) mesh.featureRows.put(plants.entitySets.get(setIndex).name, rows[setIndex]); + } + return mesh; + } +} diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.properties b/bindings/java/groimp-plugin/src/main/resources/plugin.properties new file mode 100644 index 0000000..8a0c5f7 --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.properties @@ -0,0 +1,4 @@ +pluginName = fspm-TF +provider = OpenAlea (fspm-TF) +/io/filetypes/fspmTF.Name = fspm-TF frame (*.glb) +/io/mimetypes/application/x-java-jvm-local-objectref/fspmTFExport.Name = fspm-TF frame diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.xml b/bindings/java/groimp-plugin/src/main/resources/plugin.xml new file mode 100644 index 0000000..dc01b69 --- /dev/null +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bindings/java/groimp-plugin/test/run_in_groimp.sh b/bindings/java/groimp-plugin/test/run_in_groimp.sh new file mode 100755 index 0000000..027cf26 --- /dev/null +++ b/bindings/java/groimp-plugin/test/run_in_groimp.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# GroIMP's fspm-TF plugin in GroIMP itself, headless: install it (and GroIMP's glTF plugin, which meshes the shapes) +# into a GroIMP 3.0 install, grow one of GroIMP's examples, export its 3D view as an fspm-TF frame, and check the file: +# valid fspm-TF (fspm-tf validate), every triangle with its node. +# +# run_in_groimp.sh GROIMP GLTF_PLUGIN FSPMTF_PLUGIN OUT +# GROIMP a GroIMP 3.0 install (the folder holding core.jar and plugins/) +# GLTF_PLUGIN GroIMP's glTF plugin's folder (plugin.xml, gltf.jar, ext/) +# FSPMTF_PLUGIN this plugin's folder (target/plugin/fspmTF) +# OUT where the frame is written +set -euo pipefail +groimp=$(realpath "$1"); gltf=$(realpath "$2"); fspmtf=$(realpath "$3"); out=$(realpath -m "$4") +mkdir -p "$out" "$groimp/plugins/gltf" "$groimp/plugins/fspmTF" "$out/home" +cp -r "$gltf/." "$groimp/plugins/gltf/" +cp -r "$fspmtf/." "$groimp/plugins/fspmTF/" +frame="$out/rapeseed.glb" +rm -f "$frame" +cd "$groimp" +# the export runs in the background: the commands' input stays open while it writes +(printf '$loadExample simpleRapeseed\nrun\nrun\nrun\nexport3d %s\n' "$frame"; sleep 30; printf 'listFunctions\n') | + java -Xmx1g -Djava.util.prefs.userRoot="$out/home" -jar core.jar -a cli > "$out/groimp.log" 2>&1 +test -s "$frame" || { echo "no frame written"; tail -40 "$out/groimp.log"; exit 1; } +fspm-tf validate "$frame" +python - "$frame" <<'PYTHON' +import sys +import numpy as np +from openalea import fspm_tf +plants = fspm_tf.read(sys.argv[1]).data_structure("plants") +mesh = plants.meshes[0] +rows = mesh.feature_rows[plants.entity_sets[-1].name] +assert len(mesh.positions) > 0 and (rows >= 0).all(), "every vertex carries its node" +assert mesh.positions[:, 2].max() > 0, "the plant stands, z up" +print(f"{len(mesh.positions)} vertices, {len(np.unique(rows))} nodes drawn, of {plants.entity_sets[-1].count}") +PYTHON diff --git a/bindings/java/groimp/pom.xml b/bindings/java/groimp/pom.xml index a8ee495..a249847 100644 --- a/bindings/java/groimp/pom.xml +++ b/bindings/java/groimp/pom.xml @@ -4,7 +4,7 @@ GroIMP plugin (devplan_bindings.md, G4). mvn -f bindings/java/pom.xml install && mvn -f bindings/java/groimp/pom.xml verify - (GroIMP's graph, utilities and xl.core 3.0.0 in the local repository: mvn -pl Graph -am install in GroIMP) + (GroIMP's graph, utilities and xl.core 3.0.0: from GroIMP's package registry) --> @@ -29,6 +29,13 @@ 5.11.4 + + + gitlab-maven + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + org.openalea diff --git a/doc/java.md b/doc/java.md index 3bfde44..930c65e 100644 --- a/doc/java.md +++ b/doc/java.md @@ -53,5 +53,20 @@ that other classes of the same scale lack is missing in their rows. successor and branch edges, and edge bits. It does this for any graph, e.g. metafspm's Plant, Axis and SubOrgan become three scales linked by refinement edges. -The adapter depends on GroIMP's Graph module only. It is meant to become a GroIMP plugin: an export filter with the -scene's geometry, and an import filter. +The adapter depends on GroIMP's Graph module only. + +## GroIMP's plugin + +`bindings/java/groimp-plugin/` is a GroIMP 3.0 plugin. In GroIMP, **File > Export** of the 3D view to a `.glb` file +writes the project as an fspm-TF frame, and the CLI's `export3d plant.glb` does the same headless: + +- the graph, by the adapter above; +- the scene's geometry as one mesh, each triangle carrying the rows of its node and of its complexes (an instance's + geometry: the node instantiating it). + +GroIMP's glTF plugin, which it requires, meshes the shapes: cylinders, boxes, spheres, cones, frusta, planes, +parallelograms, NURBS, meshes and polygons. Its folder (`target/plugin/fspmTF`) is copied into GroIMP's `plugins/`. +`test/run_in_groimp.sh` installs both plugins in GroIMP 3.0.0, grows GroIMP's `simpleRapeseed` example headless and +exports it (456,900 vertices, each with its node), then checks the frame; CI runs it. + +Not done yet: an import filter (frames into a GroIMP project), and units for GroIMP's fields, which have none. diff --git a/implementations.md b/implementations.md index c9560b9..9f09982 100644 --- a/implementations.md +++ b/implementations.md @@ -7,7 +7,7 @@ | fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.2 | | CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.2 | | fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.2 | -| GroIMP | Java | `bindings/java/groimp/` (`GroImp`), to become a GroIMP plugin | yes: graphs and fields into the model's node classes | yes: scales, topology, edge bits, managed fields | 0.2 | +| GroIMP | Java | `bindings/java/groimp/` (`GroImp`) and `bindings/java/groimp-plugin/` (a GroIMP 3.0 plugin: File > Export to `.glb`, CLI `export3d`) | yes: graphs and fields into the model's node classes (adapter) | yes: scales, topology, edge bits, managed fields, the scene's geometry by node | 0.2 | | FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.2 | | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.2 | | VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.2 | From 879f0a641b9dd535de78e8f3f70d126e6624052d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 12:58:21 +0100 Subject: [PATCH 19/39] =?UTF-8?q?Coupling=20platforms=20(specification=20?= =?UTF-8?q?=C2=A711,=20draft=20for=200.3,=20D-13):=20partial=20frames=20(F?= =?UTF-8?q?SPM=5Fscene.update:=20the=20sender,=20removed=20entities)=20in?= =?UTF-8?q?=20the=20schema,=20the=20Python=20and=20Julia=20codecs=20and=20?= =?UTF-8?q?the=20validator;=20the=20session's=20messages=20(spec/schema/co?= =?UTF-8?q?upling.schema.json:=20declarations=20with=20entity=20sets'=20ri?= =?UTF-8?q?ghts=20owns/refines/follows=20and=20outputs/inputs,=20links=20w?= =?UTF-8?q?ith=20units,=20factors=20and=20changes=20of=20scale,=20turns,?= =?UTF-8?q?=20updates,=20stop);=20openalea.fspm=5Ftf.coupling:=20resolutio?= =?UTF-8?q?n,=20stages,=20a=20coordinator=20holding=20the=20shared=20struc?= =?UTF-8?q?ture=20(id=20ranges,=20rights,=20removals=20down=20the=20scales?= =?UTF-8?q?),=20launching=20participants=20and=20recording=20the=20session?= =?UTF-8?q?,=20ZeroMQ=20and=20folder=20transports,=20replay=20of=20a=20rec?= =?UTF-8?q?orded=20sequence=20(hold,=20linear),=20the=20participant's=20si?= =?UTF-8?q?de;=20fspm-tf=20couple;=20ucum.conversion=5Ffactor;=20FspmTF.jl?= =?UTF-8?q?'s=20participant=20(folder;=20ZeroMQ=20by=20FspmTFZMQExt),=20te?= =?UTF-8?q?sted=20against=20the=20coordinator=20over=20both=20transports,?= =?UTF-8?q?=20in=20CI=20too;=20enum=20columns=20with=20missing=20values=20?= =?UTF-8?q?refused=20by=20the=20Python=20codec;=20the=20Coupling=20platfor?= =?UTF-8?q?ms=20page?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/conformance.yml | 2 +- .github/workflows/julia-codec.yml | 24 ++ CHANGELOG.md | 13 + bindings/julia/FspmTF/Project.toml | 3 + bindings/julia/FspmTF/ext/FspmTFZMQExt.jl | 36 ++ bindings/julia/FspmTF/src/FspmTF.jl | 6 + bindings/julia/FspmTF/src/codec.jl | 19 + bindings/julia/FspmTF/src/coupling.jl | 299 ++++++++++++++ bindings/julia/FspmTF/src/model.jl | 24 +- .../julia/FspmTF/test/coupling/function.jl | 45 ++ bindings/julia/FspmTF/test/runtests.jl | 30 ++ .../python/src/openalea/fspm_tf/__init__.py | 4 +- bindings/python/src/openalea/fspm_tf/cli.py | 61 ++- bindings/python/src/openalea/fspm_tf/codec.py | 22 +- .../src/openalea/fspm_tf/coupling/__init__.py | 18 + .../src/openalea/fspm_tf/coupling/channels.py | 191 +++++++++ .../openalea/fspm_tf/coupling/coordinator.py | 199 +++++++++ .../src/openalea/fspm_tf/coupling/messages.py | 185 +++++++++ .../openalea/fspm_tf/coupling/participant.py | 91 ++++ .../src/openalea/fspm_tf/coupling/replay.py | 108 +++++ .../openalea/fspm_tf/coupling/resolution.py | 169 ++++++++ .../src/openalea/fspm_tf/coupling/store.py | 388 ++++++++++++++++++ bindings/python/src/openalea/fspm_tf/model.py | 18 +- .../python/src/openalea/fspm_tf/schema.py | 4 +- bindings/python/src/openalea/fspm_tf/ucum.py | 67 ++- .../python/src/openalea/fspm_tf/validate.py | 26 +- bindings/python/test/test_coupling.py | 268 ++++++++++++ doc/api.md | 23 +- doc/coupling.md | 102 +++++ doc/index.md | 1 + doc/julia.md | 7 + doc/platforms.md | 18 + implementations.md | 4 + pyproject.toml | 6 +- spec/schema/coupling.schema.json | 102 +++++ spec/schema/fspm_scene.schema.json | 13 + spec/specification.md | 129 ++++++ 37 files changed, 2690 insertions(+), 35 deletions(-) create mode 100644 bindings/julia/FspmTF/ext/FspmTFZMQExt.jl create mode 100644 bindings/julia/FspmTF/src/coupling.jl create mode 100644 bindings/julia/FspmTF/test/coupling/function.jl create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/__init__.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/channels.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/coordinator.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/messages.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/participant.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/replay.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/resolution.py create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/store.py create mode 100644 bindings/python/test/test_coupling.py create mode 100644 doc/coupling.md create mode 100644 spec/schema/coupling.schema.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index fc4938a..04e6bd0 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -18,7 +18,7 @@ jobs: with: python-version: "3.12" - name: Install openalea.fspm_tf (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests - run: python -m pip install numpy jsonschema pytest vtk . + run: python -m pip install numpy jsonschema pytest pyzmq vtk . - name: Tests, the ParaView reader's conversion included run: pytest bindings/python/test -v - name: The committed files are the generated ones diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml index c8a3f1a..f592ef8 100644 --- a/.github/workflows/julia-codec.yml +++ b/.github/workflows/julia-codec.yml @@ -67,3 +67,27 @@ jobs: with: dumps: out/dumps written: out/written + + # FspmTF.jl as a participant of coupling sessions run by the Python coordinator (specification §11), over ZeroMQ + # and over a folder + coupling: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: "1" + - uses: julia-actions/cache@v2 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: The Python coordinator + run: pip install -e . pytest pyzmq jsonschema + - name: A Julia project with FspmTF and ZMQ + run: | + mkdir -p coupling-env + julia --project=coupling-env -e 'using Pkg; Pkg.develop(path="bindings/julia/FspmTF"); Pkg.add("ZMQ")' + - name: The coupling tests, with the Julia participant + env: + FSPM_TF_JULIA: julia --project=coupling-env + run: pytest -v bindings/python/test/test_coupling.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae4f9a..735f1fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## Unreleased: coupling platforms (specification §11, draft for 0.3) + +- Coupling sessions: platforms' models run together, each in its own process, through fspm-TF. Partial frames + (`FSPM_scene.update`: the sender, removed entities); the messages (`spec/schema/coupling.schema.json`): a + participant's declaration (its entity sets and rights, `owns`, `refines`, `follows`; its outputs and inputs), links + (factors, units, a change of scale), turns and updates; stages; one shared structure with id ranges and removals + down the scales; two transports, ZeroMQ and a folder of messages; a recorded sequence as a participant (replay, + held or interpolated linearly). +- The reference coordinator and participant (`openalea.fspm_tf.coupling`, `fspm-tf couple`), unit conversion factors + (`ucum.conversion_factor`); FspmTF.jl's participant (`Participant`, `turns`, `send_update`; ZeroMQ through the + `FspmTFZMQExt` extension), tested against the coordinator over both transports. +- The Python codec refuses enum columns (labels) with missing values instead of writing an unnamed enum value. + ## Unreleased: specification 0.2 (draft) - Specification 0.2, additive (0.1 files stay valid), from the bindings' points (`spec/specification.md`, "Changes diff --git a/bindings/julia/FspmTF/Project.toml b/bindings/julia/FspmTF/Project.toml index 5fde8d9..c0acaac 100644 --- a/bindings/julia/FspmTF/Project.toml +++ b/bindings/julia/FspmTF/Project.toml @@ -13,11 +13,13 @@ MultiScaleTreeGraph = "dd4a991b-8a45-4075-bede-262ee62d5583" PlantGeomPrimitives = "7eef3cc5-4580-4ff0-8f6f-933507db6664" PlantGeomTurtle = "7d6e2781-1c99-4c66-97ec-106669f3e96e" PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" [extensions] FspmTFMultiScaleTreeGraphExt = "MultiScaleTreeGraph" FspmTFPlantGeomTurtleExt = ["PlantGraphs", "PlantGeomTurtle", "PlantGeomPrimitives"] FspmTFPlantGraphsExt = "PlantGraphs" +FspmTFZMQExt = "ZMQ" [compat] Dates = "1" @@ -26,6 +28,7 @@ MultiScaleTreeGraph = "0.16" PlantGeomPrimitives = "1" PlantGeomTurtle = "1" PlantGraphs = "1" +ZMQ = "1" OrderedCollections = "1, 2" julia = "1.10" diff --git a/bindings/julia/FspmTF/ext/FspmTFZMQExt.jl b/bindings/julia/FspmTF/ext/FspmTFZMQExt.jl new file mode 100644 index 0000000..12fc6cf --- /dev/null +++ b/bindings/julia/FspmTF/ext/FspmTFZMQExt.jl @@ -0,0 +1,36 @@ +# The coupling's ZeroMQ transport: a participant's DEALER socket, named after it, connected to the coordinator's +# ROUTER socket; a message is its JSON header and, for turns and updates, a partial frame's GLB bytes. +module FspmTFZMQExt + +using FspmTF +using JSON +using ZMQ + +mutable struct ZmqChannel <: FspmTF.CouplingChannel + socket::ZMQ.Socket +end + +function FspmTF.zmq_channel(endpoint::String, participant::String) + socket = ZMQ.Socket(ZMQ.DEALER) + socket.routing_id = participant + socket.linger = 1000 + ZMQ.connect(socket, endpoint) + return ZmqChannel(socket) +end + +function FspmTF.send_message(channel::ZmqChannel, header::AbstractDict, payload = nothing) + parts = payload === nothing ? [JSON.json(header)] : [Vector{UInt8}(codeunits(JSON.json(header))), payload] + ZMQ.send_multipart(channel.socket, parts) + return nothing +end + +function FspmTF.receive_message(channel::ZmqChannel, timeout = nothing) + channel.socket.rcvtimeo = timeout === nothing ? -1 : round(Int, 1000 * timeout) + parts = ZMQ.recv_multipart(channel.socket, Vector{UInt8}) + header = JSON.parse(String(parts[1])) + return header, length(parts) > 1 ? parts[2] : nothing +end + +FspmTF.close_channel(channel::ZmqChannel) = close(channel.socket) + +end diff --git a/bindings/julia/FspmTF/src/FspmTF.jl b/bindings/julia/FspmTF/src/FspmTF.jl index 688cda1..941317f 100644 --- a/bindings/julia/FspmTF/src/FspmTF.jl +++ b/bindings/julia/FspmTF/src/FspmTF.jl @@ -13,6 +13,9 @@ binary (GLB) file per time step. The specification is `spec/specification.md` in With MultiScaleTreeGraph loaded, `from_mtg` and `to_mtg` convert an MTG to and from a data structure. With VirtualPlantLab's PlantGraphs loaded, `from_graph` and `to_graph` convert its graphs; with PlantGeomTurtle too, `add_geometry!` adds a graph's turtle geometry as a mesh. + +A platform takes part in a coupling session (specification §11) with `Participant(declaration)`, then answers each of +`turns(participant)` with `send_update(participant, frame)`; over a folder ("files:path"), or ZeroMQ with ZMQ loaded. """ module FspmTF @@ -29,11 +32,14 @@ include("codec.jl") include("sequence.jl") include("dump.jl") include("values.jl") +include("coupling.jl") export Frame, DataStructure, EntitySet, Relation, Table, Mapping, Grid, Mesh, Variable, NO_ID, EDGE_TYPES export read_frame, write_frame, from_bytes, to_bytes, read_index, read_sequence, write_sequence, write_index export frame_to_dict, write_dump, row_count, variable_names, entity_ids, entity_set, data_structure export from_mtg, to_mtg, from_graph, to_graph, add_geometry! +export Update, Declaration, EntitySetSpec, VariableSpec, Link, Participant, Turn, next_turn, turns, + send_update, fail """ from_mtg(mtg; id="plants", scales=Dict(), variables=Dict(), platform="MultiScaleTreeGraph", model=nothing) diff --git a/bindings/julia/FspmTF/src/codec.jl b/bindings/julia/FspmTF/src/codec.jl index 19d8138..c82dacc 100644 --- a/bindings/julia/FspmTF/src/codec.jl +++ b/bindings/julia/FspmTF/src/codec.jl @@ -115,11 +115,24 @@ function to_bytes(frame::Frame) push!(mappings, entry) end + update = nothing + if frame.update !== nothing + check_identifier(frame.update.participant, "participant") + update = json_object("participant" => frame.update.participant) + if !isempty(frame.update.removed) + update["removed"] = Any[json_object("dataStructure" => key[1], "entitySet" => key[2], + "propertyTable" => add_table(Table("$(key[2])__removed"; + columns = Columns("id" => ids)), "$(key[1])__$(key[2])__removed")) + for (key, ids) in frame.update.removed] + end + end + scene = json_object("version" => SPECIFICATION_VERSION, "frame" => json_object("time" => Float64(frame.time), "timeUnit" => frame.time_unit, "step" => frame.step)) frame.vocabulary_version === nothing || (scene["vocabulary"] = frame.vocabulary_version) isempty(frame.writer) || (scene["writer"] = frame.writer) + update === nothing || (scene["update"] = update) scene["dataStructures"] = data_structures isempty(mappings) || (scene["mappings"] = mappings) isempty(variables) || (scene["variables"] = variables) @@ -421,6 +434,12 @@ function from_bytes(data::AbstractVector{UInt8}) push!(frame.mappings, table(entry["propertyTable"], (name, columns, declarations) -> Mapping(name, columns, declarations, source, target, get(entry, "method", nothing)))) end + if haskey(scene, "update") + entry = scene["update"] + frame.update = Update(entry["participant"]; removed = OrderedDict( + (item["dataStructure"], item["entitySet"]) => Vector{Int64}(table(item["propertyTable"], Table).columns["id"]) + for item in get(entry, "removed", Any[]))) + end other_extensions = json_object(name => value for (name, value) in extensions if !(name in (METADATA, EXTENSION, FEATURES))) isempty(other_extensions) || (frame.unknown["extensions"] = other_extensions) diff --git a/bindings/julia/FspmTF/src/coupling.jl b/bindings/julia/FspmTF/src/coupling.jl new file mode 100644 index 0000000..2801150 --- /dev/null +++ b/bindings/julia/FspmTF/src/coupling.jl @@ -0,0 +1,299 @@ +# A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and +# requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is fspm-TF's +# (Python: `fspm-tf couple`). Transports: a folder ("files:path", here) and ZeroMQ ("tcp://…", "ipc://…": with ZMQ +# loaded, the package extension FspmTFZMQExt). + +"""The version of the coupling protocol this package speaks.""" +const PROTOCOL_VERSION = "0.3" +const ENDPOINT_VARIABLE = "FSPM_TF_ENDPOINT" +const PARTICIPANT_VARIABLE = "FSPM_TF_PARTICIPANT" + +""" + EntitySetSpec(data_structure, name; rank=nothing, rights="follows") + +An entity set a participant works on: its rank (a base scale's position by default) and its rights on its topology, +`"owns"` (adds and removes its entities), `"refines"` (adds and removes the entities it creates, under others'), +`"follows"` (reads). +""" +struct EntitySetSpec + data_structure::String + name::String + rank::Union{Nothing,Int} + rights::String +end + +function EntitySetSpec(data_structure::AbstractString, name::AbstractString; rank = nothing, rights = "follows") + rights in ("owns", "refines", "follows") || throw(ArgumentError("rights '$rights': owns, refines or follows")) + if rank === nothing && name in BASE_SCALES + rank = findfirst(==(name), BASE_SCALES) + end + return EntitySetSpec(String(data_structure), String(name), rank, String(rights)) +end + +""" + VariableSpec(data_structure, entity_set, name, direction; unit="1", standard_name=nothing, kind=nothing, + optional=false) + +A variable a participant provides (`direction` `"provides"`: an output) or requires (`"requires"`: an input). +""" +struct VariableSpec + data_structure::String + entity_set::String + name::String + direction::String + unit::String + standard_name::Union{Nothing,String} + kind::Union{Nothing,String} + optional::Bool +end + +function VariableSpec(data_structure::AbstractString, entity_set::AbstractString, name::AbstractString, + direction::AbstractString; unit = "1", standard_name = nothing, kind = nothing, optional = false) + direction in ("provides", "requires") || throw(ArgumentError("direction '$direction': provides or requires")) + return VariableSpec(String(data_structure), String(entity_set), String(name), String(direction), String(unit), + standard_name, kind, optional) +end + +""" + Declaration(participant; platform=nothing, entity_sets=EntitySetSpec[], variables=VariableSpec[]) + +A participant's declaration (its name: `FSPM_TF_PARTICIPANT` when empty and launched by the coordinator). +""" +Base.@kwdef mutable struct Declaration + participant::String = "" + platform::Union{Nothing,String} = nothing + entity_sets::Vector{EntitySetSpec} = EntitySetSpec[] + variables::Vector{VariableSpec} = VariableSpec[] +end + +Declaration(participant::AbstractString; kwargs...) = Declaration(; participant = String(participant), kwargs...) + +function declaration_json(declaration::Declaration) + entity_sets = [json_object("dataStructure" => item.data_structure, "name" => item.name, "rank" => item.rank, + "rights" => item.rights) for item in declaration.entity_sets] + for variable in declaration.variables # an entity set for each one its variables are on, followed + if !any(item -> (item.data_structure, item.name) == (variable.data_structure, variable.entity_set), + declaration.entity_sets) + spec = EntitySetSpec(variable.data_structure, variable.entity_set) + push!(declaration.entity_sets, spec) + push!(entity_sets, json_object("dataStructure" => spec.data_structure, "name" => spec.name, + "rank" => spec.rank, "rights" => spec.rights)) + end + end + variables = Any[] + for variable in declaration.variables + entry = json_object("dataStructure" => variable.data_structure, "entitySet" => variable.entity_set, + "name" => variable.name, "direction" => variable.direction, "unit" => variable.unit) + variable.standard_name === nothing || (entry["standardName"] = variable.standard_name) + variable.kind === nothing || (entry["kind"] = variable.kind) + variable.optional && (entry["optional"] = true) + push!(variables, entry) + end + value = json_object("participant" => declaration.participant, "entitySets" => entity_sets, + "variables" => variables) + declaration.platform === nothing || (value["platform"] = declaration.platform) + return value +end + +""" + Link(receiver, variable, provider, source; entity_set=nothing, source_entity_set=nothing, factor=1.0, + aggregation=nothing, interpolation="hold") + +An input's provider, as a participant may propose it to the coordinator (see the Python `Link`). +""" +Base.@kwdef struct Link + receiver::String + variable::String + provider::String + source::String + entity_set::Union{Nothing,String} = nothing + source_entity_set::Union{Nothing,String} = nothing + factor::Float64 = 1.0 + aggregation::Union{Nothing,String} = nothing + interpolation::String = "hold" +end + +Link(receiver, variable, provider, source; kwargs...) = + Link(; receiver = String(receiver), variable = String(variable), provider = String(provider), + source = String(source), kwargs...) + +function link_json(link::Link) + value = json_object("receiver" => link.receiver, "variable" => link.variable, "provider" => link.provider, + "source" => link.source) + link.entity_set === nothing || (value["entitySet"] = link.entity_set) + link.source_entity_set === nothing || (value["sourceEntitySet"] = link.source_entity_set) + link.aggregation === nothing || (value["aggregation"] = link.aggregation) + link.factor == 1.0 || (value["factor"] = link.factor) + link.interpolation == "hold" || (value["interpolation"] = link.interpolation) + return value +end + +# ── transports ─────────────────────────────────────────────────────────────────── +"""A participant's channel: `send_message(channel, header, payload)`, `receive_message(channel, timeout)`.""" +abstract type CouplingChannel end + +"""A participant's folder queues: `//to_coordinator/` and `to_participant/`.""" +mutable struct FolderChannel <: CouplingChannel + incoming::String + outgoing::String + next_read::Int + next_write::Int +end + +function FolderChannel(folder::AbstractString, participant::AbstractString) + incoming = joinpath(folder, participant, "to_participant") + outgoing = joinpath(folder, participant, "to_coordinator") + mkpath(incoming) + mkpath(outgoing) + return FolderChannel(incoming, outgoing, 0, 0) +end + +stem(folder, number) = joinpath(folder, lpad(number, 8, '0')) + +function send_message(channel::FolderChannel, header::AbstractDict, payload = nothing) + path = stem(channel.outgoing, channel.next_write) + header = copy(header) + if payload !== nothing + write(path * ".glb", payload) + header["payload"] = basename(path) * ".glb" + end + write(path * ".json.part", JSON.json(header)) + mv(path * ".json.part", path * ".json"; force = true) + channel.next_write += 1 + return nothing +end + +function receive_message(channel::FolderChannel, timeout = nothing) + path = stem(channel.incoming, channel.next_read) + deadline = timeout === nothing ? nothing : time() + timeout + while !isfile(path * ".json") + deadline !== nothing && time() > deadline && throw(ErrorException("no message in $timeout s")) + sleep(0.005) + end + header = JSON.parse(read(path * ".json", String)) + payload = haskey(header, "payload") ? read(joinpath(channel.incoming, pop!(header, "payload"))) : nothing + channel.next_read += 1 + return header, payload +end + +close_channel(::CouplingChannel) = nothing + +""" + zmq_channel(endpoint, participant) -> CouplingChannel + +A participant's ZeroMQ channel (a DEALER socket named after it). Defined when ZMQ is loaded (`FspmTFZMQExt`). +""" +function zmq_channel end + +function participant_channel(endpoint::AbstractString, participant::AbstractString) + startswith(endpoint, "files:") && return FolderChannel(endpoint[length("files:")+1:end], participant) + hasmethod(zmq_channel, Tuple{String,String}) || + throw(ArgumentError("the endpoint $endpoint needs ZeroMQ: `using ZMQ` (or use a folder, \"files:path\")")) + return zmq_channel(String(endpoint), String(participant)) +end + +# ── the participant ────────────────────────────────────────────────────────────── +""" + Turn + +A turn: `step` (0: the initial state, phase `"init"`), the time it starts from and the time it reaches (`until`), and +`frame`, the participant's inputs (its entity sets, their entities, its inputs; in `frame.update.removed` the entities +removed since its last turn). +""" +struct Turn + step::Int + phase::String + time::Float64 + until::Float64 + frame::Frame +end + +""" + Participant + +A participant in a session (`Participant(declaration; …)` joins it). `id_offset`: the ids of the entities it creates are in +`[id_offset, id_offset + id_count)`. +""" +mutable struct Participant + channel::CouplingChannel + declaration::Declaration + id_offset::Int64 + id_count::Int64 + session::Any + couplings::Vector{Any} + step::Int + stopped::Union{Nothing,String} + timeout::Union{Nothing,Float64} +end + +""" + Participant(declaration; endpoint=ENV["FSPM_TF_ENDPOINT"], links=Link[], timeout=nothing) -> Participant + +Join the session at `endpoint`: send the declaration and the links it proposes, wait for the welcome. +""" +function Participant(declaration::Declaration; endpoint = get(ENV, ENDPOINT_VARIABLE, ""), links = Link[], + timeout = nothing) + isempty(endpoint) && throw(ArgumentError("no endpoint: give one, or set $ENDPOINT_VARIABLE")) + isempty(declaration.participant) && (declaration.participant = get(ENV, PARTICIPANT_VARIABLE, "")) + channel = participant_channel(endpoint, declaration.participant) + send_message(channel, json_object("type" => "hello", "protocol" => PROTOCOL_VERSION, + "declaration" => declaration_json(declaration), + "links" => Any[link_json(link) for link in links])) + header, _ = receive_message(channel, timeout) + header["type"] == "stop" && error("the coordinator refused $(declaration.participant): $(get(header, "reason", ""))") + header["type"] == "welcome" || error("expected a welcome, got $(header["type"])") + return Participant(channel, declaration, Int64(header["idOffset"]), Int64(header["idCount"]), header["session"], + collect(get(header, "couplings", Any[])), -1, nothing, timeout === nothing ? nothing : + Float64(timeout)) +end + +""" + next_turn(participant) -> Union{Turn,Nothing} + +The next turn, or `nothing` when the coordinator stopped the session (`participant.stopped`: its reason). +""" +function next_turn(participant::Participant) + header, payload = receive_message(participant.channel, participant.timeout) + if header["type"] == "stop" + participant.stopped = String(get(header, "reason", "")) + close_channel(participant.channel) + return nothing + end + header["type"] == "turn" || error("expected a turn, got $(header["type"])") + participant.step = Int(header["step"]) + return Turn(Int(header["step"]), String(header["phase"]), Float64(header["time"]), Float64(header["until"]), + from_bytes(payload)) +end + +"""Iterate a participant's turns: `for turn in turns(participant) … end`.""" +struct Turns + participant::Participant +end +turns(participant::Participant) = Turns(participant) +Base.IteratorSize(::Type{Turns}) = Base.SizeUnknown() +Base.eltype(::Type{Turns}) = Turn +function Base.iterate(turns::Turns, state = nothing) + turn = next_turn(turns.participant) + return turn === nothing ? nothing : (turn, nothing) +end + +""" + send_update(participant, frame) + +Send the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the +entities it adds or changes); in `frame.update.removed`, the entities it removed. +""" +function send_update(participant::Participant, frame::Frame) + frame.update === nothing && (frame.update = Update(participant.declaration.participant)) + frame.update.participant = participant.declaration.participant + send_message(participant.channel, json_object("type" => "update", "step" => participant.step), to_bytes(frame)) + return nothing +end + +""" + fail(participant, message) + +Tell the coordinator this participant cannot go on (the session stops). +""" +fail(participant::Participant, message::AbstractString) = + send_message(participant.channel, json_object("type" => "error", "message" => String(message))) diff --git a/bindings/julia/FspmTF/src/model.jl b/bindings/julia/FspmTF/src/model.jl index 53fa2f0..f4a995a 100644 --- a/bindings/julia/FspmTF/src/model.jl +++ b/bindings/julia/FspmTF/src/model.jl @@ -296,12 +296,29 @@ function entity_set(data_structure::DataStructure, name::AbstractString) throw(KeyError("data structure '$(data_structure.id)' has no entity set '$name'")) end +""" + Update(participant; removed=OrderedDict()) + +What makes a frame partial: a coupling session's message (specification §11). `participant`: its sender +(`"coordinator"` for a turn's inputs). Its tables hold some rows and columns only. `removed`: (data structure id, +entity set name) => the ids of the entities removed. +""" +mutable struct Update + participant::String + removed::OrderedDict{Tuple{String,String},Vector{Int64}} +end + +Update(participant::AbstractString; removed = OrderedDict{Tuple{String,String},Vector{Int64}}()) = + Update(String(participant), OrderedDict{Tuple{String,String},Vector{Int64}}( + (String(key[1]), String(key[2])) => Vector{Int64}(ids) for (key, ids) in removed)) + """ Frame(; data_structures=DataStructure[], mappings=Mapping[], time=0.0, time_unit="d", step=0, - writer=OrderedDict(), vocabulary_version=nothing, unknown=OrderedDict()) + writer=OrderedDict(), vocabulary_version=nothing, update=nothing, unknown=OrderedDict()) -A scene at one time step (`time` in `time_unit`, `step`), or a parameter set. `unknown`: the content of a read file -this package does not interpret (other root extensions, the root's `extras`), written back as it was. +A scene at one time step (`time` in `time_unit`, `step`), or a parameter set. `update`: set for a partial frame +(`Update`), a coupling session's message. `unknown`: the content of a read file this package does not interpret +(other root extensions, the root's `extras`), written back as it was. """ Base.@kwdef mutable struct Frame data_structures::Vector{DataStructure} = DataStructure[] @@ -311,6 +328,7 @@ Base.@kwdef mutable struct Frame step::Int = 0 writer::AbstractDict{String,Any} = OrderedDict{String,Any}() vocabulary_version::Union{Nothing,String} = nothing + update::Union{Nothing,Update} = nothing unknown::AbstractDict{String,Any} = OrderedDict{String,Any}() end diff --git a/bindings/julia/FspmTF/test/coupling/function.jl b/bindings/julia/FspmTF/test/coupling/function.jl new file mode 100644 index 0000000..4b7cec4 --- /dev/null +++ b/bindings/julia/FspmTF/test/coupling/function.jl @@ -0,0 +1,45 @@ +# A Julia participant for the coupling tests (bindings/python/test/test_coupling.py launches it with the Python +# coordinator): the "function" of the two-scale session. It refines each organ it is sent into two segments and +# provides each segment's sugar, 1e-6 mol per m of its length (its input). +using FspmTF +using ZMQ + +declaration = Declaration("function"; platform = "julia-test", + entity_sets = [EntitySetSpec("plant", "Organ"), EntitySetSpec("plant", "SubOrgan"; rights = "refines")], + variables = [VariableSpec("plant", "SubOrgan", "segment_length", "requires"; unit = "m", standard_name = "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides"; unit = "mol", kind = "extensive")]) +participant = Participant(declaration; timeout = 60) + +segments = Dict{Int64,Tuple{Int64,Int64}}() # segment id => (its organ, its parent) +order = Int64[] # the segments in creation order +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + removed = get(turn.frame.update.removed, ("plant", "SubOrgan"), Int64[]) + for segment in removed + delete!(segments, segment) + end + filter!(segment -> haskey(segments, segment), order) + lengths = Dict{Int64,Float64}() + if any(set -> set.name == "SubOrgan", plant.entity_sets) + known = entity_set(plant, "SubOrgan") + for (identifier, length) in zip(known.columns["id"], known.columns["segment_length"]) + lengths[identifier] = isnan(length) ? 0.0 : length + end + end + with_segments = Set(first(value) for value in values(segments)) + for organ in entity_set(plant, "Organ").columns["id"] + organ in with_segments && continue + start = participant.id_offset + length(segments) + 2 * turn.step * 1000 + segments[start] = (organ, NO_ID) + segments[start+1] = (organ, start) + push!(order, start, start + 1) + end + columns = FspmTF.Columns( + "id" => order, + "complex" => [segments[segment][1] for segment in order], + "parent" => [segments[segment][2] for segment in order], + "edge_type" => [segments[segment][2] == NO_ID ? "none" : "successor" for segment in order], + "sugar" => [1e-6 * get(lengths, segment, 0.0) for segment in order]) + send_update(participant, Frame(data_structures = [DataStructure("plant"; + entity_sets = [EntitySet("SubOrgan"; columns = columns, rank = 6)])])) +end diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl index 5f6e1c2..25cf25e 100644 --- a/bindings/julia/FspmTF/test/runtests.jl +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -263,4 +263,34 @@ expected_of(relative) = JSON.parse(read(joinpath(CONFORMANCE, relative[1:end-4] end end end + + @testset "coupling" begin + # a partial frame: its sender and removed entities kept + frame = Frame(data_structures = [DataStructure("plant"; entity_sets = [ + EntitySet("SubOrgan"; rank = 6, columns = FspmTF.Columns( + "id" => Int64[10, 11], "complex" => Int64[3, 3], "parent" => Int64[NO_ID, 10], + "edge_type" => ["none", "successor"], "sugar" => [1.0, 2.0]))])], + update = Update("vpl"; removed = Dict(("plant", "SubOrgan") => Int64[5]))) + again = from_bytes(to_bytes(frame)) + @test again.update.participant == "vpl" + @test again.update.removed[("plant", "SubOrgan")] == [5] + @test entity_set(data_structure(again, "plant"), "SubOrgan").columns["complex"] == [3, 3] + # a declaration as JSON: an entity set followed for each variable's + declaration = Declaration("function"; entity_sets = [EntitySetSpec("plant", "SubOrgan"; rights = "refines")], + variables = [VariableSpec("plant", "Organ", "length", "requires"; unit = "m", + standard_name = "length")]) + value = FspmTF.declaration_json(declaration) + @test [item["name"] for item in value["entitySets"]] == ["SubOrgan", "Organ"] + @test value["entitySets"][1]["rank"] == 6 && value["entitySets"][2]["rights"] == "follows" + @test value["variables"][1]["standardName"] == "length" + # the folder transport: a participant's queues, a message and its payload + folder = mktempdir() + participant = FspmTF.FolderChannel(folder, "function") + FspmTF.send_message(participant, Dict("type" => "update", "step" => 0), to_bytes(frame)) + coordinator = FspmTF.FolderChannel(joinpath(folder, "function", "to_coordinator"), mktempdir(), 0, 0) + header, payload = FspmTF.receive_message(coordinator, 5) + @test header["type"] == "update" && from_bytes(payload).update.participant == "vpl" + @test isfile(joinpath(folder, "function", "to_coordinator", "00000000.glb")) + @test_throws ErrorException FspmTF.receive_message(coordinator, 0.05) # nothing more: a timeout + end end diff --git a/bindings/python/src/openalea/fspm_tf/__init__.py b/bindings/python/src/openalea/fspm_tf/__init__.py index 49a90b7..621df50 100644 --- a/bindings/python/src/openalea/fspm_tf/__init__.py +++ b/bindings/python/src/openalea/fspm_tf/__init__.py @@ -6,10 +6,10 @@ SPECIFICATION_VERSION = "0.2" from openalea.fspm_tf.model import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, # noqa: E402 - Table, Variable) + Table, Update, Variable) from openalea.fspm_tf.codec import (from_bytes, read, read_index, read_sequence, to_bytes, write, # noqa: E402 write_index, write_sequence) __all__ = ["SPECIFICATION_VERSION", "DataStructure", "EntitySet", "Frame", "Grid", "Mapping", "Mesh", "NO_ID", - "Relation", "Table", "Variable", "from_bytes", "read", "read_index", "read_sequence", "to_bytes", "write", + "Relation", "Table", "Update", "Variable", "from_bytes", "read", "read_index", "read_sequence", "to_bytes", "write", "write_index", "write_sequence"] diff --git a/bindings/python/src/openalea/fspm_tf/cli.py b/bindings/python/src/openalea/fspm_tf/cli.py index 247223e..14b599b 100644 --- a/bindings/python/src/openalea/fspm_tf/cli.py +++ b/bindings/python/src/openalea/fspm_tf/cli.py @@ -9,6 +9,11 @@ differ fspm-tf compare-all DUMPS CONFORMANCE every conformance file's expected values against the dump at the same path (x.expected.json: DUMPS/x.json); exit 1 if any differ + fspm-tf couple --steps N --interval DT [--participant NAME=COMMAND]... [--expect NAME]... + [--replay NAME=INDEX]... [--links LINKS.json] [--stages A,B;C] [--endpoint E] [--record DIR] + run a coupling session (specification §11): launch the participants' + commands, wait for the expected ones, replay sequences; exit 1 if it + stops on an error """ import argparse import sys @@ -87,8 +92,47 @@ def _compare_all(arguments) -> int: return 1 if failed or not results else 0 +def _pairs(values, what: str) -> dict: + pairs = {} + for value in values or []: + name, separator, rest = value.partition("=") + if not separator or not name or not rest: + raise SystemExit(f"--{what} {value}: NAME=VALUE expected") + pairs[name] = rest + return pairs + + +def _couple(arguments) -> int: + import json + from openalea.fspm_tf.coupling import Coordinator, CouplingError + links = [] + if arguments.links: + with open(arguments.links, encoding="utf-8") as file: + links = json.load(file) + stages = None if not arguments.stages else [stage.split(",") for stage in arguments.stages.split(";")] + coordinator = Coordinator(arguments.endpoint, steps=arguments.steps, interval=arguments.interval, + time_unit=arguments.time_unit, start=arguments.start, links=links, stages=stages, + expect=arguments.expect or [], replays=_pairs(arguments.replay, "replay"), + record=arguments.record, timeout=arguments.timeout) + print(f"coordinator at {coordinator.endpoint}", flush=True) + for name, command in _pairs(arguments.participant, "participant").items(): + coordinator.launch(name, command) + try: + coordinator.run() + except CouplingError as error: + print(f"the session stopped: {error}", file=sys.stderr) + return 1 + for coupling in coordinator.couplings: + source = "-" if coupling.source is None else f"{coupling.provider}.{coupling.source.name}" + print(f" {coupling.receiver}.{coupling.variable.name} <- {source} ({coupling.how}" + f"{', ' + coupling.aggregation if coupling.aggregation else ''}, factor {coupling.factor:g})") + print(f"{arguments.steps} steps, stages {coordinator.stages}" + + (f", recorded in {arguments.record}" if arguments.record else "")) + return 0 + + def main(argv=None) -> int: - parser = argparse.ArgumentParser(prog="fspm-tf", description="fspm-TF files: validate, inspect, display") + parser = argparse.ArgumentParser(prog="fspm-tf", description="fspm-TF files: validate, inspect, display; couple platforms") commands = parser.add_subparsers(dest="command", required=True) validate = commands.add_parser("validate", help="check frames (.glb) or sequences (scene.fspm.json)") validate.add_argument("files", nargs="+") @@ -112,6 +156,21 @@ def main(argv=None) -> int: compare_all.add_argument("dumps", help="the folder of the codec's dumps (x.json for x.expected.json)") compare_all.add_argument("conformance", help="fspm-TF's conformance folder") compare_all.set_defaults(run=_compare_all) + couple = commands.add_parser("couple", help="run a coupling session between platforms") + couple.add_argument("--steps", type=int, required=True) + couple.add_argument("--interval", type=float, required=True, help="the exchange interval") + couple.add_argument("--time-unit", default="d") + couple.add_argument("--start", type=float, default=0.) + couple.add_argument("--participant", action="append", help="NAME=COMMAND: a participant to launch") + couple.add_argument("--expect", action="append", help="NAME: a participant started separately") + couple.add_argument("--replay", action="append", help="NAME=INDEX: a recorded sequence as a participant") + couple.add_argument("--links", help="a JSON file: the session's links") + couple.add_argument("--stages", help="the stages, e.g. 'growth;function,light' (default: owners, refiners, " + "others)") + couple.add_argument("--endpoint", default="tcp://127.0.0.1:*", help="a ZeroMQ endpoint, or files:FOLDER") + couple.add_argument("--record", help="a folder: the session recorded as a sequence") + couple.add_argument("--timeout", type=float, default=600., help="seconds to wait for a message") + couple.set_defaults(run=_couple) arguments = parser.parse_args(argv) return arguments.run(arguments) diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/fspm_tf/codec.py index 7d5d950..217556e 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/fspm_tf/codec.py @@ -20,7 +20,7 @@ from openalea.fspm_tf import SPECIFICATION_VERSION from openalea.fspm_tf.glb import BufferBuilder, pack, unpack, view_bytes from openalea.fspm_tf.model import (DataStructure, EDGE_TYPES, ENUM_COLUMNS, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, - Relation, Table, Variable) + Relation, Table, Update, Variable) EXTENSION = "FSPM_scene" METADATA = "EXT_structural_metadata" @@ -130,12 +130,25 @@ def add_table(table: Table, class_id: str) -> int: **({"method": mapping.method} if mapping.method else {}), "propertyTable": add_table(mapping, f"mapping__{mapping.name}")}) + update = None + if frame.update is not None: + _check_identifier(frame.update.participant, "participant") + update = {"participant": frame.update.participant} + if frame.update.removed: + update["removed"] = [ + {"dataStructure": data_structure_id, "entitySet": set_name, + "propertyTable": add_table(Table(f"{set_name}__removed", columns={"id": ids}), + f"{data_structure_id}__{set_name}__removed")} + for (data_structure_id, set_name), ids in frame.update.removed.items()] + scene = {"version": SPECIFICATION_VERSION, "frame": {"time": float(frame.time), "timeUnit": frame.time_unit, "step": int(frame.step)}} if frame.vocabulary_version: scene["vocabulary"] = frame.vocabulary_version if frame.writer: scene["writer"] = dict(frame.writer) + if update is not None: + scene["update"] = update scene["dataStructures"] = data_structures if mappings: scene["mappings"] = mappings @@ -293,6 +306,8 @@ def _enum_codes(name: str, column: np.ndarray, class_id: str, schema: dict) -> t if unknown: raise ValueError(f"edge_type of '{class_id}': unknown values {sorted(unknown)} (allowed: {EDGE_TYPES})") return EDGE_TYPE_ENUM, np.array([EDGE_TYPES.index(value) for value in column], dtype=np.uint8) + if any(value is None for value in column): + raise ValueError(f"{name} of '{class_id}': an enum column has no missing values (give each row a {name})") names = sorted(set(column)) enum_id = f"{class_id}__{name}" value_type = "UINT8" if len(names) <= 256 else "UINT16" @@ -408,6 +423,11 @@ def table(index: int, kind, **fields): source=(entry["source"]["dataStructure"], entry["source"]["entitySet"]), target=(entry["target"]["dataStructure"], entry["target"]["entitySet"]), method=entry.get("method"))) + if "update" in scene: + update = scene["update"] + frame.update = Update(participant=update["participant"], removed={ + (item["dataStructure"], item["entitySet"]): table(item["propertyTable"], Table).columns["id"] + for item in update.get("removed", [])}) unknown_extensions = {name: value for name, value in extensions.items() if name not in (METADATA, EXTENSION, FEATURES)} if unknown_extensions: diff --git a/bindings/python/src/openalea/fspm_tf/coupling/__init__.py b/bindings/python/src/openalea/fspm_tf/coupling/__init__.py new file mode 100644 index 0000000..74e145c --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/__init__.py @@ -0,0 +1,18 @@ +""" +Coupling platforms through fspm-TF (specification §11): a session between participants, each a platform's model +in its own process (or a recorded sequence), coordinated by a coordinator that holds the shared structure. + +- messages: the declarations, links and errors (Declaration, EntitySetSpec, VariableSpec, Link, CouplingError); +- resolution: each input's provider and the stages (resolve, stages); +- store: the session's structure, the updates merged into it and the inputs read from it (SessionStore); +- channels: the transports, ZeroMQ and a folder; +- participant: a platform's side (Participant, Turn); +- coordinator: the coordinator (Coordinator), launching participants and recording the session; +- replay: a recorded sequence as a participant (Replay). +""" +from openalea.fspm_tf.coupling.messages import (ID_COUNT, PROTOCOL_VERSION, CouplingError, Declaration, # noqa: F401 + EntitySetSpec, Link, VariableSpec) +from openalea.fspm_tf.coupling.resolution import Coupling, resolve, stages # noqa: F401 +from openalea.fspm_tf.coupling.participant import Participant, Turn # noqa: F401 +from openalea.fspm_tf.coupling.coordinator import Coordinator, python_command # noqa: F401 +from openalea.fspm_tf.coupling.replay import Replay # noqa: F401 diff --git a/bindings/python/src/openalea/fspm_tf/coupling/channels.py b/bindings/python/src/openalea/fspm_tf/coupling/channels.py new file mode 100644 index 0000000..9cc7730 --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/channels.py @@ -0,0 +1,191 @@ +""" +The session's transports. A message is a JSON header and, for turns and updates, a payload: a partial frame's GLB +bytes. Both transports carry the same messages: + +- ZeroMQ (endpoint "tcp://host:port" or "ipc://path"): the coordinator binds a ROUTER socket, each participant + connects a DEALER socket whose identity is its name; a message is two parts (header, payload) or one (header). +- a folder (endpoint "files:path"): each participant has a folder // with two message queues, + to_coordinator/ and to_participant/; message n is n.json (its header) and n.glb (its payload), the payload written + first and the header renamed into place last (a header present: a complete message). Nothing else is needed than + files, so any platform can take part, and the folder is the session's log. + +Both sides poll; every wait has a timeout (seconds, None: none). +""" +import json +import os +import time + +POLL = 0.005 # seconds between two looks at a folder queue +ENDPOINT_VARIABLE = "FSPM_TF_ENDPOINT" # what a launched participant connects to +PARTICIPANT_VARIABLE = "FSPM_TF_PARTICIPANT" # its name + + +class ChannelTimeout(TimeoutError): + pass + + +def _header(data: bytes) -> dict: + return json.loads(data.decode("utf-8")) + + +def _bytes(header: dict) -> bytes: + return json.dumps(header, separators=(",", ":")).encode("utf-8") + + +# ── ZeroMQ ─────────────────────────────────────────────────────────────────────── +class ZmqCoordinatorChannel: + def __init__(self, endpoint: str = "tcp://127.0.0.1:*"): + import zmq + self._zmq = zmq + self.context = zmq.Context.instance() + self.socket = self.context.socket(zmq.ROUTER) + self.socket.setsockopt(zmq.LINGER, 1000) + self.socket.bind(endpoint) + self.endpoint = self.socket.getsockopt_string(zmq.LAST_ENDPOINT) + + def receive(self, timeout=None): + """(participant, header, payload or None).""" + if not self.socket.poll(None if timeout is None else int(timeout * 1000)): + raise ChannelTimeout(f"no message in {timeout} s") + parts = self.socket.recv_multipart() + return parts[0].decode("utf-8"), _header(parts[1]), parts[2] if len(parts) > 2 else None + + def send(self, participant: str, header: dict, payload: bytes = None) -> None: + parts = [participant.encode("utf-8"), _bytes(header)] + ([payload] if payload is not None else []) + self.socket.send_multipart(parts) + + def close(self) -> None: + self.socket.close() + + +class ZmqParticipantChannel: + def __init__(self, endpoint: str, participant: str): + import zmq + self.context = zmq.Context.instance() + self.socket = self.context.socket(zmq.DEALER) + self.socket.setsockopt(zmq.IDENTITY, participant.encode("utf-8")) + self.socket.setsockopt(zmq.LINGER, 1000) + self.socket.connect(endpoint) + + def receive(self, timeout=None): + """(header, payload or None).""" + if not self.socket.poll(None if timeout is None else int(timeout * 1000)): + raise ChannelTimeout(f"no message in {timeout} s") + parts = self.socket.recv_multipart() + return _header(parts[0]), parts[1] if len(parts) > 1 else None + + def send(self, header: dict, payload: bytes = None) -> None: + self.socket.send_multipart([_bytes(header)] + ([payload] if payload is not None else [])) + + def close(self) -> None: + self.socket.close() + + +# ── a folder ───────────────────────────────────────────────────────────────────── +class _Queue: + """A folder of numbered messages, written by one side and read by the other.""" + + def __init__(self, folder: str): + self.folder = folder + os.makedirs(folder, exist_ok=True) + self.next_read = 0 + self.next_write = 0 + + def write(self, header: dict, payload: bytes = None) -> None: + stem = os.path.join(self.folder, f"{self.next_write:08d}") + if payload is not None: + with open(stem + ".glb", "wb") as file: + file.write(payload) + header = dict(header, payload=os.path.basename(stem) + ".glb") + with open(stem + ".json.part", "wb") as file: + file.write(_bytes(header)) + os.replace(stem + ".json.part", stem + ".json") + self.next_write += 1 + + def read(self): + """The next message (header, payload or None), or None if not there yet.""" + stem = os.path.join(self.folder, f"{self.next_read:08d}") + if not os.path.exists(stem + ".json"): + return None + with open(stem + ".json", "rb") as file: + header = _header(file.read()) + payload = None + if "payload" in header: + with open(os.path.join(self.folder, header.pop("payload")), "rb") as file: + payload = file.read() + self.next_read += 1 + return header, payload + + +class FolderCoordinatorChannel: + def __init__(self, folder: str): + self.folder = os.path.abspath(folder) + os.makedirs(self.folder, exist_ok=True) + for name in os.listdir(self.folder): # a session's folder is the log of that session only + if os.path.exists(os.path.join(self.folder, name, "to_participant", f"{0:08d}.json")): + raise ValueError(f"{self.folder} holds a previous session ({name}): use a new folder") + self.endpoint = "files:" + self.folder + self.incoming, self.outgoing = {}, {} + + def _discover(self) -> None: + for name in sorted(os.listdir(self.folder)): + if name not in self.incoming and os.path.isdir(os.path.join(self.folder, name, "to_coordinator")): + self.incoming[name] = _Queue(os.path.join(self.folder, name, "to_coordinator")) + self.outgoing[name] = _Queue(os.path.join(self.folder, name, "to_participant")) + + def receive(self, timeout=None): + deadline = None if timeout is None else time.monotonic() + timeout + while True: + self._discover() + for name, queue in self.incoming.items(): + message = queue.read() + if message is not None: + return (name,) + message + if deadline is not None and time.monotonic() > deadline: + raise ChannelTimeout(f"no message in {timeout} s") + time.sleep(POLL) + + def send(self, participant: str, header: dict, payload: bytes = None) -> None: + self._discover() + self.outgoing[participant].write(header, payload) + + def close(self) -> None: + pass + + +class FolderParticipantChannel: + def __init__(self, folder: str, participant: str): + base = os.path.join(folder, participant) + self.incoming = _Queue(os.path.join(base, "to_participant")) + self.outgoing = _Queue(os.path.join(base, "to_coordinator")) + + def receive(self, timeout=None): + deadline = None if timeout is None else time.monotonic() + timeout + while True: + message = self.incoming.read() + if message is not None: + return message + if deadline is not None and time.monotonic() > deadline: + raise ChannelTimeout(f"no message in {timeout} s") + time.sleep(POLL) + + def send(self, header: dict, payload: bytes = None) -> None: + self.outgoing.write(header, payload) + + def close(self) -> None: + pass + + +def coordinator_channel(endpoint: str): + """The coordinator's side of *endpoint*: "files:path", or a ZeroMQ endpoint to bind ("tcp://127.0.0.1:*": a + free port).""" + if endpoint.startswith("files:"): + return FolderCoordinatorChannel(endpoint[len("files:"):]) + return ZmqCoordinatorChannel(endpoint) + + +def participant_channel(endpoint: str, participant: str): + """A participant's side of *endpoint*.""" + if endpoint.startswith("files:"): + return FolderParticipantChannel(endpoint[len("files:"):], participant) + return ZmqParticipantChannel(endpoint, participant) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py b/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py new file mode 100644 index 0000000..dab53cb --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py @@ -0,0 +1,199 @@ +""" +The coordinator: it runs a session between participants it launches (commands) or that connect to it, and replayed +sequences. + +1. Each participant sends its hello (its declaration, and links it proposes); the session's links are those given to + the coordinator and those proposed, the given ones first. +2. Resolution (resolution.resolve, resolution.stages): each input's provider, the stages. Each participant is + welcomed with its id range and its couplings, or the session stops with the reason. +3. Initial state: stage by stage, each participant is sent a turn of step 0 (the entities so far and its inputs) and + answers with its initial outputs and entities. +4. Steps 1 … steps: stage by stage, the participants of a stage are sent their turn (time t → t + interval: their + inputs, the entities added or removed since their last turn), and their updates are merged once all of them + answered (so within a stage they see the same state, and the next stage sees theirs). +5. Stop. A participant's error, or its update the store refuses, stops the session: every participant is told why. + +*record*: a folder where the session is written as a sequence of full frames (every entity set and output), after the +initial state and each step. +""" +import os +import subprocess +import sys +import time as clock + +from openalea.fspm_tf.codec import from_bytes, to_bytes, write, write_index +from openalea.fspm_tf.coupling.channels import (ChannelTimeout, ENDPOINT_VARIABLE, PARTICIPANT_VARIABLE, + coordinator_channel) +from openalea.fspm_tf.coupling.messages import (ID_COUNT, PROTOCOL_VERSION, CouplingError, Declaration, Link) +from openalea.fspm_tf.coupling.replay import Replay +from openalea.fspm_tf.coupling.resolution import ranks, resolve, stages +from openalea.fspm_tf.coupling.store import SessionStore + + +class Coordinator: + def __init__(self, endpoint: str = "tcp://127.0.0.1:*", steps: int = 1, interval: float = 1., + time_unit: str = "d", start: float = 0., links=(), stages=None, expect=(), replays=None, + record: str = None, timeout: float = 600.): + self.channel = coordinator_channel(endpoint) + self.endpoint = self.channel.endpoint + self.steps, self.interval, self.time_unit, self.start = steps, interval, time_unit, start + self.links = [link if isinstance(link, Link) else Link.from_json(link) for link in links] + self.given_stages = stages + self.expect = list(expect) + self.replays = {name: Replay(name, path) for name, path in (replays or {}).items()} + self.record, self.timeout = record, timeout + self.processes = {} + self.declarations, self.offsets, self.removed_seen = {}, {}, {} + self.couplings, self.stages, self.store = [], [], None + self.recorded = [] + + # ── launching ───────────────────────────────────────────────────────────────── + def launch(self, name: str, command, environment=None, cwd=None) -> None: + """Start participant *name*: *command* (a list, or a string run by the shell) with FSPM_TF_ENDPOINT and + FSPM_TF_PARTICIPANT set; the session waits for its hello.""" + environment = dict(os.environ if environment is None else environment) + environment[ENDPOINT_VARIABLE] = self.endpoint + environment[PARTICIPANT_VARIABLE] = name + self.processes[name] = subprocess.Popen(command, env=environment, cwd=cwd, shell=isinstance(command, str)) + self.expect.append(name) + + def _check_processes(self) -> None: + for name, process in self.processes.items(): + code = process.poll() + if code not in (None, 0) or (code == 0 and name not in self.finished): + raise CouplingError(f"participant {name} exited with code {code} during the session") + + def _receive(self): + """The next message, watching the launched participants and the timeout.""" + deadline = None if self.timeout is None else clock.monotonic() + self.timeout + while True: + try: + return self.channel.receive(0.2) + except ChannelTimeout: + self._check_processes() + if deadline is not None and clock.monotonic() > deadline: + raise CouplingError(f"no message in {self.timeout} s") from None + + # ── the session ────────────────────────────────────────────────────────────── + def run(self) -> SessionStore: + """Run the session; the session's store at its end.""" + self.finished = set() + try: + self._join() + self._run_steps() + except Exception as error: + self._stop(f"{type(error).__name__}: {error}") + self._wait_processes() + raise + self._stop("end of the session") + self._wait_processes() + if self.record and self.recorded: + write_index(os.path.join(self.record, "scene.fspm.json"), self.recorded, time_unit=self.time_unit) + return self.store + + def _join(self) -> None: + pending = set(self.expect) + proposed = [] + for name, replay in self.replays.items(): + self.declarations[name] = replay.declaration + while pending: + participant, header, _ = self._receive() + if header.get("type") == "error": + raise CouplingError(f"{participant}: {header.get('message')}") + if header.get("type") != "hello": + raise CouplingError(f"{participant}: expected its hello, got {header.get('type')}") + if header.get("protocol") != PROTOCOL_VERSION: + raise CouplingError(f"{participant}: protocol {header.get('protocol')}, the coordinator's is " + f"{PROTOCOL_VERSION}") + declaration = Declaration.from_json(header["declaration"]) + if declaration.participant != participant: + raise CouplingError(f"{participant} declares itself as {declaration.participant}") + if participant in self.declarations: + raise CouplingError(f"two participants named {participant}") + if participant not in pending: + raise CouplingError(f"{participant} is not expected (expected: {sorted(self.expect)})") + self.declarations[participant] = declaration + proposed += [Link.from_json(link) for link in header.get("links", [])] + pending.discard(participant) + given = {(link.receiver, link.variable, link.entity_set) for link in self.links} + links = self.links + [link for link in proposed + if (link.receiver, link.variable, link.entity_set) not in given] + self.couplings = resolve(self.declarations, links) + self.stages = stages(self.declarations, self.given_stages) + self.store = SessionStore(ranks(self.declarations)) + order = [name for name in self.expect if name in self.declarations] + list(self.replays) + for position, name in enumerate(order): # deterministic: the expected order, then replays + self.offsets[name] = (position + 1) * ID_COUNT + self.removed_seen[name] = 0 + for name, replay in self.replays.items(): + replay.id_offset = self.offsets[name] + replay.linear = {(coupling.source.data_structure, coupling.source.entity_set, coupling.source.name) + for coupling in self.couplings + if coupling.provider == name and coupling.interpolation == "linear"} + session = {"start": self.start, "interval": self.interval, "timeUnit": self.time_unit, "steps": self.steps, + "stages": self.stages, "participants": list(self.declarations)} + for name in self.declarations: + if name in self.replays: + continue + self.channel.send(name, {"type": "welcome", "idOffset": self.offsets[name], "idCount": ID_COUNT, + "session": session, + "couplings": [coupling.to_json() for coupling in self.couplings + if coupling.receiver == name]}) + + def _run_steps(self) -> None: + for step in range(self.steps + 1): + time = self.start + max(step - 1, 0) * self.interval + until = self.start + step * self.interval + for stage in self.stages: + updates = {} + live = [name for name in stage if name not in self.replays] + for name in live: + frame = self.store.inputs(name, self.declarations[name], self.couplings, self.removed_seen[name], + time, self.time_unit, step) + self.removed_seen[name] = len(self.store.removed_log) + self.channel.send(name, {"type": "turn", "step": step, "phase": "init" if step == 0 else "step", + "time": time, "until": until}, to_bytes(frame)) + for name in stage: + if name in self.replays: + updates[name] = self.replays[name].update(until, self.time_unit) + waiting = set(live) + while waiting: + participant, header, payload = self._receive() + if header.get("type") == "error": + raise CouplingError(f"{participant}: {header.get('message')}") + if header.get("type") != "update" or participant not in waiting: + raise CouplingError(f"{participant}: unexpected {header.get('type')} at step {step}") + if header.get("step") != step: + raise CouplingError(f"{participant}: update of step {header.get('step')} at step {step}") + updates[participant] = from_bytes(payload) + waiting.discard(participant) + for name in stage: # merged in the stage's order, once all answered + self.store.merge(name, self.declarations[name], updates[name], self.offsets[name]) + if self.record: + os.makedirs(self.record, exist_ok=True) + path = f"frame_{step:05d}.glb" + write(os.path.join(self.record, path), self.store.frame(until, self.time_unit, step)) + self.recorded.append({"time": until, "step": step, "file": path}) + + def _stop(self, reason: str) -> None: + for name in self.declarations: + if name in self.replays: + continue + try: + self.channel.send(name, {"type": "stop", "reason": reason}) + except Exception: # a participant already gone + pass + self.finished = set(self.declarations) + + def _wait_processes(self) -> None: + for name, process in self.processes.items(): + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + self.channel.close() + + +def python_command(script: str, *arguments) -> list: + """The command running a Python *script* with this interpreter.""" + return [sys.executable, script, *map(str, arguments)] diff --git a/bindings/python/src/openalea/fspm_tf/coupling/messages.py b/bindings/python/src/openalea/fspm_tf/coupling/messages.py new file mode 100644 index 0000000..106efb3 --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/messages.py @@ -0,0 +1,185 @@ +""" +A coupling session's messages (specification §11), as JSON headers, some with a partial frame (GLB bytes) as payload. + +Participant to coordinator:: + + hello {"type": "hello", "protocol": "0.3", "declaration": {...}, "links": [...]} + update {"type": "update", "step": k} + its outputs (a partial frame) + error {"type": "error", "message": "..."} + +Coordinator to participant:: + + welcome {"type": "welcome", "idOffset": ..., "idCount": ..., "session": {...}, "couplings": [...]} + turn {"type": "turn", "step": k, "phase": "init" | "step", "time": t, "until": t'} + its inputs + stop {"type": "stop", "reason": "..."} + +A declaration says what a participant works on (its entity sets and its rights on their topology), what it provides +(its outputs) and requires (its inputs). Links say which output feeds which input; the coordinator completes them by +standard name, then by name. +""" +from dataclasses import dataclass, field +from typing import Optional + +from openalea.fspm_tf.model import BASE_SCALES + +PROTOCOL_VERSION = "0.3" +RIGHTS = ("owns", "refines", "follows") +AGGREGATIONS = ("sum", "mean", "max", "min", "copy") +INTERPOLATIONS = ("hold", "linear") +ID_COUNT = 2 ** 32 # the ids a participant may create: [offset, offset + ID_COUNT) + + +class CouplingError(RuntimeError): + """A coupling that cannot run: a declaration, link or update the protocol refuses, or a participant's error.""" + + +@dataclass +class EntitySetSpec: + """An entity set a participant works on, with its rank (coarser first: a base scale's position by default) and + its rights on its topology: "owns" (adds and removes its entities), "refines" (adds and removes the entities it + creates, under others'), "follows" (reads).""" + data_structure: str + name: str + rank: Optional[int] = None + rights: str = "follows" + + def __post_init__(self): + if self.rank is None and self.name in BASE_SCALES: + self.rank = BASE_SCALES.index(self.name) + 1 + if self.rights not in RIGHTS: + raise CouplingError(f"entity set {self.data_structure}/{self.name}: rights '{self.rights}', one of {RIGHTS}") + + def to_json(self) -> dict: + return {"dataStructure": self.data_structure, "name": self.name, "rank": self.rank, "rights": self.rights} + + @classmethod + def from_json(cls, value: dict) -> "EntitySetSpec": + return cls(value["dataStructure"], value["name"], value.get("rank"), value.get("rights", "follows")) + + +@dataclass +class VariableSpec: + """A variable a participant provides (an output) or requires (an input), on one entity set: its name in the + participant, its unit (UCUM), standard name and kind. An optional input may stay unprovided.""" + data_structure: str + entity_set: str + name: str + direction: str + unit: str = "1" + standard_name: Optional[str] = None + kind: Optional[str] = None + optional: bool = False + + def __post_init__(self): + if self.direction not in ("provides", "requires"): + raise CouplingError(f"variable {self.name}: direction '{self.direction}', 'provides' or 'requires'") + + @property + def where(self) -> tuple: + return self.data_structure, self.entity_set + + def to_json(self) -> dict: + value = {"dataStructure": self.data_structure, "entitySet": self.entity_set, "name": self.name, + "direction": self.direction, "unit": self.unit} + if self.standard_name: + value["standardName"] = self.standard_name + if self.kind: + value["kind"] = self.kind + if self.optional: + value["optional"] = True + return value + + @classmethod + def from_json(cls, value: dict) -> "VariableSpec": + return cls(value["dataStructure"], value["entitySet"], value["name"], value["direction"], + value.get("unit", "1"), value.get("standardName"), value.get("kind"), value.get("optional", False)) + + +@dataclass +class Declaration: + """A participant's declaration: its name, platform, entity sets and variables.""" + participant: str + platform: Optional[str] = None + entity_sets: list = field(default_factory=list) + variables: list = field(default_factory=list) + + def provides(self) -> list: + return [variable for variable in self.variables if variable.direction == "provides"] + + def requires(self) -> list: + return [variable for variable in self.variables if variable.direction == "requires"] + + def entity_set(self, data_structure: str, name: str) -> Optional[EntitySetSpec]: + for entity_set in self.entity_sets: + if (entity_set.data_structure, entity_set.name) == (data_structure, name): + return entity_set + return None + + def rights(self, data_structure: str, name: str) -> str: + entity_set = self.entity_set(data_structure, name) + return entity_set.rights if entity_set is not None else "follows" + + def complete(self) -> "Declaration": + """The declaration with an entity set (following it) for each one its variables are on and it did not + list.""" + for variable in self.variables: + if self.entity_set(*variable.where) is None: + self.entity_sets.append(EntitySetSpec(variable.data_structure, variable.entity_set)) + return self + + def to_json(self) -> dict: + value = {"participant": self.participant, "entitySets": [item.to_json() for item in self.entity_sets], + "variables": [item.to_json() for item in self.variables]} + if self.platform: + value["platform"] = self.platform + return value + + @classmethod + def from_json(cls, value: dict) -> "Declaration": + return cls(value["participant"], value.get("platform"), + [EntitySetSpec.from_json(item) for item in value.get("entitySets", [])], + [VariableSpec.from_json(item) for item in value.get("variables", [])]).complete() + + +@dataclass +class Link: + """An input's provider: the receiver's variable (*entity_set* when its name is ambiguous) fed by the provider's + *source* variable, times *factor* (after the unit conversion), changed of scale by *aggregation* (sum, mean, max, + min from a finer entity set; copy from a coarser one), and between a replayed sequence's times by + *interpolation* (hold, linear).""" + receiver: str + variable: str + provider: str + source: str + entity_set: Optional[str] = None + source_entity_set: Optional[str] = None + factor: float = 1. + aggregation: Optional[str] = None + interpolation: str = "hold" + + def __post_init__(self): + if self.aggregation is not None and self.aggregation not in AGGREGATIONS: + raise CouplingError(f"link {self.receiver}.{self.variable}: aggregation '{self.aggregation}', one of " + f"{AGGREGATIONS}") + if self.interpolation not in INTERPOLATIONS: + raise CouplingError(f"link {self.receiver}.{self.variable}: interpolation '{self.interpolation}', one " + f"of {INTERPOLATIONS}") + + def to_json(self) -> dict: + value = {"receiver": self.receiver, "variable": self.variable, "provider": self.provider, + "source": self.source} + for key, name in (("entity_set", "entitySet"), ("source_entity_set", "sourceEntitySet"), + ("aggregation", "aggregation")): + if getattr(self, key) is not None: + value[name] = getattr(self, key) + if self.factor != 1.: + value["factor"] = self.factor + if self.interpolation != "hold": + value["interpolation"] = self.interpolation + return value + + @classmethod + def from_json(cls, value: dict) -> "Link": + return cls(value["receiver"], value["variable"], value["provider"], value["source"], + value.get("entitySet"), value.get("sourceEntitySet"), float(value.get("factor", 1.)), + value.get("aggregation"), value.get("interpolation", "hold")) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/participant.py b/bindings/python/src/openalea/fspm_tf/coupling/participant.py new file mode 100644 index 0000000..99526d6 --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/participant.py @@ -0,0 +1,91 @@ +""" +A participant's side of a session: what a platform calls to take part, whatever its own model is:: + + from openalea.fspm_tf.coupling import Declaration, Participant + + participant = Participant.connect(declaration) # endpoint and name from the environment if launched + for turn in participant.turns(): # until the coordinator stops the session + inputs = turn.frame # its entity sets, their entities and its inputs + ... # its model, from turn.time to turn.until + participant.send(frame) # its outputs and topology changes (a partial frame) + +The ids of the entities it creates are in [participant.id_offset, participant.id_offset + ID_COUNT). +""" +import os +from dataclasses import dataclass + +from openalea.fspm_tf.codec import from_bytes, to_bytes +from openalea.fspm_tf.coupling.channels import (ENDPOINT_VARIABLE, PARTICIPANT_VARIABLE, participant_channel) +from openalea.fspm_tf.coupling.messages import PROTOCOL_VERSION, CouplingError, Declaration, Link +from openalea.fspm_tf.model import Frame, Update + + +@dataclass +class Turn: + """A turn: *step* (0: the initial state, "init"), the time it starts from and the time it reaches (*until*; + equal at init), and the frame of the participant's inputs (its entity sets, their entities, its inputs, and in + frame.update.removed the entities removed since its last turn).""" + step: int + phase: str + time: float + until: float + frame: Frame + + +class Participant: + def __init__(self, channel, declaration: Declaration, timeout=None): + self.channel, self.declaration, self.timeout = channel, declaration, timeout + self.id_offset = None + self.session = None + self.couplings = [] + self.step = None + self.stopped = None + + @classmethod + def connect(cls, declaration: Declaration, endpoint: str = None, links=(), timeout=None) -> "Participant": + """Join the session at *endpoint* (default: the environment's FSPM_TF_ENDPOINT, set by the coordinator + launching the participant; its name: FSPM_TF_PARTICIPANT if the declaration has none): send the declaration + (and the *links* it proposes), wait for the welcome.""" + endpoint = endpoint or os.environ.get(ENDPOINT_VARIABLE) + if not endpoint: + raise CouplingError(f"no endpoint: give one, or set {ENDPOINT_VARIABLE}") + if not declaration.participant: + declaration.participant = os.environ.get(PARTICIPANT_VARIABLE, "") + participant = cls(participant_channel(endpoint, declaration.participant), declaration.complete(), timeout) + participant.channel.send({"type": "hello", "protocol": PROTOCOL_VERSION, + "declaration": declaration.to_json(), + "links": [link.to_json() if isinstance(link, Link) else link for link in links]}) + header, _ = participant.channel.receive(timeout) + if header["type"] == "stop": + raise CouplingError(f"the coordinator refused {declaration.participant}: {header.get('reason')}") + if header["type"] != "welcome": + raise CouplingError(f"expected a welcome, got {header['type']}") + participant.id_offset = header["idOffset"] + participant.session = header["session"] + participant.couplings = header.get("couplings", []) + return participant + + def turns(self): + """The turns, until the coordinator stops the session (self.stopped: its reason).""" + while True: + header, payload = self.channel.receive(self.timeout) + if header["type"] == "stop": + self.stopped = header.get("reason", "") + self.channel.close() + return + if header["type"] != "turn": + raise CouplingError(f"expected a turn, got {header['type']}") + self.step = header["step"] + yield Turn(header["step"], header["phase"], header["time"], header["until"], from_bytes(payload)) + + def send(self, frame: Frame) -> None: + """Send the turn's outputs: *frame* holds, per entity set, the ids and the outputs of the rows sent (and the + topology columns of the entities it adds or changes); frame.update.removed the entities it removed.""" + if frame.update is None: + frame.update = Update(self.declaration.participant) + frame.update.participant = self.declaration.participant + self.channel.send({"type": "update", "step": self.step}, to_bytes(frame)) + + def fail(self, message: str) -> None: + """Tell the coordinator this participant cannot go on (the session stops).""" + self.channel.send({"type": "error", "message": message}) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/replay.py b/bindings/python/src/openalea/fspm_tf/coupling/replay.py new file mode 100644 index 0000000..8cf22de --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/replay.py @@ -0,0 +1,108 @@ +""" +A recorded sequence as a participant: a platform's log (a sequence of full frames) providing its variables to a +session without the platform running. It owns its entity sets' topology: an entity appearing between two recorded +frames is added, one disappearing is removed. Between recorded times its values are those of the last frame at or +before the session's time ("hold"), or interpolated linearly for the variables linked with interpolation "linear". +""" +import numpy as np + +from openalea.fspm_tf.codec import read, read_index +from openalea.fspm_tf.coupling.messages import Declaration, EntitySetSpec, VariableSpec +from openalea.fspm_tf.model import NO_ID, DataStructure, EntitySet, Frame, Update +from openalea.fspm_tf.ucum import conversion_factor + +_ID_COLUMNS = ("id", "complex", "parent") + + +class Replay: + """A sequence (its index, scene.fspm.json) replayed as participant *name*.""" + + def __init__(self, name: str, index_path: str): + self.name = name + index = read_index(index_path) + self.time_unit = index.get("timeUnit", "d") + self.entries = sorted(index["frames"], key=lambda entry: entry["time"]) + self.paths = [entry["path"] for entry in self.entries] + self.times = np.array([entry["time"] for entry in self.entries], dtype=float) + self._cache = {} + self.id_offset = 0 + self.linear = set() # (data structure, entity set, variable) interpolated + self.sent = {} # (data structure, entity set): ids sent at the last turn + self.id_map = {} # recorded id: session id + self.declaration = self._declare(self._frame(0)) + + def _frame(self, position: int) -> Frame: + if position not in self._cache: + self._cache = {position: read(self.paths[position])} # one frame kept + return self._cache[position] + + def _declare(self, frame: Frame) -> Declaration: + declaration = Declaration(self.name, platform=frame.writer.get("platform", "replay")) + for data_structure in frame.data_structures: + for entity_set in data_structure.entity_sets: + declaration.entity_sets.append(EntitySetSpec(data_structure.id, entity_set.name, entity_set.rank, + "owns")) + for name in entity_set.variable_names(): + variable = entity_set.declaration(name) + declaration.variables.append(VariableSpec(data_structure.id, entity_set.name, name, "provides", + variable.unit, variable.standard_name, variable.kind)) + return declaration + + def update(self, time: float, time_unit: str) -> Frame: + """The update at the session's *time* (in *time_unit*): the recorded entities and values then.""" + recorded_time = time * conversion_factor(time_unit, self.time_unit) + position = int(np.searchsorted(self.times, recorded_time, side="right")) - 1 + position = max(position, 0) + frame = self._frame(position) + following = None + if self.linear and position + 1 < len(self.times) and recorded_time > self.times[position]: + following = read(self.paths[position + 1]) + weight = (recorded_time - self.times[position]) / (self.times[position + 1] - self.times[position]) + structures, removed = [], {} + for data_structure in frame.data_structures: + out = DataStructure(data_structure.id) + for entity_set in data_structure.entity_sets: + columns = {} + for name, values in entity_set.columns.items(): + if name in _ID_COLUMNS: + values = np.array([self._session_id(int(value)) for value in np.asarray(values).tolist()], + dtype=np.int64) + columns[name] = values + if following is not None: + self._interpolate(data_structure.id, entity_set, following, weight, columns) + out.entity_sets.append(EntitySet(entity_set.name, columns=columns, rank=entity_set.rank, + variables=dict(entity_set.variables))) + key = (data_structure.id, entity_set.name) + ids = set(columns["id"].tolist()) if "id" in columns else set() + gone = self.sent.get(key, set()) - ids + if gone: + removed[key] = np.array(sorted(gone), dtype=np.int64) + self.sent[key] = ids + structures.append(out) + return Frame(data_structures=structures, update=Update(self.name, removed)) + + def _session_id(self, recorded: int) -> int: + """The session id of a recorded id: the replay's ids renumbered from its offset, in order of appearance.""" + if recorded == NO_ID: + return NO_ID + if recorded not in self.id_map: + self.id_map[recorded] = self.id_offset + len(self.id_map) + return self.id_map[recorded] + + def _interpolate(self, data_structure_id, entity_set, following, weight, columns) -> None: + try: + later = following.data_structure(data_structure_id).entity_set(entity_set.name) + except KeyError: + return + later_rows = {identifier: row for row, identifier in enumerate(later.columns["id"].tolist())} + rows = np.array([later_rows.get(identifier, -1) for identifier in entity_set.columns["id"].tolist()]) + found = rows >= 0 + for name in entity_set.variable_names(): + if (data_structure_id, entity_set.name, name) not in self.linear or name not in later.columns: + continue + values = np.asarray(columns[name]) + if values.dtype.kind != "f": + continue + values = values.copy() + values[found] = (1 - weight) * values[found] + weight * np.asarray(later.columns[name])[rows[found]] + columns[name] = values diff --git a/bindings/python/src/openalea/fspm_tf/coupling/resolution.py b/bindings/python/src/openalea/fspm_tf/coupling/resolution.py new file mode 100644 index 0000000..6fd3dbc --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/resolution.py @@ -0,0 +1,169 @@ +""" +A session's resolution: from the participants' declarations and the links, each input's provider (Coupling), the +stages, and the checks that make a session run (one provider per input, convertible units, one owner per entity +set, no two participants changing one entity set's topology in the same stage). +""" +from dataclasses import dataclass +from typing import Optional + +from openalea.fspm_tf.coupling.messages import CouplingError, Declaration, Link, VariableSpec +from openalea.fspm_tf.ucum import UnitError, conversion_factor + + +@dataclass +class Coupling: + """An input and its provider: *factor* converts the source's unit to the input's (times the link's factor); + *aggregation* changes its scale (None: the same entity set); *how*: "link", "standard name" or "name".""" + receiver: str + variable: VariableSpec + provider: Optional[str] + source: Optional[VariableSpec] + factor: float = 1. + aggregation: Optional[str] = None + interpolation: str = "hold" + how: str = "link" + + def to_json(self) -> dict: + value = {"variable": self.variable.name, "entitySet": self.variable.entity_set, + "dataStructure": self.variable.data_structure, "provider": self.provider, "how": self.how} + if self.source is not None: + value.update(source=self.source.name, sourceEntitySet=self.source.entity_set, factor=self.factor) + if self.aggregation: + value["aggregation"] = self.aggregation + if self.interpolation != "hold": + value["interpolation"] = self.interpolation + return value + + +def ranks(declarations: dict) -> dict: + """{(data structure, entity set): rank}, the same in every declaration that gives one.""" + found = {} + for declaration in declarations.values(): + for entity_set in declaration.entity_sets: + key = (entity_set.data_structure, entity_set.name) + if entity_set.rank is None: + continue + if found.setdefault(key, entity_set.rank) != entity_set.rank: + raise CouplingError(f"entity set {key[0]}/{key[1]}: ranks {found[key]} and {entity_set.rank} declared") + return found + + +def resolve(declarations: dict, links=()) -> list: + """The couplings of every input of *declarations* ({participant: Declaration}), from *links* (Link) first, then + by standard name, then by name; CouplingError when an input has none (unless optional) or several.""" + known_ranks = ranks(declarations) + owners = {} + for declaration in declarations.values(): + for entity_set in declaration.entity_sets: + if entity_set.rights == "owns": + key = (entity_set.data_structure, entity_set.name) + if key in owners: + raise CouplingError(f"entity set {key[0]}/{key[1]}: owned by {owners[key]} and " + f"{declaration.participant}") + owners[key] = declaration.participant + by_receiver = {} + for link in links: + if link.receiver not in declarations: + raise CouplingError(f"link to {link.receiver}.{link.variable}: no participant {link.receiver}") + if link.provider not in declarations: + raise CouplingError(f"link to {link.receiver}.{link.variable}: no participant {link.provider}") + by_receiver.setdefault(link.receiver, []).append(link) + + couplings = [] + for receiver, declaration in declarations.items(): + for variable in declaration.requires(): + links_here = [link for link in by_receiver.get(receiver, []) if link.variable == variable.name + and link.entity_set in (None, variable.entity_set)] + if len(links_here) > 1: + raise CouplingError(f"{receiver}.{variable.name}: {len(links_here)} links") + if links_here: + link = links_here[0] + sources = [source for source in declarations[link.provider].provides() if source.name == link.source + and link.source_entity_set in (None, source.entity_set)] + if len(sources) != 1: + raise CouplingError(f"link {receiver}.{variable.name} <- {link.provider}.{link.source}: " + f"{len(sources)} such outputs of {link.provider}") + couplings.append(_coupling(receiver, variable, link.provider, sources[0], known_ranks, "link", + link.factor, link.aggregation, link.interpolation)) + continue + coupling = None + for how, same in (("standard name", lambda source: variable.standard_name is not None + and source.standard_name == variable.standard_name), + ("name", lambda source: source.name == variable.name)): + candidates = [(name, source) for name, other in declarations.items() if name != receiver + for source in other.provides() + if source.data_structure == variable.data_structure and same(source)] + if len(candidates) > 1: # the same entity set first + candidates = [(name, source) for name, source in candidates + if source.entity_set == variable.entity_set] or candidates + if len(candidates) > 1: + raise CouplingError(f"{receiver}.{variable.name}: provided by several outputs (by {how}): " + f"{[f'{name}.{source.name}' for name, source in candidates]}; link it") + if candidates: + coupling = _coupling(receiver, variable, *candidates[0], known_ranks, how) + break + if coupling is None: + if not variable.optional: + raise CouplingError(f"{receiver}.{variable.name}: no participant provides it (link it, or " + "declare it optional)") + coupling = Coupling(receiver, variable, None, None, how="not provided") + couplings.append(coupling) + return couplings + + +def _coupling(receiver, variable, provider, source, known_ranks, how, factor=1., aggregation=None, + interpolation="hold") -> Coupling: + if source.data_structure != variable.data_structure: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: different data structures " + f"({source.data_structure}, {variable.data_structure}) need a mapping (not supported yet)") + try: + factor = factor * conversion_factor(source.unit, variable.unit) + except UnitError as error: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: {error}") from None + if source.entity_set != variable.entity_set: + source_rank = known_ranks.get(source.where) + target_rank = known_ranks.get(variable.where) + if source_rank is None or target_rank is None: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: the ranks of " + f"{source.entity_set} and {variable.entity_set} are needed (declare them)") + if source_rank > target_rank: + aggregation = aggregation or ("sum" if source.kind == "extensive" else "mean") + if aggregation == "copy": + raise CouplingError(f"{receiver}.{variable.name}: 'copy' goes from a coarser entity set") + else: + if aggregation not in (None, "copy"): + raise CouplingError(f"{receiver}.{variable.name}: '{aggregation}' goes from a finer entity set") + if source.kind == "extensive" and aggregation is None: + raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: an extensive " + "variable copied to finer entities counts it several times; link it with " + "aggregation 'copy' if meant") + aggregation = "copy" + elif aggregation is not None: + raise CouplingError(f"{receiver}.{variable.name}: aggregation '{aggregation}' on the same entity set") + return Coupling(receiver, variable, provider, source, factor, aggregation, interpolation, how) + + +def stages(declarations: dict, given=None) -> list: + """The stages: *given* (lists of participant names), else the owners, then the refiners, then the others; and + the check that no stage holds two participants changing one entity set's topology.""" + if given is None: + groups = {"owns": [], "refines": [], "follows": []} + for name, declaration in declarations.items(): + rights = {entity_set.rights for entity_set in declaration.entity_sets} + groups["owns" if "owns" in rights else "refines" if "refines" in rights else "follows"].append(name) + given = [group for group in groups.values() if group] + given = [list(stage) for stage in given] + listed = [name for stage in given for name in stage] + if sorted(listed) != sorted(declarations): + raise CouplingError(f"stages {given}: each participant ({sorted(declarations)}) exactly once") + for stage in given: + changing = {} + for name in stage: + for entity_set in declarations[name].entity_sets: + if entity_set.rights != "follows": + key = (entity_set.data_structure, entity_set.name) + if key in changing: + raise CouplingError(f"stage {stage}: {changing[key]} and {name} both change " + f"{key[0]}/{key[1]}'s topology") + changing[key] = name + return given diff --git a/bindings/python/src/openalea/fspm_tf/coupling/store.py b/bindings/python/src/openalea/fspm_tf/coupling/store.py new file mode 100644 index 0000000..4be2245 --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/store.py @@ -0,0 +1,388 @@ +""" +The session's structure, kept by the coordinator: per data structure and entity set, the entities (their ids and +topology, and which participant created each), and the values each participant provided. Updates are merged into +it, checked against the participants' rights; each participant's inputs are read from it, converted. +""" +import numpy as np + +from openalea.fspm_tf.coupling.messages import CouplingError, ID_COUNT +from openalea.fspm_tf.model import (BASE_SCALES, NO_ID, DataStructure, EntitySet, Frame, Update, Variable) + +TOPOLOGY = ("label", "complex", "parent", "edge_type", "sibling_order") + + +class _Column: + """A provider's values on an entity set's rows, with the rows it has given a value (*present*).""" + + def __init__(self, values: np.ndarray, size: int): + values = np.asarray(values) + if values.dtype.kind in "OUS": + dtype = object + elif values.dtype.kind == "b": + dtype = bool + elif values.dtype.kind in "iu": + dtype = np.int64 + else: + dtype = np.float64 + width = values.shape[1:] if values.ndim > 1 else () + self.values = np.zeros((size,) + width, dtype=dtype) + if dtype == np.float64: + self.values[:] = np.nan + self.present = np.zeros(size, dtype=bool) + + def place(self, rows: np.ndarray, values) -> None: + mask = np.ma.getmaskarray(values) if np.ma.isMaskedArray(values) else None + data = np.ma.getdata(values) if mask is not None else np.asarray(values) + if mask is not None and mask.ndim > 1: + mask = mask.any(axis=tuple(range(1, mask.ndim))) + if self.values.dtype == np.int64 and data.dtype.kind == "f": + self.values = self.values.astype(np.float64) # an integer column now holding floats + self.values[rows] = data + self.present[rows] = True if mask is None else ~mask + + def keep(self, keep: np.ndarray) -> None: + self.values, self.present = self.values[keep], self.present[keep] + + def grow(self, count: int) -> None: + extra = np.zeros((count,) + self.values.shape[1:], dtype=self.values.dtype) + if self.values.dtype == np.float64: + extra[:] = np.nan + self.values = np.concatenate([self.values, extra]) + self.present = np.concatenate([self.present, np.zeros(count, dtype=bool)]) + + +class EntitySetStore: + """An entity set's entities (ids, topology, creators) and the values the participants provided on them.""" + + def __init__(self, data_structure: str, name: str, rank): + self.data_structure, self.name, self.rank = data_structure, name, rank + self.ids = np.zeros(0, dtype=np.int64) + self.row = {} + self.topology = {"label": np.zeros(0, dtype=object), "complex": np.zeros(0, dtype=np.int64), + "parent": np.zeros(0, dtype=np.int64), "edge_type": np.zeros(0, dtype=object), + "sibling_order": np.zeros(0, dtype=np.int64)} + self.creator = np.zeros(0, dtype=object) + self.columns = {} # (participant, variable): _Column + self.declarations = {} # (participant, variable): Variable + + def __len__(self): + return self.ids.size + + def rows(self, ids) -> np.ndarray: + return np.array([self.row[int(identifier)] for identifier in ids], dtype=np.int64) + + def add(self, ids: np.ndarray, topology: dict, creator: str) -> None: + count = ids.size + start = self.ids.size + self.ids = np.concatenate([self.ids, ids]) + for name, default in (("label", self.name), ("complex", NO_ID), ("parent", NO_ID), ("edge_type", "none"), + ("sibling_order", NO_ID)): # an entity without a label: its entity set's name + values = topology.get(name) + column = self.topology[name] + added = np.full(count, default, dtype=column.dtype) if values is None else np.asarray(values, + dtype=column.dtype) + self.topology[name] = np.concatenate([column, added]) + self.creator = np.concatenate([self.creator, np.full(count, creator, dtype=object)]) + for column in self.columns.values(): + column.grow(count) + self.row.update({int(identifier): start + offset for offset, identifier in enumerate(ids.tolist())}) + + def remove(self, ids) -> None: + keep = ~np.isin(self.ids, np.asarray(list(ids), dtype=np.int64)) + self.ids = self.ids[keep] + for name in self.topology: + self.topology[name] = self.topology[name][keep] + self.creator = self.creator[keep] + for column in self.columns.values(): + column.keep(keep) + self.row = {int(identifier): row for row, identifier in enumerate(self.ids.tolist())} + + def place(self, participant: str, name: str, rows: np.ndarray, values, variable) -> None: + key = (participant, name) + if key not in self.columns: + self.columns[key] = _Column(np.ma.getdata(values) if np.ma.isMaskedArray(values) else values, + self.ids.size) + if variable is not None: + self.declarations[key] = variable + self.columns[key].place(rows, values) + + +class SessionStore: + """The session's data structures (DataStructureStore: {entity set name: EntitySetStore}), which entity set each + id is in, and the log of removed entities (each participant is sent those removed since its last turn).""" + + def __init__(self, ranks: dict): + self.ranks = dict(ranks) # (data structure, entity set): rank + self.sets = {} # (data structure, entity set): EntitySetStore + self.set_of = {} # (data structure, id): entity set name + self.removed_log = [] # (data structure, entity set, ids) + self.removed_ids = set() # (data structure, id) of every entity removed + + def entity_set(self, data_structure: str, name: str, rank=None) -> EntitySetStore: + key = (data_structure, name) + if key not in self.sets: + rank = self.ranks.get(key, rank) + if rank is None and name in BASE_SCALES: + rank = BASE_SCALES.index(name) + 1 + self.ranks[key] = rank + self.sets[key] = EntitySetStore(data_structure, name, rank) + return self.sets[key] + + # ── merging a participant's update ───────────────────────────────────────────── + def merge(self, participant: str, declaration, frame: Frame, id_offset: int) -> None: + """Merge *participant*'s update: its new entities, topology changes and removals (within its rights), and its + outputs (only the variables it provides).""" + where = f"{participant}'s update" + provided = {(variable.data_structure, variable.entity_set, variable.name): variable + for variable in declaration.provides()} + for data_structure in frame.data_structures: + for entity_set in sorted(data_structure.entity_sets, + key=lambda item: (item.rank if item.rank is not None else 0)): + self._merge_set(participant, declaration, data_structure.id, entity_set, provided, id_offset, where) + if frame.update is not None: + for (data_structure, set_name), ids in sorted(frame.update.removed.items(), + key=lambda item: self.ranks.get(item[0]) or 0): + self._remove(participant, declaration, data_structure, set_name, ids, where) + + def _merge_set(self, participant, declaration, data_structure, entity_set, provided, id_offset, where): + key = (data_structure, entity_set.name) + store = self.entity_set(data_structure, entity_set.name, entity_set.rank) + if "id" not in entity_set.columns: + raise CouplingError(f"{where}: {data_structure}/{entity_set.name} has no 'id' column") + ids = np.asarray(entity_set.columns["id"], dtype=np.int64) + rights = declaration.rights(*key) + known = np.array([int(identifier) in store.row for identifier in ids.tolist()], dtype=bool) + new = ids[~known] + if new.size: + if rights == "follows": + raise CouplingError(f"{where}: {new.size} new entities in {data_structure}/{entity_set.name}, which " + f"it follows (declare 'owns' or 'refines' to add entities)") + outside = new[(new < id_offset) | (new >= id_offset + ID_COUNT)] + if outside.size: + raise CouplingError(f"{where}: ids {outside[:5].tolist()} outside its range [{id_offset}, " + f"{id_offset + ID_COUNT})") + elsewhere = [int(identifier) for identifier in new.tolist() if (data_structure, int(identifier)) + in self.set_of] + if elsewhere: + raise CouplingError(f"{where}: ids {elsewhere[:5]} already in another entity set") + topology = {name: np.asarray(entity_set.columns[name])[~known] for name in TOPOLOGY + if name in entity_set.columns} + store.add(new, topology, participant) + for identifier in new.tolist(): + self.set_of[(data_structure, identifier)] = entity_set.name + rows = store.rows(ids) + changes = [name for name in TOPOLOGY if name in entity_set.columns] + if known.any() and changes: # topology of existing entities: its own, or its set's + for name in changes: + given = np.asarray(entity_set.columns[name])[known] + current = store.topology[name][rows[known]] + differs = given != current + if differs.any(): + allowed = (rights == "owns") | (store.creator[rows[known]] == participant) + if (differs & ~allowed).any(): + raise CouplingError(f"{where}: changes the {name} of entities of {data_structure}/" + f"{entity_set.name} it did not create") + store.topology[name][rows[known][differs]] = given[differs].astype(store.topology[name].dtype) + for name in ("complex", "parent"): + if name in entity_set.columns: + values = np.asarray(entity_set.columns[name], dtype=np.int64) + missing = [int(value) for value in values.tolist() if value != NO_ID + and (data_structure, int(value)) not in self.set_of] + if missing: + raise CouplingError(f"{where}: {data_structure}/{entity_set.name}.{name} refers to unknown " + f"entities {missing[:5]}") + for name in entity_set.variable_names(): + variable = provided.get((data_structure, entity_set.name, name)) + if variable is None: + raise CouplingError(f"{where}: '{name}' on {data_structure}/{entity_set.name} is not one of its " + "declared outputs") + store.place(participant, name, rows, entity_set.columns[name], + Variable(unit=variable.unit, standard_name=variable.standard_name, kind=variable.kind)) + + def _remove(self, participant, declaration, data_structure, set_name, ids, where): + store = self.sets.get((data_structure, set_name)) + ids = [int(identifier) for identifier in np.asarray(ids).tolist() + if (data_structure, int(identifier)) not in self.removed_ids] # e.g. with their complex already + if not ids: + return + if store is None or any(identifier not in store.row for identifier in ids): + raise CouplingError(f"{where}: removes entities not in {data_structure}/{set_name}") + if declaration.rights(data_structure, set_name) != "owns": + created = store.creator[store.rows(ids)] + if (created != participant).any(): + raise CouplingError(f"{where}: removes entities of {data_structure}/{set_name} it did not create") + self.remove(data_structure, set_name, ids) + + def remove(self, data_structure: str, set_name: str, ids) -> None: + """Remove *ids* and, down the finer entity sets, the entities whose complex is removed.""" + removed = set(ids) + pending = [(set_name, removed)] + while pending: + name, identifiers = pending.pop() + store = self.sets[(data_structure, name)] + store.remove(identifiers) + self.removed_log.append((data_structure, name, np.array(sorted(identifiers), dtype=np.int64))) + for identifier in identifiers: + self.set_of.pop((data_structure, identifier), None) + self.removed_ids.add((data_structure, identifier)) + for (other_structure, other_name), other in self.sets.items(): + if other_structure != data_structure or not len(other): + continue + parent = other.topology["parent"] + orphans = np.isin(parent, list(identifiers)) + if orphans.any(): # successors of removed entities become roots + parent[orphans] = NO_ID + other.topology["edge_type"][orphans] = "none" + inside = other.ids[np.isin(other.topology["complex"], list(identifiers))] + if inside.size: + pending.append((other_name, set(inside.tolist()))) + + # ── a participant's inputs ───────────────────────────────────────────────────── + def ancestors(self, data_structure: str, ids: np.ndarray, target: str) -> np.ndarray: + """Each id's complex (or complex's complex, …) in the entity set *target*, NO_ID if none.""" + found = np.full(ids.size, NO_ID, dtype=np.int64) + for position, identifier in enumerate(ids.tolist()): + current = identifier + while current != NO_ID: + name = self.set_of.get((data_structure, current)) + if name is None: + break + if name == target: + found[position] = current + break + store = self.sets[(data_structure, name)] + current = int(store.topology["complex"][store.row[current]]) + return found + + def values(self, coupling) -> tuple: + """The input of *coupling* on its entity set's rows: (values, missing mask).""" + variable, source = coupling.variable, coupling.source + target = self.entity_set(variable.data_structure, variable.entity_set) + count = len(target) + origin = self.sets.get((source.data_structure, source.entity_set)) + column = None if origin is None else origin.columns.get((coupling.provider, source.name)) + if column is None: + return np.full(count, np.nan), np.ones(count, dtype=bool) + values, present = column.values, column.present + if coupling.aggregation is None: + rows = np.array([origin.row.get(int(identifier), -1) for identifier in target.ids.tolist()], + dtype=np.int64) + found = rows >= 0 + out = _empty(values, count) + out[found] = values[rows[found]] + missing = ~found + missing[found] |= ~present[rows[found]] + elif coupling.aggregation == "copy": + above = self.ancestors(variable.data_structure, target.ids, source.entity_set) + rows = np.array([origin.row.get(int(identifier), -1) for identifier in above.tolist()], dtype=np.int64) + found = rows >= 0 + out = _empty(values, count) + out[found] = values[rows[found]] + missing = ~found + missing[found] |= ~present[rows[found]] + else: + if values.dtype.kind not in "fi": + raise CouplingError(f"{coupling.receiver}.{variable.name}: {coupling.aggregation} of non-numeric " + f"values") + above = self.ancestors(variable.data_structure, origin.ids, variable.entity_set) + out = np.full(count, np.nan) + missing = np.ones(count, dtype=bool) + groups = {} + for row, identifier in enumerate(above.tolist()): + if identifier != NO_ID and present[row]: + groups.setdefault(identifier, []).append(values[row]) + reduce = {"sum": np.sum, "mean": np.mean, "max": np.max, "min": np.min}[coupling.aggregation] + for identifier, group in groups.items(): + row = target.row.get(identifier) + if row is not None: + out[row] = reduce(np.asarray(group, dtype=float)) + missing[row] = False + if coupling.factor != 1. and out.dtype.kind in "fi": + out = out.astype(np.float64) * coupling.factor + return out, missing + + def inputs(self, receiver: str, declaration, couplings, removed_since: int, time: float, time_unit: str, + step: int) -> Frame: + """*receiver*'s turn frame: the entity sets it works on (all their entities and topology) with its inputs, + and the entities removed since its last turn (the removed log from *removed_since*).""" + wanted = [(entity_set.data_structure, entity_set.name) for entity_set in declaration.entity_sets] + structures = {} + for key in wanted: + data_structure, name = key + store = self.sets.get(key) + if store is None: + continue + columns = {"id": store.ids.copy()} + for column in TOPOLOGY: + values = store.topology[column] + if column == "sibling_order" and not (values != NO_ID).any(): + continue + columns[column] = values.copy() + entity_set = EntitySet(name, columns=columns, rank=store.rank, + scale=name if name in BASE_SCALES and store.rank == BASE_SCALES.index(name) + 1 + else None) + for coupling in couplings: + if coupling.receiver != receiver or coupling.variable.where != key or coupling.source is None: + continue + values, missing = self.values(coupling) + entity_set.columns[coupling.variable.name] = _with_missing(values, missing) + entity_set.variables[coupling.variable.name] = Variable( + unit=coupling.variable.unit, standard_name=coupling.variable.standard_name, + kind=coupling.variable.kind, role="input") + structures.setdefault(data_structure, DataStructure(data_structure)).entity_sets.append(entity_set) + removed = {} + for data_structure, name, ids in self.removed_log[removed_since:]: + if (data_structure, name) in wanted: + previous = removed.get((data_structure, name), np.zeros(0, dtype=np.int64)) + removed[(data_structure, name)] = np.union1d(previous, ids) + return Frame(data_structures=list(structures.values()), time=time, time_unit=time_unit, step=step, + update=Update("coordinator", removed)) + + def frame(self, time: float, time_unit: str, step: int) -> Frame: + """The whole session as a frame (its record): every entity set, each participant's outputs (named + participant__variable when two participants provide one name).""" + structures = {} + for (data_structure, name), store in sorted(self.sets.items(), key=lambda item: (item[0][0], + item[1].rank or 0)): + columns = {"id": store.ids.copy()} + for column in TOPOLOGY: + if column == "sibling_order" and not (store.topology[column] != NO_ID).any(): + continue + columns[column] = store.topology[column].copy() + counts = {} + for _, variable in store.columns: + counts[variable] = counts.get(variable, 0) + 1 + entity_set = EntitySet(name, columns=columns, rank=store.rank, + scale=name if name in BASE_SCALES and store.rank == BASE_SCALES.index(name) + 1 + else None) + for (participant, variable), column in store.columns.items(): + column_name = variable if counts[variable] == 1 else f"{participant}__{variable}" + entity_set.columns[column_name] = _with_missing(column.values, ~column.present) + if (participant, variable) in store.declarations: + entity_set.variables[column_name] = store.declarations[(participant, variable)] + structures.setdefault(data_structure, DataStructure(data_structure)).entity_sets.append(entity_set) + return Frame(data_structures=list(structures.values()), time=time, time_unit=time_unit, step=step, + writer={"platform": "fspm-tf", "adapter": "openalea.fspm_tf.coupling"}) + + +def _empty(values: np.ndarray, count: int) -> np.ndarray: + out = np.zeros((count,) + values.shape[1:], dtype=values.dtype) + if values.dtype.kind == "f": + out[:] = np.nan + return out + + +def _with_missing(values: np.ndarray, missing: np.ndarray): + """The column of *values*, its *missing* rows as the format writes them: NaN, masked, None.""" + if not missing.any(): + return values + if values.dtype.kind == "f": + values = values.copy() + values[missing] = np.nan + return values + if values.dtype == object: + values = values.copy() + values[missing] = None + return values + mask = np.broadcast_to(missing.reshape((-1,) + (1,) * (values.ndim - 1)), values.shape) + return np.ma.MaskedArray(values, mask=mask) diff --git a/bindings/python/src/openalea/fspm_tf/model.py b/bindings/python/src/openalea/fspm_tf/model.py index b0a4b43..6dca612 100644 --- a/bindings/python/src/openalea/fspm_tf/model.py +++ b/bindings/python/src/openalea/fspm_tf/model.py @@ -9,6 +9,7 @@ Grid a regular or adaptive grid's layout Mesh triangles, each vertex carrying the row of its entity in one or several entity sets Mapping source entity ↔ target entity weights between two data structures + Update a partial frame's sender and removed entities (a coupling session's message) Every table holds columns of equal length (its rows), and declarations (Variable) for its variables. The columns reserved by the specification (RESERVED) are the structure: ids, the MTG relations, a relation's or a mapping's ends. @@ -203,10 +204,24 @@ class Mapping(Table): table_kind = "mapping" +@dataclass +class Update: + """What makes a frame partial: a coupling session's message (specification §11). *participant*: its sender + ("coordinator" for a turn's inputs). Its tables hold some rows and columns only: the sender's outputs and its + topology changes, entities referring to others the frame does not hold. *removed*: {(data structure id, entity + set name): int64 ids} of the entities removed.""" + participant: str + removed: dict = field(default_factory=dict) + + def __post_init__(self): + self.removed = {key: np.asarray(ids, dtype=np.int64).reshape(-1) for key, ids in self.removed.items()} + + @dataclass class Frame: """A scene at one time step (*time* in *time_unit*, *step*), or a parameter set (data structures holding only - parameters). *unknown*: content of a read file this package does not interpret, written back as it was.""" + parameters). *update*: set for a partial frame (Update), a coupling session's message. *unknown*: content of a + read file this package does not interpret, written back as it was.""" data_structures: list = field(default_factory=list) mappings: list = field(default_factory=list) time: float = 0. @@ -214,6 +229,7 @@ class Frame: step: int = 0 writer: dict = field(default_factory=dict) vocabulary_version: Optional[str] = None + update: Optional[Update] = None unknown: dict = field(default_factory=dict) def data_structure(self, id: str) -> DataStructure: diff --git a/bindings/python/src/openalea/fspm_tf/schema.py b/bindings/python/src/openalea/fspm_tf/schema.py index b9f3e8b..94f55d7 100644 --- a/bindings/python/src/openalea/fspm_tf/schema.py +++ b/bindings/python/src/openalea/fspm_tf/schema.py @@ -1,5 +1,5 @@ -"""The JSON Schemas of fspm-TF (spec/schema/): fspm_scene.schema.json (the FSPM_scene extension) and -index.schema.json (a sequence's scene.fspm.json).""" +"""The JSON Schemas of fspm-TF (spec/schema/): fspm_scene.schema.json (the FSPM_scene extension), +index.schema.json (a sequence's scene.fspm.json) and coupling.schema.json (a coupling session's messages).""" import json from functools import lru_cache diff --git a/bindings/python/src/openalea/fspm_tf/ucum.py b/bindings/python/src/openalea/fspm_tf/ucum.py index 0ad27b0..de44be6 100644 --- a/bindings/python/src/openalea/fspm_tf/ucum.py +++ b/bindings/python/src/openalea/fspm_tf/ucum.py @@ -1,10 +1,13 @@ """ Units as UCUM codes (https://ucum.org), the subset fspm-TF accepts: SI and common atoms, metric prefixes on the metric ones, products ('.'), quotients ('/'), integer exponents, parentheses, annotations ({...}, dimensionless) and -'1'. dimension(code) gives a unit's dimension, to check that two units are convertible:: +'1'. dimension(code) gives a unit's dimension, to check that two units are convertible, and conversion_factor the +factor from one to the other:: dimension("mm3/(s.MPa)") == dimension("m4.s.kg-1") True: the same physical dimension + conversion_factor("g", "kg") 0.001: a value in g times it is in kg """ +import math import re # Dimensions: length, mass, time, temperature, amount, luminous intensity, current @@ -31,6 +34,16 @@ def _dims(**exponents) -> tuple: "min": _dims(T=1), "h": _dims(T=1), "d": _dims(T=1), "wk": _dims(T=1), "mo": _dims(T=1), "a": _dims(T=1), "deg": NONE, "%": NONE, "[ppm]": NONE, "[ppb]": NONE, }) +# Each atom's magnitude in the coherent units m, g, s, K, mol, cd, A (g, not kg: prefixes apply to g) +MAGNITUDES = { + "N": 1e3, "Pa": 1e3, "J": 1e3, "W": 1e3, "V": 1e3, "Ohm": 1e3, "S": 1e-3, "F": 1e-3, "T": 1e3, "Wb": 1e3, + "L": 1e-3, "l": 1e-3, "bar": 1e8, "ar": 1e2, "t": 1e6, "eq": 1., "m[H2O]": 9.80665e6, "m[Hg]": 1.33322387415e8, + "min": 60., "h": 3600., "d": 86400., "wk": 604800., "mo": 2629800., "a": 31557600., "deg": math.pi / 180, + "%": 1e-2, "[ppm]": 1e-6, "[ppb]": 1e-9, +} +PREFIX_FACTORS = {"da": 1e1, "Y": 1e24, "Z": 1e21, "E": 1e18, "P": 1e15, "T": 1e12, "G": 1e9, "M": 1e6, "k": 1e3, + "h": 1e2, "d": 1e-1, "c": 1e-2, "m": 1e-3, "u": 1e-6, "n": 1e-9, "p": 1e-12, "f": 1e-15, + "a": 1e-18, "z": 1e-21, "y": 1e-24} SPECIAL = {"Cel"} # an offset scale: not allowed in products, quotients, powers PREFIXES = ("da", "Y", "Z", "E", "P", "T", "G", "M", "k", "h", "d", "c", "m", "u", "n", "p", "f", "a", "z", "y") @@ -42,18 +55,42 @@ class UnitError(ValueError): def _atom(text: str) -> tuple: - """(dimension, special) of a unit symbol, with its prefix.""" + """(dimension, special, magnitude) of a unit symbol, with its prefix.""" if text in ATOMS: - return ATOMS[text], text in SPECIAL + return ATOMS[text], text in SPECIAL, MAGNITUDES.get(text, 1.) for prefix in PREFIXES: if text.startswith(prefix) and text[len(prefix):] in METRIC_ATOMS: - return METRIC_ATOMS[text[len(prefix):]], text[len(prefix):] in SPECIAL + atom = text[len(prefix):] + return METRIC_ATOMS[atom], atom in SPECIAL, PREFIX_FACTORS[prefix] * MAGNITUDES.get(atom, 1.) raise UnitError(f"unknown unit symbol '{text}'") def dimension(code: str) -> tuple: """The dimension of the UCUM *code* (exponents of L, M, T, Θ, N, J, I); raises UnitError if not a unit fspm-TF accepts.""" + return _parse(code)[0] + + +def magnitude(code: str) -> float: + """The size of the unit *code* in the coherent units m, g, s, K, mol, cd, A (e.g. 'kPa': 1e6 g.m-1.s-2).""" + return _parse(code)[2] + + +def conversion_factor(code: str, other: str) -> float: + """The factor converting a value in *code* to *other* (value_in_other = value_in_code * factor); UnitError if + they are not convertible, or if one is Cel and the other not (an offset scale: no factor).""" + dims, special, size = _parse(code) + other_dims, other_special, other_size = _parse(other) + if dims != other_dims: + raise UnitError(f"'{code}' and '{other}' are not convertible (different dimensions)") + if special != other_special: + raise UnitError(f"'{code}' and '{other}': Cel is an offset scale, converted to K by adding 273.15, not by a " + "factor") + return size / other_size + + +def _parse(code: str) -> tuple: + """(dimension, special, magnitude) of the UCUM *code*.""" if not isinstance(code, str) or not code or code != code.strip() or " " in code: raise UnitError(f"'{code}' is not a UCUM code (no spaces: write 'mol/(m2.s)' or 'mol.m-2.s-1')") tokens = _TOKEN.findall(code) @@ -63,15 +100,16 @@ def dimension(code: str) -> tuple: def term(): nonlocal position - total, special_seen, count = [0] * 7, False, 0 + total, special_seen, count, size = [0] * 7, False, 0, 1. sign = 1 if position < len(tokens) and tokens[position] == "/": sign, position = -1, position + 1 while True: - dims, special = component() + dims, special, factor = component() count += 1 special_seen = special_seen or special total = [a + sign * b for a, b in zip(total, dims)] + size = size * factor if sign > 0 else size / factor if position < len(tokens) and tokens[position] in (".", "/"): sign = 1 if tokens[position] == "." else -1 position += 1 @@ -79,7 +117,7 @@ def term(): break if special_seen and count > 1: raise UnitError(f"'{code}': Cel (a temperature scale) cannot be combined with other units; use K") - return tuple(total), special_seen + return tuple(total), special_seen, size def component(): nonlocal position @@ -88,34 +126,35 @@ def component(): token = tokens[position] position += 1 if token == "(": - dims, special = term() + dims, special, factor = term() if position >= len(tokens) or tokens[position] != ")": raise UnitError(f"'{code}': unbalanced parenthesis") position += 1 elif token.startswith("{"): - return NONE, False + return NONE, False, 1. elif re.fullmatch(r"[+-]?\d+", token): if int(token) != 1 and not token.lstrip("+-").startswith("10"): raise UnitError(f"'{code}': numbers other than 1 are not units") - return NONE, False + return NONE, False, float(int(token)) elif token in (".", "/", ")"): raise UnitError(f"'{code}': unexpected '{token}'") else: - dims, special = _atom(token) + dims, special, factor = _atom(token) if position < len(tokens) and re.fullmatch(r"[+-]?\d+", tokens[position]): exponent = int(tokens[position]) position += 1 if special: raise UnitError(f"'{code}': Cel cannot have an exponent") dims = tuple(exponent * value for value in dims) + factor = factor ** exponent if position < len(tokens) and tokens[position].startswith("{"): position += 1 - return dims, special + return dims, special, factor - dims, _ = term() + parsed = term() if position != len(tokens): raise UnitError(f"'{code}': unexpected '{tokens[position]}'") - return dims + return parsed def is_valid(code: str) -> bool: diff --git a/bindings/python/src/openalea/fspm_tf/validate.py b/bindings/python/src/openalea/fspm_tf/validate.py index d99c666..72d6201 100644 --- a/bindings/python/src/openalea/fspm_tf/validate.py +++ b/bindings/python/src/openalea/fspm_tf/validate.py @@ -210,14 +210,16 @@ def _check_frame(frame, document: dict, report: Report) -> None: if not (data_structure.grid is not None and name == data_structure.grid.cells)] if explicit and sum(len(values) for values in explicit) != len(set().union(*explicit)): report.error(where, "ids are unique across the data structure's entity sets and relations") - _check_mtg(data_structure, sets, ids, report) + partial = frame.update is not None # a partial frame refers to entities it does not hold + _check_mtg(data_structure, sets, ids, report, partial=partial) for relation in data_structure.relations: for end, set_name in (("source", relation.source_set), ("target", relation.target_set)): if set_name not in ids: - report.error(f"{where}/{relation.name}", f"{end} set '{set_name}' does not exist") + if not partial: + report.error(f"{where}/{relation.name}", f"{end} set '{set_name}' does not exist") elif end not in relation.columns: report.error(f"{where}/{relation.name}", f"no '{end}' column") - else: + elif not partial: _check_references(f"{where}/{relation.name}/{end}", relation.columns[end], ids[set_name], report, allow_none=False) names = [relation.name for relation in data_structure.relations] @@ -248,7 +250,13 @@ def _check_frame(frame, document: dict, report: Report) -> None: data_structures = {data_structure.id: data_structure for data_structure in frame.data_structures} if len(data_structures) != len(frame.data_structures): report.error("dataStructures", "two data structures have the same id") + if frame.update is not None: + for (data_structure_id, set_name), removed in frame.update.removed.items(): + if (removed < 0).any(): + report.error(f"update/removed/{data_structure_id}/{set_name}", "ids are non-negative") for mapping in frame.mappings: + if frame.update is not None: + continue where = f"mapping {mapping.name}" for end, (data_structure_id, set_name) in (("source", mapping.source), ("target", mapping.target)): try: @@ -297,14 +305,17 @@ def _check_references(where, values, ids: set, report, allow_none: bool = True) report.error(where, f"{len(missing)} id(s) not in the referenced entity set, e.g. {missing[:5]}") -def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: - """complex points to an entity of a coarser entity set; parent and edge_type agree; parents form no cycle.""" +def _check_mtg(data_structure, sets: dict, ids: dict, report, partial: bool = False) -> None: + """complex points to an entity of a coarser entity set; parent and edge_type agree; parents form no cycle. In a + partial frame (*partial*), complex and parent may refer to entities it does not hold, and may be absent.""" ranked = sorted((entity_set for entity_set in data_structure.entity_sets if entity_set.rank is not None), key=lambda entity_set: entity_set.rank) for position, entity_set in enumerate(ranked): where = f"{data_structure.id}/{entity_set.name}" columns = entity_set.columns - if position > 0: + if partial: + pass + elif position > 0: coarser = ranked[position - 1] if "complex" not in columns: report.error(where, f"no 'complex' column (its entities belong to {coarser.name})") @@ -322,7 +333,8 @@ def _check_mtg(data_structure, sets: dict, ids: dict, report) -> None: if "parent" not in columns: continue parent, edge_type = columns["parent"], columns["edge_type"] - _check_references(f"{where}/parent", parent, ids[entity_set.name], report) + if not partial: + _check_references(f"{where}/parent", parent, ids[entity_set.name], report) unknown = set(edge_type.tolist()) - set(EDGE_TYPES) if unknown: report.error(f"{where}/edge_type", f"unknown values {sorted(unknown)}") diff --git a/bindings/python/test/test_coupling.py b/bindings/python/test/test_coupling.py new file mode 100644 index 0000000..f79555e --- /dev/null +++ b/bindings/python/test/test_coupling.py @@ -0,0 +1,268 @@ +""" +Coupling sessions (openalea.fspm_tf.coupling): two participants on one plant at two scales, a grower owning the +organs and a function refining them into segments and feeding a supply back, over ZeroMQ and over a folder; the +resolution's checks and the rights; a recorded session replayed as a participant. +""" +import os +import threading + +import numpy as np +import pytest + +from openalea import fspm_tf +from openalea.fspm_tf.coupling import (Coordinator, CouplingError, Declaration, EntitySetSpec, Link, Participant, + VariableSpec, resolve, stages) +from openalea.fspm_tf.model import NO_ID +from openalea.fspm_tf.validate import validate_sequence + +pytest.importorskip("zmq") + +# The command running Julia with FspmTF and ZMQ (e.g. "julia --project=env"): the tests with a Julia participant +JULIA = os.environ.get("FSPM_TF_JULIA") +JULIA_FUNCTION = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "julia", "FspmTF", "test", "coupling", + "function.jl") + +# The session's links: the segments' length is half their organ's (copied from the coarser organs); the organs' +# supply is their segments' sugar (summed, mol to umol) +LINKS = [Link("function", "segment_length", "grower", "length", factor=0.5, aggregation="copy"), + Link("grower", "supply", "function", "sugar")] + + +def grower_declaration(name="grower"): + return Declaration(name, "test", [EntitySetSpec("plant", "Organ", rights="owns")], [ + VariableSpec("plant", "Organ", "length", "provides", "cm", "length", "extensive"), + VariableSpec("plant", "Organ", "supply", "requires", "umol", kind="extensive")]) + + +def function_declaration(name="function", rights="refines"): + return Declaration(name, "test", [EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights=rights)], [ + VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol", kind="extensive")]) + + +def organ_frame(name, organs, removed=None): + """The grower's update: its organs in a chain, each 2 cm longer than the previous one.""" + count = len(organs) + columns = {"id": np.array(organs, dtype=np.int64), "length": 2. * np.arange(1, count + 1), + "parent": np.array([NO_ID] + organs[:-1], dtype=np.int64), + "edge_type": np.array(["none"] + ["successor"] * (count - 1), dtype=object), + "label": np.array(["Leaf"] * count, dtype=object)} + return fspm_tf.Frame(data_structures=[fspm_tf.DataStructure("plant", entity_sets=[ + fspm_tf.EntitySet("Organ", rank=5, columns=columns)])], update=fspm_tf.Update(name, removed or {})) + + +def run_grower(endpoint, log): + """One organ more each step; the first one removed at step 3. The supply it receives is logged.""" + participant = Participant.connect(grower_declaration(), endpoint, timeout=300) + organs = [] + for turn in participant.turns(): + organ_set = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + if organ_set is not None and organ_set.count: + log[turn.step] = dict(zip(organ_set.columns["id"].tolist(), + np.asarray(organ_set.columns["supply"]).tolist())) + organs.append(participant.id_offset + turn.step) + removed = {} + if turn.step == 3: + removed = {("plant", "Organ"): [organs.pop(0)]} + participant.send(organ_frame("grower", organs, removed)) + + +def run_function(endpoint, log, rights="refines"): + """Two segments under each new organ; each segment's sugar 1e-6 mol per m of its length.""" + participant = Participant.connect(function_declaration(rights=rights), endpoint, timeout=30) + segments = {} # segment id: (its organ, its parent) + for turn in participant.turns(): + plant = turn.frame.data_structure("plant") + removed = turn.frame.update.removed.get(("plant", "SubOrgan"), np.zeros(0)).tolist() + log[turn.step] = sorted(removed) + for segment in removed: + segments.pop(segment, None) # removed with their organ by the coordinator + lengths = {} + try: + known = plant.entity_set("SubOrgan") + lengths = dict(zip(known.columns["id"].tolist(), np.asarray(known.columns["segment_length"]).tolist())) + except KeyError: + pass + with_segments = {organ for organ, _ in segments.values()} + for organ in plant.entity_set("Organ").columns["id"].tolist(): + if organ not in with_segments: + first = participant.id_offset + len(segments) + 2 * turn.step * 1000 + segments[first], segments[first + 1] = (organ, NO_ID), (organ, first) + ids = list(segments) + columns = {"id": np.array(ids, dtype=np.int64), + "complex": np.array([segments[segment][0] for segment in ids], dtype=np.int64), + "parent": np.array([segments[segment][1] for segment in ids], dtype=np.int64), + "edge_type": np.array(["none" if segments[segment][1] == NO_ID else "successor" + for segment in ids], dtype=object), + "sugar": np.array([1e-6 * np.nan_to_num(lengths.get(segment, 0.)) for segment in ids])} + participant.send(fspm_tf.Frame(data_structures=[fspm_tf.DataStructure("plant", entity_sets=[ + fspm_tf.EntitySet("SubOrgan", rank=6, columns=columns)])])) + + +def run_session(endpoint, record=None, function=run_function, **options): + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["grower", "function"], links=LINKS, + record=record, timeout=30, **options) + logs = {"grower": {}, "function": {}} + threads = [threading.Thread(target=target, args=(coordinator.endpoint, logs[name]), daemon=True) + for name, target in (("grower", run_grower), ("function", function))] + for thread in threads: + thread.start() + try: + store = coordinator.run() + finally: + for thread in threads: + thread.join(30) + return store, logs + + +@pytest.fixture(params=["zmq", "folder"]) +def endpoint(request, tmp_path): + return "tcp://127.0.0.1:*" if request.param == "zmq" else f"files:{tmp_path / 'session'}" + + +def check_session(store, supply, removed, record): + first, second, third = (2 ** 32 + step for step in range(3)) + # a segment's length: half its organ's (0.5 * 0.01 m/cm); its sugar 1e-6 mol/m of it; summed and in umol: + # an organ's supply is 0.01 umol per cm of the organ, from the step after its segments were made + assert supply[1] == pytest.approx({first: 0.}) + assert supply[2] == pytest.approx({first: 0.02, second: 0.}) + assert supply[3] == pytest.approx({first: 0.02, second: 0.04, third: 0.}) + assert set(supply[4]) == {second, third, third + 1} # the first organ removed at step 3 + assert supply[4][second] == pytest.approx(0.02) # now the first of the chain: 2 cm + if removed is not None: + assert len(removed[3]) == 2 # its segments, removed with it in the step + assert removed[4] == [] + organs = store.sets[("plant", "Organ")] + assert sorted(organs.ids.tolist()) == [second, third, third + 1, third + 2] + assert len(store.sets[("plant", "SubOrgan")]) == 8 + report = validate_sequence(os.path.join(record, "scene.fspm.json")) + assert report.ok, report + last = fspm_tf.read(os.path.join(record, "frame_00004.glb")) + assert set(last.data_structure("plant").entity_set("SubOrgan").columns) >= {"id", "complex", "sugar"} + + +def test_two_platforms_on_one_plant_at_two_scales(endpoint, tmp_path): + store, logs = run_session(endpoint, record=str(tmp_path / "record")) + check_session(store, logs["grower"], logs["function"], str(tmp_path / "record")) + + +@pytest.mark.skipif(not JULIA, reason="FSPM_TF_JULIA (the command running Julia with FspmTF and ZMQ) is not set") +def test_a_julia_participant(endpoint, tmp_path): + """The function participant in Julia (FspmTF.jl), launched by the coordinator.""" + record = str(tmp_path / "record") + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["grower"], links=LINKS, record=record, + timeout=300) + coordinator.launch("function", f"{JULIA} {JULIA_FUNCTION}") + log = {} + thread = threading.Thread(target=run_grower, args=(coordinator.endpoint, log), daemon=True) + thread.start() + store = coordinator.run() + thread.join(30) + check_session(store, log, None, record) + + +def test_the_messages_follow_their_schema(tmp_path): + """Every message of a session (the folder transport's log: both directions) is valid against + spec/schema/coupling.schema.json.""" + import glob + import json + import jsonschema + from openalea.fspm_tf.schema import load + run_session(f"files:{tmp_path / 'session'}") + validator = jsonschema.Draft202012Validator(load("coupling")) + headers = sorted(glob.glob(str(tmp_path / "session" / "*" / "*" / "*.json"))) + types = set() + for path in headers: + with open(path, encoding="utf-8") as file: + header = json.load(file) + errors = list(validator.iter_errors(header)) + assert not errors, f"{path}: {errors[0].message}" + types.add(header["type"]) + assert types == {"hello", "welcome", "turn", "update", "stop"} + + +def run_follower_adding(endpoint, log): + run_function(endpoint, log, rights="follows") + + +def test_a_participant_adding_entities_it_follows_stops_the_session(tmp_path): + with pytest.raises(CouplingError, match="which it follows"): + run_session("tcp://127.0.0.1:*", function=run_follower_adding) + + +def test_resolution(): + declarations = {"grower": grower_declaration(), "function": function_declaration()} + with pytest.raises(CouplingError, match="extensive variable copied"): + resolve(declarations, [LINKS[1]]) # length by standard name, copied to segments + with pytest.raises(CouplingError, match="no participant provides"): + resolve(declarations, [LINKS[0]]) # the supply unlinked + couplings = {coupling.variable.name: coupling for coupling in resolve(declarations, LINKS)} + assert couplings["segment_length"].factor == pytest.approx(0.005) + assert couplings["supply"].aggregation == "sum" and couplings["supply"].factor == pytest.approx(1e6) + assert stages(declarations) == [["grower"], ["function"]] + with pytest.raises(CouplingError, match="owned by"): + resolve({"a": grower_declaration("a"), "b": grower_declaration("b")}) + with pytest.raises(CouplingError, match="not convertible"): + resolve(declarations, [LINKS[0], Link("grower", "supply", "grower", "length")]) + with pytest.raises(CouplingError, match="both change"): + stages({"a": function_declaration("a"), "b": function_declaration("b")}, [["a", "b"]]) + + +def run_reader(endpoint, log): + declaration = Declaration("reader", "test", [EntitySetSpec("plant", "Organ")], [ + VariableSpec("plant", "Organ", "length", "requires", "cm", "length")]) + participant = Participant.connect(declaration, endpoint, timeout=30) + for turn in participant.turns(): + organs = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + log[turn.until] = [] if organs is None else np.asarray(organs.columns["length"]).tolist() + participant.send(fspm_tf.Frame()) + + +def test_a_recorded_session_replayed(tmp_path): + run_session("tcp://127.0.0.1:*", record=str(tmp_path / "record")) + # the recorded organ lengths: 0 d [2], 1 d [2, 4], 2 d [2, 4, 6], 3 d [2, 4, 6] (the first removed, the chain + # renumbered), 4 d [2, 4, 6, 8] + for interpolation, expected in (("hold", [2., 4., 6.]), ("linear", [2., 3., 5.])): + coordinator = Coordinator("tcp://127.0.0.1:*", steps=8, interval=0.5, expect=["reader"], timeout=30, + replays={"recorded": str(tmp_path / "record" / "scene.fspm.json")}, + links=[Link("reader", "length", "recorded", "length", interpolation=interpolation)]) + log = {} + thread = threading.Thread(target=run_reader, args=(coordinator.endpoint, log), daemon=True) + thread.start() + coordinator.run() + thread.join(30) + assert log[0.] == [2.] + assert log[1.5] == pytest.approx([2., 4.]) + assert log[2.5] == pytest.approx(expected) # between 2 d and 3 d + assert log[3.] == pytest.approx([2., 4., 6.]) # after the removal + assert len(log[4.]) == 4 + + +READER_SCRIPT = """ +import numpy as np +from openalea import fspm_tf +from openalea.fspm_tf.coupling import Declaration, EntitySetSpec, Participant, VariableSpec +declaration = Declaration("", "test", [EntitySetSpec("plant", "Organ")], + [VariableSpec("plant", "Organ", "length", "requires", "m", "length")]) +participant = Participant.connect(declaration, timeout=60) +for turn in participant.turns(): + participant.send(fspm_tf.Frame()) +""" + + +def test_the_command_line(tmp_path): + """fspm-tf couple: a launched participant (its name and endpoint from the environment) and a replay, over a + folder, recorded.""" + import sys + from openalea.fspm_tf.cli import main + run_session("tcp://127.0.0.1:*", record=str(tmp_path / "record")) + script = tmp_path / "reader.py" + script.write_text(READER_SCRIPT) + code = main(["couple", "--steps", "3", "--interval", "1", "--endpoint", f"files:{tmp_path / 'session'}", + "--replay", f"recorded={tmp_path / 'record' / 'scene.fspm.json'}", + "--participant", f"reader={sys.executable} {script}", "--record", str(tmp_path / "again"), + "--timeout", "60"]) + assert code == 0 + assert validate_sequence(str(tmp_path / "again" / "scene.fspm.json")).ok + assert (tmp_path / "session" / "reader" / "to_coordinator" / "00000000.json").exists() # the session's log diff --git a/doc/api.md b/doc/api.md index fc35111..47ecd88 100644 --- a/doc/api.md +++ b/doc/api.md @@ -14,7 +14,7 @@ the vocabulary and units. ```{eval-rst} .. automodule:: openalea.fspm_tf.model - :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable + :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable, Update ``` ## Validation @@ -38,7 +38,7 @@ the vocabulary and units. :members: standard_names, platform_names, labels, enums, version .. automodule:: openalea.fspm_tf.ucum - :members: dimension, is_valid, convertible + :members: dimension, is_valid, convertible, conversion_factor ``` ## Comparing with expected values @@ -55,6 +55,25 @@ the vocabulary and units. :members: frame_to_dict ``` +## Coupling + +```{eval-rst} +.. automodule:: openalea.fspm_tf.coupling.messages + :members: Declaration, EntitySetSpec, VariableSpec, Link, CouplingError + +.. automodule:: openalea.fspm_tf.coupling.participant + :members: Participant, Turn + +.. automodule:: openalea.fspm_tf.coupling.coordinator + :members: Coordinator + +.. automodule:: openalea.fspm_tf.coupling.resolution + :members: resolve, stages, Coupling + +.. automodule:: openalea.fspm_tf.coupling.replay + :members: Replay +``` + ## Command line ```{eval-rst} diff --git a/doc/coupling.md b/doc/coupling.md new file mode 100644 index 0000000..f68dba3 --- /dev/null +++ b/doc/coupling.md @@ -0,0 +1,102 @@ +# Coupling platforms + +fspm-TF also runs models of several platforms together, one time step after the other. This is a **session**: + +- each platform's model is a **participant**, running in its own process and its own language; +- a **coordinator** (fspm-TF's, in Python) starts them, passes each one the values it needs from the others, and + keeps the plant they share. + +Two platforms can work on the same plant: one grows the organs, another splits them into segments and computes a +function on them, a third reads the plant from a recorded run. Each one only says what it works on, what it +provides and what it needs; the coordinator does the rest. + +The rules are in the specification's §11 (a draft for version 0.3). + +## What a participant says + +A participant **declares**: + +- the entity sets it works on (`Organ`, `SubOrgan`, …) and what it may do with their entities: `owns` (it creates + and removes them), `refines` (it adds its own entities under others', e.g. segments under organs), `follows` (it + reads them); +- the variables it **provides** (its outputs) and **requires** (its inputs), each with its unit and, if the + vocabulary has one, its standard name. + +A **link** says which output feeds which input, when the names differ: a factor, and a change of scale (a sum or a +mean of finer entities, a copy of a coarser one's value). Without a link, an input is fed by the output with the +same standard name, else the same name. Units are converted. + +## A session in Python + +A participant needs the fspm-TF codec, and nothing else of the other platforms: + +```python +from openalea import fspm_tf +from openalea.fspm_tf.coupling import Declaration, EntitySetSpec, Participant, VariableSpec + +declaration = Declaration("function", entity_sets=[EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights="refines")], + variables=[VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol")]) +participant = Participant.connect(declaration) # the session's address and its name come from the coordinator +for turn in participant.turns(): # until the coordinator stops the session + plant = turn.frame.data_structure("plant") # the organs and segments, the segments' lengths + ... # the model, from turn.time to turn.until + participant.send(frame) # its new segments and their sugar +``` + +Each turn is a frame holding the participant's entity sets and inputs, and the answer is a frame holding its new +entities and outputs (only those: the coordinator keeps the rest). The entities it creates take ids from +`participant.id_offset`. + +## A session in Julia + +The same participant in Julia (FspmTF.jl; ZeroMQ needs `using ZMQ`): + +```julia +using FspmTF, ZMQ + +participant = Participant(Declaration("function"; entity_sets = [...], variables = [...])) +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + # the model + send_update(participant, frame) +end +``` + +## Running it + +The coordinator starts each participant's command, and waits for the ones started otherwise (`--expect`): + +```bash +fspm-tf couple --steps 30 --interval 1 --time-unit d \ + --participant growth="julia --project=vpl growth.jl" \ + --participant function="python function.py" \ + --links links.json --record run/ +``` + +- `--links`: a JSON list of links, e.g. + `[{"receiver": "growth", "variable": "supply", "provider": "function", "source": "sugar"}]`. +- `--record`: the whole session written as an ordinary sequence of frames, to read in ParaView or to replay. +- `--replay name=run/scene.fspm.json`: a recorded run takes part as a participant, without its platform running. + A platform's routine log can thus feed another platform's model. +- `--endpoint files:folder`: the participants exchange files in a folder instead of ZeroMQ messages. It is slower, + but needs nothing but a file system, and the folder keeps every message. +- `--stages "growth;function"`: who runs first in each step (by default, the participants creating entities, then + the others). Each step, a stage sees what the previous stages did in that step. + +In Python, `openalea.fspm_tf.coupling.Coordinator` does the same from a script. + +## In metafspm + +metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for another platform's model inside a metafspm +model: a component computing nothing, whose state variables are the other model's outputs and whose inputs are sent +to it. Its tests couple metafspm's root growth to a carbon model written in VirtualPlantLab. + +## How it behaves + +- The coupling is explicit: a participant's inputs at a step are what the others gave at their previous turn. +- Entities removed by their owner are removed, with their finer entities, from every participant that follows them. +- A participant receiving new entities gives its own variables on them their initial values, and shares its own + quantities when an entity splits, by its own rules. +- A participant's error stops the session; every participant is told why. diff --git a/doc/index.md b/doc/index.md index 144890c..fb35514 100644 --- a/doc/index.md +++ b/doc/index.md @@ -36,6 +36,7 @@ A notebook: fspm-TF in Python Reading and writing in Julia Reading and writing in C++, and CPlantBox Reading and writing in Java, and GroIMP +Coupling platforms ``` ## Implementing fspm-TF diff --git a/doc/julia.md b/doc/julia.md index ced4c31..935c4c9 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -121,6 +121,13 @@ write_sequence("frames", frames) # one GLB per frame, and scene.fs A column is a `Vector` (numbers, `Bool`, `String`) or a `Matrix` (one row per entity: a fixed-size vector per entity). NaN and infinities are written with the specification's stand-ins and read back as they were. +## Coupling with other platforms + +FspmTF.jl takes part in coupling sessions run by fspm-TF's coordinator ([Coupling platforms](coupling.md)): +`Participant(declaration)` joins one (its address and name from the environment when the coordinator launches it), +`turns(participant)` gives each turn's inputs, `send_update(participant, frame)` answers with its outputs. The folder +transport needs nothing more; ZeroMQ needs `using ZMQ` (the `FspmTFZMQExt` extension). + ## Checking it ```bash diff --git a/doc/platforms.md b/doc/platforms.md index 9a9a32f..2c79e9f 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -99,3 +99,21 @@ after the codec's own steps: The Python codec does exactly this (`bindings/python/conformance_run.py`, the `python-codec-conformance` job). A codec that passes is listed in [Implementations](implementations.md) with the version it passes. + +## Taking part in a coupling session + +To couple its models with other platforms' (specification §11, draft for 0.3), a codec adds a participant's side; +the coordinator is fspm-TF's own (`fspm-tf couple`), so no platform implements it. A participant: + +1. reads and writes **partial frames**: `FSPM_scene.update` (its sender, and a property table of the removed ids per + entity set), tables holding some rows and columns only; +2. sends its **declaration** and receives the welcome (its id offset, its inputs' providers), as JSON headers + (`spec/schema/coupling.schema.json`); +3. answers each **turn** (a partial frame of its inputs and the entities added or removed since its last turn) with + an **update** (its outputs and its own topology changes), until `stop`; +4. speaks both transports: a folder of numbered JSON headers and GLB payloads (files only), and ZeroMQ (a DEALER + socket named after the participant; one or two message frames). + +The Python package (`openalea.fspm_tf.coupling.participant`) and FspmTF.jl (`Participant`, `turns`, `send_update`) +are the two participants so far; the Python tests run FspmTF.jl's against the coordinator over both transports. An +adapter then maps the turns to the platform's model, as metafspm's `RemoteComponent` does. diff --git a/implementations.md b/implementations.md index 9f09982..eea029e 100644 --- a/implementations.md +++ b/implementations.md @@ -12,6 +12,10 @@ | MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.2 | | VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.2 | +Coupling sessions (specification §11, draft for 0.3): the Python package holds the coordinator and a participant; +FspmTF.jl a participant (both transports); metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for +another platform's model in a metafspm model, tested with a VirtualPlantLab model (PlantGraphs). + Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters (a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional extension when the platform's library is a plain dependency (Julia's package extensions). An implementation is listed with diff --git a/pyproject.toml b/pyproject.toml index 4a25d11..a537419 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta" include-package-data = false # force explicit declaration of data (disable automatic inclusion) # the code in bindings/python/src; the specification (spec/: its text, schemas and vocabulary) shipped as # openalea.fspm_tf.spec. This file stays at the root, next to spec/ (a project cannot ship files outside its folder) -packages = ["openalea.fspm_tf", "openalea.fspm_tf.spec"] +packages = ["openalea.fspm_tf", "openalea.fspm_tf.coupling", "openalea.fspm_tf.spec"] package-dir = {"" = "bindings/python/src", "openalea.fspm_tf.spec" = "spec"} [tool.setuptools.package-data] @@ -52,8 +52,12 @@ dependencies = [ ] [project.optional-dependencies] +coupling = [ + "pyzmq", # the coupling's in-memory transport (openalea.fspm_tf.coupling); its folder transport needs nothing +] test = [ "pytest", + "pyzmq", ] dev = [ "pytest >=6", diff --git a/spec/schema/coupling.schema.json b/spec/schema/coupling.schema.json new file mode 100644 index 0000000..3162ae9 --- /dev/null +++ b/spec/schema/coupling.schema.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openalea/fspm-TF/spec/schema/coupling.schema.json", + "title": "fspm-TF coupling messages", + "description": "The JSON header of a coupling session's message (specification §11, draft for 0.3). Turns and updates also carry a payload: a partial frame's GLB bytes.", + "oneOf": [ + {"$ref": "#/$defs/hello"}, {"$ref": "#/$defs/welcome"}, {"$ref": "#/$defs/turn"}, {"$ref": "#/$defs/update"}, + {"$ref": "#/$defs/error"}, {"$ref": "#/$defs/stop"} + ], + "$defs": { + "identifier": {"type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$"}, + "name": {"type": "string", "minLength": 1}, + "payload": {"description": "In the folder transport: the payload's file name.", "type": "string"}, + "entitySet": { + "type": "object", "required": ["dataStructure", "name", "rights"], + "properties": { + "dataStructure": {"$ref": "#/$defs/identifier"}, "name": {"$ref": "#/$defs/name"}, + "rank": {"type": ["integer", "null"]}, "rights": {"enum": ["owns", "refines", "follows"]} + }, + "additionalProperties": false + }, + "variable": { + "type": "object", "required": ["dataStructure", "entitySet", "name", "direction", "unit"], + "properties": { + "dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"$ref": "#/$defs/name"}, + "name": {"$ref": "#/$defs/name"}, "direction": {"enum": ["provides", "requires"]}, "unit": {"type": "string"}, + "standardName": {"type": "string"}, "kind": {"type": "string"}, "optional": {"type": "boolean"} + }, + "additionalProperties": false + }, + "declaration": { + "type": "object", "required": ["participant", "entitySets", "variables"], + "properties": { + "participant": {"$ref": "#/$defs/identifier"}, "platform": {"type": "string"}, + "entitySets": {"type": "array", "items": {"$ref": "#/$defs/entitySet"}}, + "variables": {"type": "array", "items": {"$ref": "#/$defs/variable"}} + }, + "additionalProperties": false + }, + "link": { + "type": "object", "required": ["receiver", "variable", "provider", "source"], + "properties": { + "receiver": {"$ref": "#/$defs/identifier"}, "variable": {"$ref": "#/$defs/name"}, + "provider": {"$ref": "#/$defs/identifier"}, "source": {"$ref": "#/$defs/name"}, + "entitySet": {"$ref": "#/$defs/name"}, "sourceEntitySet": {"$ref": "#/$defs/name"}, + "factor": {"type": "number"}, "aggregation": {"enum": ["sum", "mean", "max", "min", "copy"]}, + "interpolation": {"enum": ["hold", "linear"]} + }, + "additionalProperties": false + }, + "hello": { + "type": "object", "required": ["type", "protocol", "declaration"], + "properties": { + "type": {"const": "hello"}, "protocol": {"type": "string"}, "declaration": {"$ref": "#/$defs/declaration"}, + "links": {"type": "array", "items": {"$ref": "#/$defs/link"}}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "welcome": { + "type": "object", "required": ["type", "idOffset", "idCount", "session"], + "properties": { + "type": {"const": "welcome"}, "idOffset": {"type": "integer", "minimum": 0}, + "idCount": {"type": "integer", "minimum": 1}, + "session": { + "type": "object", "required": ["start", "interval", "timeUnit", "steps", "stages", "participants"], + "properties": { + "start": {"type": "number"}, "interval": {"type": "number", "exclusiveMinimum": 0}, + "timeUnit": {"type": "string"}, "steps": {"type": "integer", "minimum": 0}, + "stages": {"type": "array", "items": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}}, + "participants": {"type": "array", "items": {"$ref": "#/$defs/identifier"}} + } + }, + "couplings": {"type": "array", "items": {"type": "object"}}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "turn": { + "type": "object", "required": ["type", "step", "phase", "time", "until"], + "properties": { + "type": {"const": "turn"}, "step": {"type": "integer", "minimum": 0}, "phase": {"enum": ["init", "step"]}, + "time": {"type": "number"}, "until": {"type": "number"}, "payload": {"$ref": "#/$defs/payload"} + }, + "additionalProperties": false + }, + "update": { + "type": "object", "required": ["type", "step"], + "properties": {"type": {"const": "update"}, "step": {"type": "integer", "minimum": 0}, + "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + }, + "error": { + "type": "object", "required": ["type", "message"], + "properties": {"type": {"const": "error"}, "message": {"type": "string"}, "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + }, + "stop": { + "type": "object", "required": ["type"], + "properties": {"type": {"const": "stop"}, "reason": {"type": "string"}, "payload": {"$ref": "#/$defs/payload"}}, + "additionalProperties": false + } + } +} diff --git a/spec/schema/fspm_scene.schema.json b/spec/schema/fspm_scene.schema.json index d570ca9..b0bccda 100644 --- a/spec/schema/fspm_scene.schema.json +++ b/spec/schema/fspm_scene.schema.json @@ -14,6 +14,19 @@ "step": {"type": "integer", "minimum": 0}}, "additionalProperties": false}, "writer": {"type": "object"}, + "update": { + "description": "Set in a partial frame, a coupling session's message (specification §11): its sender, and the entities it removed.", + "type": "object", "required": ["participant"], + "properties": { + "participant": {"$ref": "#/$defs/identifier"}, + "removed": {"type": "array", "items": { + "type": "object", "required": ["dataStructure", "entitySet", "propertyTable"], + "properties": {"dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"type": "string", "minLength": 1}, + "propertyTable": {"$ref": "#/$defs/index"}}, + "additionalProperties": false}} + }, + "additionalProperties": false + }, "dataStructures": {"type": "array", "items": {"$ref": "#/$defs/dataStructure"}}, "mappings": {"type": "array", "items": {"$ref": "#/$defs/mapping"}}, "variables": { diff --git a/spec/specification.md b/spec/specification.md index 3872945..d99eb13 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -16,6 +16,7 @@ The language-neutral files of this version are part of the specification: | `spec/vocabulary/labels.csv` | entity labels, their scales and Plant Ontology terms (§6) | | `spec/vocabulary/enums.json` | the values of `edge_type`, `kind`, `role`, data structure and grid kinds; the base scales | | `conformance/` | reference files and their expected values (§9): the specification as test cases | +| `spec/schema/coupling.schema.json` | JSON Schema of a coupling session's messages (§11, draft for 0.3) | When these sources disagree, `specification.md` prevails, then the schemas, then the conformance files; the disagreement is a defect of the lower one, to be fixed. @@ -341,6 +342,134 @@ An implementation conforms to 0.2 when two things hold: A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.2 this covers the glTF root's other extensions and `extras`, when they hold no buffer views. +## 11. Coupling (draft for 0.3) + +Platforms couple their models through fspm-TF in a **session**: each **participant** is a platform's model running +in its own process, or a recorded sequence (§11.6), and a **coordinator** runs the session. The coordinator holds the +session's shared structure (the entities of every data structure and the values each participant provides), applies +the links between them and sends each participant its inputs. A participant only declares itself, then answers each +of its turns with its outputs. Nothing about a given coupling is fixed in advance: declarations and links are passed +when the session starts. + +### 11.1 Partial frames + +Turns and updates are **partial frames**: frames whose `FSPM_scene` has an `update` object. + +- `update.participant` is the sender's name, or `"coordinator"`. +- `update.removed` lists, per data structure and entity set, a property table of one `id` column: the entities + removed (by the sender, or since the receiver's last turn). +- Its entity sets hold some rows and some columns only. A row is identified by its `id`. +- `complex` and `parent` MAY refer to entities the frame does not hold, and MAY be absent. Relations and mappings MAY + likewise refer to absent entities. +- Every other rule of a frame holds. The validator checks partial frames with the reference checks lifted. + +### 11.2 Messages + +A message is a JSON header and, for a turn or an update, a payload: a partial frame's GLB bytes. + +| Message | From | Header | Payload | +|---|---|---|---| +| hello | participant | `type`, `protocol` (`"0.3"`), `declaration`, `links` (the links it proposes, may be empty) | – | +| welcome | coordinator | `type`, `idOffset`, `idCount`, `session` (`start`, `interval`, `timeUnit`, `steps`, `stages`, `participants`), `couplings` (its inputs' providers) | – | +| turn | coordinator | `type`, `step`, `phase` (`"init"` at step 0, else `"step"`), `time`, `until` | its inputs | +| update | participant | `type`, `step` | its outputs | +| error | participant | `type`, `message` | – | +| stop | coordinator | `type`, `reason` | – | + +**Declaration.** +- `participant`: its name, an identifier. +- `platform`: optional. +- `entitySets`: each entity set it works on, as `dataStructure`, `name`, `rank`, and `rights` on its topology: + - `owns`: it adds and removes the entities. An entity set has one owner at most. + - `refines`: it adds entities, and removes the ones it created. + - `follows`: it reads. +- `variables`: each one with `dataStructure`, `entitySet`, `name`, `direction` (`provides`: an output; `requires`: an + input), `unit` (UCUM), and optionally `standardName`, `kind`, `optional`. + +**Link.** A link names the input it feeds and the output that feeds it: +- the input: `receiver`, `variable`, `entitySet` (when the name is ambiguous); +- the output: `provider`, `source`, `sourceEntitySet`; +- its conversion: `factor` (applied after the unit conversion), `aggregation` and `interpolation` (`hold` or + `linear`, §11.6). + +**Aggregation.** A link may change the scale: +- `sum`, `mean`, `max` or `min` from a finer entity set, through the `complex` chain. The default is `sum` for an + extensive source, `mean` otherwise. +- `copy` from a coarser one. An extensive source needs `copy` stated explicitly, because copying it to finer entities + counts it several times. + +Formulas are not part of a link: a receiver computes them from its linked inputs. + +### 11.3 Resolution + +Before the first step, the coordinator gives each required input exactly one provider, in this order: +1. a link given to the coordinator; +2. a link proposed by a participant; +3. the output with the same standard name, preferring one on the same entity set; +4. the output with the same name. + +It refuses the session, and stops every participant with the reason, when any of these holds: +- an input that is not optional has no provider, or has several at the same priority; +- the units are not convertible; +- an entity set has two owners; +- a stage holds two participants that change one entity set's topology; +- a link joins two data structures, which needs a mapping (not in this draft). + +### 11.4 Steps and stages + +- The session runs `steps` exchange steps of `interval` (in `timeUnit`), from `start`. +- Participants are grouped in **stages**: given to the coordinator, or by default the owners, then the refiners, then + the others. +- **Initial state (step 0).** Stage by stage, each participant is sent a turn with the entities so far, and answers + with its initial entities and outputs. +- **Each step k ≥ 1.** Stage by stage: + 1. each participant of the stage is sent a turn from `time` to `until`, holding its inputs and the entities added or + removed since its last turn; + 2. once all of them answered, their updates are merged, in the stage's order. + + Participants in one stage therefore see the same state, and the next stage sees theirs. The coupling is explicit: + one pass per step. +- After the last step, every participant is sent `stop`. An error message, or an update the coordinator refuses, also + stops the session, and every participant gets the reason. + +### 11.5 One structure, several participants + +- **Ids.** A participant's new entities take ids in `[idOffset, idOffset + idCount)`, so ids never collide. + Participants keep their own ids, and map them, if they need to. +- **Rights.** A participant may write: + - only the outputs it declared; + - the topology columns only of entities it created, or of an entity set it owns. +- **Removal.** Removing an entity removes, down the finer entity sets, the entities whose complex it is. Its successors + in its own entity set become roots. +- **New entities.** A participant receives new entities with their `parent` and `complex`, and initializes its own + variables on them. When an entity splits, the participant shares its own extensive quantities and dilutes its own + concentrations by its rules; the entity's lineage is its `parent`. +- **Labels.** An entity created without a label takes its entity set's name. + +### 11.6 Transports + +Both transports carry the same messages. + +- **ZeroMQ** (endpoint `tcp://…` or `ipc://…`). + - The coordinator binds a ROUTER socket. + - Each participant connects a DEALER socket whose identity is its name. + - A message is one or two frames: the header, then the payload. +- **A folder** (endpoint `files:`). + - Participant `p` has two queues, `/p/to_coordinator/` and `/p/to_participant/`. + - Message n of a queue is `n.json`, n as 8 digits from `00000000`. + - Its payload is `n.glb`, named in the header's `payload` key. + - The payload is written first, and the header is renamed into place last, so a header present means a complete + message. + - The folder is the session's log; a session needs a new folder. +- **Launching.** A participant launched by the coordinator finds the endpoint in the environment variable + `FSPM_TF_ENDPOINT` and its name in `FSPM_TF_PARTICIPANT`. +- **Replay.** A recorded sequence (§8) may be a participant, so a platform's log feeds another platform's inputs without + the first one running. + - It owns its entity sets, and provides their variables. + - Its ids are renumbered within its range. + - Between two recorded times, its values are those of the last frame at or before the session's time (`hold`), or + interpolated linearly for links stating `"interpolation": "linear"`. + ## Changes from 0.1 All additions; a 0.1 file is a valid 0.2 file. From b5e6a609b9f9205ee83f77cc868db17efc4b564c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 15:08:27 +0100 Subject: [PATCH 20/39] Specification 0.3 (draft) and coupling's next steps: frames say 0.3 (0.1 and 0.2 files stay valid, conformance files regenerated, expected values unchanged); links between two data structures through a mapping a participant provides (sum: amounts shared by weight, kept; mean: weighted mean), replays providing their mappings and declaring every set and variable of their frames; coupling conformance sessions (conformance/coupling: two_scales, two_data_structures; generate.py; fspm-tf couple-conformance) passed by the Python, Julia, C++ and Java participants over ZeroMQ and a folder; partial frames (update) in every codec and dump; the C++ participant (fspm_tf/coupling.hpp, libzmq) and the Java one (org.openalea.fspmtf.coupling, JeroMQ optional); CPlantBox's growing log and GroIMP's log (four runs) replayed as participants; the frame tools skip conformance/coupling; the coordinator's lookups vectorized and the Python codec's enum and text columns encoded without per-row loops (a 100,000-segment step: 99 ms instead of 230 ms); CI jobs for all of it --- .github/workflows/cpp-codec.yml | 14 + .github/workflows/java-codec.yml | 12 + .github/workflows/julia-codec.yml | 5 + CHANGELOG.md | 15 +- README.md | 2 +- bindings/cpp/CMakeLists.txt | 20 +- bindings/cpp/README.md | 4 +- .../cpp/cplantbox/test/test_cplantbox.cpp | 29 ++- bindings/cpp/include/fspm_tf/coupling.hpp | 91 +++++++ bindings/cpp/include/fspm_tf/model.hpp | 16 +- bindings/cpp/src/codec.cpp | 28 ++ bindings/cpp/src/coupling.cpp | 209 +++++++++++++++ bindings/cpp/src/dump.cpp | 11 +- bindings/cpp/test/test_fspm_tf.cpp | 25 +- bindings/cpp/tools/conformance_run.cpp | 1 + bindings/cpp/tools/coupling_conformance.cpp | 42 +++ bindings/java/README.md | 5 +- .../java/groimp-plugin/test/run_in_groimp.sh | 28 +- bindings/java/pom.xml | 10 +- .../main/java/org/openalea/fspmtf/Codec.java | 28 +- .../org/openalea/fspmtf/ConformanceRun.java | 3 +- .../main/java/org/openalea/fspmtf/Dump.java | 10 + .../main/java/org/openalea/fspmtf/Frame.java | 2 + .../main/java/org/openalea/fspmtf/Update.java | 23 ++ .../org/openalea/fspmtf/coupling/Channel.java | 23 ++ .../coupling/ConformanceParticipant.java | 38 +++ .../fspmtf/coupling/FolderChannel.java | 80 ++++++ .../org/openalea/fspmtf/coupling/Json.java | 10 + .../openalea/fspmtf/coupling/Participant.java | 100 +++++++ .../org/openalea/fspmtf/coupling/Turn.java | 9 + .../openalea/fspmtf/coupling/ZmqChannel.java | 58 +++++ .../java/org/openalea/fspmtf/CodecTest.java | 16 +- bindings/julia/FspmTF/src/FspmTF.jl | 4 +- bindings/julia/FspmTF/src/coupling.jl | 39 +++ bindings/julia/FspmTF/src/dump.jl | 10 +- bindings/julia/FspmTF/test/runtests.jl | 3 +- bindings/julia/README.md | 2 +- bindings/julia/conformance_run.jl | 1 + bindings/julia/coupling_conformance.jl | 23 ++ bindings/python/conformance_run.py | 2 + .../python/src/openalea/fspm_tf/__init__.py | 2 +- bindings/python/src/openalea/fspm_tf/cli.py | 25 ++ bindings/python/src/openalea/fspm_tf/codec.py | 23 +- .../src/openalea/fspm_tf/coupling/__init__.py | 2 +- .../openalea/fspm_tf/coupling/conformance.py | 113 ++++++++ .../src/openalea/fspm_tf/coupling/messages.py | 35 ++- .../src/openalea/fspm_tf/coupling/replay.py | 56 +++- .../openalea/fspm_tf/coupling/resolution.py | 39 ++- .../src/openalea/fspm_tf/coupling/store.py | 149 ++++++++--- bindings/python/src/openalea/fspm_tf/dump.py | 13 +- bindings/python/src/openalea/fspm_tf/model.py | 2 +- bindings/python/test/test_coupling.py | 166 ++++++++++++ conformance/README.md | 12 +- conformance/adaptive_soil.glb | Bin 2428 -> 2428 bytes conformance/coupling/generate.py | 217 +++++++++++++++ .../expected/record/frame_00000.json | 219 ++++++++++++++++ .../expected/record/frame_00001.json | 234 +++++++++++++++++ .../expected/record/frame_00002.json | 240 +++++++++++++++++ .../expected/record/frame_00003.json | 246 ++++++++++++++++++ .../expected/record/frame_00004.json | 246 ++++++++++++++++++ .../expected/turn_0000.json | 11 + .../expected/turn_0001.json | 181 +++++++++++++ .../expected/turn_0002.json | 181 +++++++++++++ .../expected/turn_0003.json | 186 +++++++++++++ .../expected/turn_0004.json | 186 +++++++++++++ .../two_data_structures/peer/frame_000000.glb | Bin 0 -> 1420 bytes .../two_data_structures/peer/frame_000001.glb | Bin 0 -> 1420 bytes .../two_data_structures/peer/frame_000002.glb | Bin 0 -> 1436 bytes .../two_data_structures/peer/frame_000003.glb | Bin 0 -> 1436 bytes .../two_data_structures/peer/frame_000004.glb | Bin 0 -> 1436 bytes .../two_data_structures/peer/scene.fspm.json | 31 +++ .../coupling/two_data_structures/session.json | 15 ++ .../subject/declaration.json | 49 ++++ .../subject/update_0000.glb | Bin 0 -> 1988 bytes .../subject/update_0001.glb | Bin 0 -> 2092 bytes .../subject/update_0002.glb | Bin 0 -> 2092 bytes .../subject/update_0003.glb | Bin 0 -> 2140 bytes .../subject/update_0004.glb | Bin 0 -> 2140 bytes .../expected/record/frame_00000.json | 149 +++++++++++ .../expected/record/frame_00001.json | 167 ++++++++++++ .../expected/record/frame_00002.json | 185 +++++++++++++ .../expected/record/frame_00003.json | 185 +++++++++++++ .../expected/record/frame_00004.json | 203 +++++++++++++++ .../two_scales/expected/turn_0000.json | 82 ++++++ .../two_scales/expected/turn_0001.json | 144 ++++++++++ .../two_scales/expected/turn_0002.json | 161 ++++++++++++ .../two_scales/expected/turn_0003.json | 177 +++++++++++++ .../two_scales/expected/turn_0004.json | 178 +++++++++++++ .../coupling/two_scales/peer/frame_000000.glb | Bin 0 -> 1908 bytes .../coupling/two_scales/peer/frame_000001.glb | Bin 0 -> 1932 bytes .../coupling/two_scales/peer/frame_000002.glb | Bin 0 -> 1956 bytes .../coupling/two_scales/peer/frame_000003.glb | Bin 0 -> 1956 bytes .../coupling/two_scales/peer/frame_000004.glb | Bin 0 -> 1988 bytes .../coupling/two_scales/peer/scene.fspm.json | 31 +++ conformance/coupling/two_scales/session.json | 16 ++ .../two_scales/subject/declaration.json | 36 +++ .../two_scales/subject/update_0000.glb | Bin 0 -> 1956 bytes .../two_scales/subject/update_0001.glb | Bin 0 -> 2028 bytes .../two_scales/subject/update_0002.glb | Bin 0 -> 2092 bytes .../two_scales/subject/update_0003.glb | Bin 0 -> 2092 bytes .../two_scales/subject/update_0004.glb | Bin 0 -> 2156 bytes conformance/growing_sequence/frame_000000.glb | Bin 12892 -> 12892 bytes conformance/growing_sequence/frame_000001.glb | Bin 14852 -> 14852 bytes conformance/growing_sequence/frame_000002.glb | Bin 17764 -> 17764 bytes conformance/growing_sequence/scene.fspm.json | 2 +- conformance/missing_values.glb | Bin 6988 -> 6988 bytes conformance/parameter_set.glb | Bin 3484 -> 3484 bytes conformance/root_system.glb | Bin 15820 -> 15820 bytes conformance/section_cells.glb | Bin 5284 -> 5284 bytes conformance/soil_and_roots.glb | Bin 18356 -> 18356 bytes conformance/wheat_shoot.glb | Bin 6956 -> 6956 bytes doc/coupling.md | 60 ++++- doc/cpp.md | 13 + doc/governance.md | 7 +- doc/java.md | 13 +- doc/julia.md | 4 +- doc/platforms.md | 15 +- implementations.md | 26 +- spec/schema/coupling.schema.json | 16 +- spec/schema/fspm_scene.schema.json | 4 +- spec/schema/index.schema.json | 4 +- spec/specification.md | 64 ++++- 122 files changed, 5726 insertions(+), 146 deletions(-) create mode 100644 bindings/cpp/include/fspm_tf/coupling.hpp create mode 100644 bindings/cpp/src/coupling.cpp create mode 100644 bindings/cpp/tools/coupling_conformance.cpp create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/Update.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/ConformanceParticipant.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/Json.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/Turn.java create mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/coupling/ZmqChannel.java create mode 100644 bindings/julia/coupling_conformance.jl create mode 100644 bindings/python/src/openalea/fspm_tf/coupling/conformance.py create mode 100644 conformance/coupling/generate.py create mode 100644 conformance/coupling/two_data_structures/expected/record/frame_00000.json create mode 100644 conformance/coupling/two_data_structures/expected/record/frame_00001.json create mode 100644 conformance/coupling/two_data_structures/expected/record/frame_00002.json create mode 100644 conformance/coupling/two_data_structures/expected/record/frame_00003.json create mode 100644 conformance/coupling/two_data_structures/expected/record/frame_00004.json create mode 100644 conformance/coupling/two_data_structures/expected/turn_0000.json create mode 100644 conformance/coupling/two_data_structures/expected/turn_0001.json create mode 100644 conformance/coupling/two_data_structures/expected/turn_0002.json create mode 100644 conformance/coupling/two_data_structures/expected/turn_0003.json create mode 100644 conformance/coupling/two_data_structures/expected/turn_0004.json create mode 100644 conformance/coupling/two_data_structures/peer/frame_000000.glb create mode 100644 conformance/coupling/two_data_structures/peer/frame_000001.glb create mode 100644 conformance/coupling/two_data_structures/peer/frame_000002.glb create mode 100644 conformance/coupling/two_data_structures/peer/frame_000003.glb create mode 100644 conformance/coupling/two_data_structures/peer/frame_000004.glb create mode 100644 conformance/coupling/two_data_structures/peer/scene.fspm.json create mode 100644 conformance/coupling/two_data_structures/session.json create mode 100644 conformance/coupling/two_data_structures/subject/declaration.json create mode 100644 conformance/coupling/two_data_structures/subject/update_0000.glb create mode 100644 conformance/coupling/two_data_structures/subject/update_0001.glb create mode 100644 conformance/coupling/two_data_structures/subject/update_0002.glb create mode 100644 conformance/coupling/two_data_structures/subject/update_0003.glb create mode 100644 conformance/coupling/two_data_structures/subject/update_0004.glb create mode 100644 conformance/coupling/two_scales/expected/record/frame_00000.json create mode 100644 conformance/coupling/two_scales/expected/record/frame_00001.json create mode 100644 conformance/coupling/two_scales/expected/record/frame_00002.json create mode 100644 conformance/coupling/two_scales/expected/record/frame_00003.json create mode 100644 conformance/coupling/two_scales/expected/record/frame_00004.json create mode 100644 conformance/coupling/two_scales/expected/turn_0000.json create mode 100644 conformance/coupling/two_scales/expected/turn_0001.json create mode 100644 conformance/coupling/two_scales/expected/turn_0002.json create mode 100644 conformance/coupling/two_scales/expected/turn_0003.json create mode 100644 conformance/coupling/two_scales/expected/turn_0004.json create mode 100644 conformance/coupling/two_scales/peer/frame_000000.glb create mode 100644 conformance/coupling/two_scales/peer/frame_000001.glb create mode 100644 conformance/coupling/two_scales/peer/frame_000002.glb create mode 100644 conformance/coupling/two_scales/peer/frame_000003.glb create mode 100644 conformance/coupling/two_scales/peer/frame_000004.glb create mode 100644 conformance/coupling/two_scales/peer/scene.fspm.json create mode 100644 conformance/coupling/two_scales/session.json create mode 100644 conformance/coupling/two_scales/subject/declaration.json create mode 100644 conformance/coupling/two_scales/subject/update_0000.glb create mode 100644 conformance/coupling/two_scales/subject/update_0001.glb create mode 100644 conformance/coupling/two_scales/subject/update_0002.glb create mode 100644 conformance/coupling/two_scales/subject/update_0003.glb create mode 100644 conformance/coupling/two_scales/subject/update_0004.glb diff --git a/.github/workflows/cpp-codec.yml b/.github/workflows/cpp-codec.yml index a0b718e..71d9078 100644 --- a/.github/workflows/cpp-codec.yml +++ b/.github/workflows/cpp-codec.yml @@ -43,6 +43,14 @@ jobs: with: dumps: out/dumps written: out/written + # the coupling conformance sessions (specification §9, §11): the C++ participant, over ZeroMQ (libzmq) and a folder + - name: The coupling conformance sessions + run: | + sudo apt-get install -y -q libzmq3-dev + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --build build --target fspm_tf_coupling_conformance + python -m pip install numpy jsonschema pyzmq . + fspm-tf couple-conformance conformance/coupling --participant "$PWD/build/fspm_tf_coupling_conformance" cplantbox-adapter: runs-on: ubuntu-latest @@ -65,3 +73,9 @@ jobs: run: | python -m pip install numpy jsonschema . fspm-tf validate build/cplantbox/cplantbox_out/*.glb + - name: CPlantBox's log replayed in a coupling session + env: + FSPM_TF_CPLANTBOX_LOG: build/cplantbox/cplantbox_out/anagallis_log/scene.fspm.json + run: | + python -m pip install pytest pyzmq + pytest -v bindings/python/test/test_coupling.py -k platform_log diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml index 02f2100..e3b2ffa 100644 --- a/.github/workflows/java-codec.yml +++ b/.github/workflows/java-codec.yml @@ -48,6 +48,12 @@ jobs: with: dumps: out/dumps written: out/written + # the coupling conformance sessions (specification §9, §11): the Java participant, over ZeroMQ (JeroMQ) and a folder + - name: The coupling conformance sessions + run: | + python -m pip install pyzmq + fspm-tf couple-conformance conformance/coupling --participant \ + "java -cp $PWD/bindings/java/target/fspm-tf-0.1.0.jar:$PWD/bindings/java/target/dependency/* org.openalea.fspmtf.coupling.ConformanceParticipant" groimp-adapter: runs-on: ubuntu-latest @@ -104,3 +110,9 @@ jobs: python -m pip install numpy jsonschema . bash bindings/java/groimp-plugin/test/run_in_groimp.sh groimp-install/usr/share/GroIMP gltf-plugin \ bindings/java/groimp-plugin/target/plugin/fspmTF out + - name: GroIMP's log replayed in a coupling session + env: + FSPM_TF_GROIMP_LOG: out/log/scene.fspm.json + run: | + python -m pip install pytest pyzmq + pytest -v bindings/python/test/test_coupling.py -k platform_log diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml index f592ef8..ce56b30 100644 --- a/.github/workflows/julia-codec.yml +++ b/.github/workflows/julia-codec.yml @@ -91,3 +91,8 @@ jobs: env: FSPM_TF_JULIA: julia --project=coupling-env run: pytest -v bindings/python/test/test_coupling.py + - name: The coupling conformance sessions, the Julia participant + run: | + julia --project=coupling-env -e 'using Pkg; Pkg.add("JSON")' + fspm-tf couple-conformance conformance/coupling --participant \ + "julia --project=$PWD/coupling-env $PWD/bindings/julia/coupling_conformance.jl" diff --git a/CHANGELOG.md b/CHANGELOG.md index 735f1fa..91728ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog -## Unreleased: coupling platforms (specification §11, draft for 0.3) +## Unreleased: specification 0.3 (draft), coupling platforms + +- Specification 0.3: the frames say 0.3 (0.1 and 0.2 files stay valid); "Changes from 0.2"; the conformance files + regenerated (their expected values unchanged). +- Links between two data structures, through a mapping a participant provides (declared, sent in its updates): + `sum` shares each source's amount among its targets by weight, `mean` gives each target its sources' weighted mean; + replayed sequences provide their mappings; a replay declares every entity set and variable of its frames. +- Coupling conformance sessions (`conformance/coupling/`, written by its `generate.py`), `fspm-tf couple-conformance`, + and a participant's program in each codec; partial frames (`update`) in every codec's dump. +- The C++ participant (`fspm_tf/coupling.hpp`; ZeroMQ with libzmq) and the Java one (`org.openalea.fspmtf.coupling`; + ZeroMQ with JeroMQ, optional); partial frames in both codecs. +- CPlantBox's and GroIMP's logs (the adapter's tests, the plugin's headless run) replayed in coupling sessions. +- The coordinator's lookups by id vectorized, and the Python codec's enum and text columns encoded without per-row + loops: a step of 100,000 segments takes 99 ms instead of 230 ms. - Coupling sessions: platforms' models run together, each in its own process, through fspm-TF. Partial frames (`FSPM_scene.update`: the sender, removed entities); the messages (`spec/schema/coupling.schema.json`): a diff --git a/README.md b/README.md index f3fbe65..70e0705 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ An exchange format for whole functional-structural plant model (FSPM) scenes, ac Each time step is one glTF file, and the variables share one vocabulary. It also aims to serve as an in-memory interface for coupling components across platforms. -Status: specification 0.2, draft (0.1 files stay valid). +Status: specification 0.3, draft (0.1 and 0.2 files stay valid): 0.3 adds coupling platforms (§11). - [The documentation](doc/index.md): for users (viewing, checking, Python) and for platform developers (implementing a codec, conformance, changing the format), built with Sphinx (`.readthedocs.yml`; locally: diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 78a6148..2079cca 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -31,12 +31,28 @@ add_library(fspm_tf src/glb.cpp src/codec.cpp src/sequence.cpp - src/dump.cpp) + src/dump.cpp + src/coupling.cpp) add_library(fspm_tf::fspm_tf ALIAS fspm_tf) target_include_directories(fspm_tf PUBLIC $ $) target_link_libraries(fspm_tf PUBLIC nlohmann_json::nlohmann_json) + +# The coupling's ZeroMQ transport, with libzmq when found (its folder transport needs nothing) +option(FSPM_TF_WITH_ZMQ "Build the coupling's ZeroMQ transport (libzmq)" ON) +if(FSPM_TF_WITH_ZMQ) + find_path(ZMQ_INCLUDE_DIR zmq.h) + find_library(ZMQ_LIBRARY NAMES zmq libzmq) + if(ZMQ_INCLUDE_DIR AND ZMQ_LIBRARY) + target_include_directories(fspm_tf PRIVATE ${ZMQ_INCLUDE_DIR}) + target_link_libraries(fspm_tf PRIVATE ${ZMQ_LIBRARY}) + target_compile_definitions(fspm_tf PRIVATE FSPM_TF_WITH_ZMQ) + message(STATUS "fspm_tf: the coupling's ZeroMQ transport, with ${ZMQ_LIBRARY}") + else() + message(STATUS "fspm_tf: libzmq not found, the coupling's folder transport only") + endif() +endif() target_compile_features(fspm_tf PUBLIC cxx_std_17) set_target_properties(fspm_tf PROPERTIES POSITION_INDEPENDENT_CODE ON) if(MSVC) @@ -67,6 +83,8 @@ if(FSPM_TF_BUILD_TESTS) set(FSPM_TF_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") add_executable(fspm_tf_conformance_run tools/conformance_run.cpp) target_link_libraries(fspm_tf_conformance_run PRIVATE fspm_tf::fspm_tf) + add_executable(fspm_tf_coupling_conformance tools/coupling_conformance.cpp) + target_link_libraries(fspm_tf_coupling_conformance PRIVATE fspm_tf::fspm_tf) add_executable(fspm_tf_tests test/test_fspm_tf.cpp) target_link_libraries(fspm_tf_tests PRIVATE fspm_tf::fspm_tf) add_test(NAME fspm_tf_tests COMMAND fspm_tf_tests "${FSPM_TF_CONFORMANCE}") diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 6396685..3582915 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -1,6 +1,6 @@ # fspm_tf (C++) -The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.2). In `cplantbox/` is +The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.3). In `cplantbox/` is CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, and turns a graph into a static CPlantBox root system. @@ -16,6 +16,8 @@ and turns a graph into a static CPlantBox root system. cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build ``` +`fspm_tf/coupling.hpp` is a participant of coupling sessions (its ZeroMQ transport built when libzmq is found). + Its one dependency is nlohmann_json (header-only, MIT), found installed or fetched. It installs as a CMake package: `find_package(fspm_tf)`, then `target_link_libraries(... fspm_tf::fspm_tf)`. See [the documentation](../../doc/cpp.md). diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp index 22be38f..fb90529 100644 --- a/bindings/cpp/cplantbox/test/test_cplantbox.cpp +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -235,6 +235,32 @@ void test_another_platform(const filesystem::path &conformance) { } // namespace +/// CPlantBox's log of a growing root system: a frame a day for six days (out/anagallis_log/), the sequence a coupling +/// session replays (bindings/python/test/test_coupling.py, FSPM_TF_CPLANTBOX_LOG). +void test_growing_log(const filesystem::path ¶meters, const filesystem::path &out) { + auto plant = std::make_shared(1); + plant->readParameters((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), "plant", true, false); + plant->initialize(false); + cplantbox::Options options; + options.model = "Anagallis_femina_Leitner_2010"; + options.parameters_in_extras = false; + std::vector frames; + std::size_t previous = 0; + bool growing = true; + for (int day = 0; day <= 6; ++day) { + if (day > 0) plant->simulate(1., false); + Frame frame = cplantbox::to_frame(*plant, options); + frame.step = day; + const auto segments = frame.data_structures[0].entity_set("SubOrgan").row_count(); + growing = growing && (day == 0 || segments > previous); + previous = segments; + frames.push_back(std::move(frame)); + } + check(growing, "the log: the root system grows every day"); + const auto index = read_index(write_sequence((out / "anagallis_log").string(), frames)); + check(index.frames.size() == 7, "the log: seven frames and their index"); +} + int main(int argument_count, char **arguments) { if (argument_count != 4) { std::cerr << "usage: fspm_tf_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; @@ -244,7 +270,8 @@ int main(int argument_count, char **arguments) { filesystem::create_directories(out); for (auto test : {std::function([&] { test_root_system(parameters, out); }), std::function([&] { test_plant_with_geometry_and_soil(parameters, out); }), - std::function([&] { test_another_platform(conformance); })}) { + std::function([&] { test_another_platform(conformance); }), + std::function([&] { test_growing_log(parameters, out); })}) { try { test(); } catch (const std::exception &error) { diff --git a/bindings/cpp/include/fspm_tf/coupling.hpp b/bindings/cpp/include/fspm_tf/coupling.hpp new file mode 100644 index 0000000..63cfb6b --- /dev/null +++ b/bindings/cpp/include/fspm_tf/coupling.hpp @@ -0,0 +1,91 @@ +// A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and +// requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is fspm-TF's +// (`fspm-tf couple`). +// +// fspm_tf::coupling::Participant participant(declaration); // a JSON declaration; endpoint from the environment +// while (auto turn = participant.next_turn()) { // until the coordinator stops the session +// ... turn->frame: its entity sets, their entities, its inputs; the model from turn->time to turn->until +// participant.send_update(frame); // its outputs and topology changes +// } +// +// Transports: a folder ("files:path") always; ZeroMQ ("tcp://…", "ipc://…") when the library is built with libzmq +// (FSPM_TF_WITH_ZMQ). +#pragma once + +#include +#include +#include +#include +#include + +#include "fspm_tf/model.hpp" + +namespace fspm_tf::coupling { + +/// The version of the coupling protocol this library speaks. +inline constexpr const char *PROTOCOL_VERSION = "0.3"; +/// The environment variables a participant launched by the coordinator reads: the session's endpoint, its name. +inline constexpr const char *ENDPOINT_VARIABLE = "FSPM_TF_ENDPOINT"; +inline constexpr const char *PARTICIPANT_VARIABLE = "FSPM_TF_PARTICIPANT"; + +/// A message: its JSON header and, for turns and updates, a partial frame's GLB bytes. +struct Message { + json header; + std::optional> payload; +}; + +/// A participant's channel to the coordinator. +class Channel { + public: + virtual ~Channel() = default; + virtual void send(const json &header, const std::vector *payload = nullptr) = 0; + /// The next message; throws Error after `timeout` seconds without one (none: wait). + virtual Message receive(std::optional timeout) = 0; +}; + +/// A participant's channel to `endpoint` ("files:path", or a ZeroMQ endpoint). +std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant); + +/// Whether this library speaks ZeroMQ (built with libzmq). +bool has_zeromq(); + +/// A turn: step (0: the initial state, phase "init"), the time it starts from and the time it reaches, and its frame, +/// the participant's inputs (frame.update->removed: the entities removed since its last turn). +struct Turn { + std::int64_t step = 0; + std::string phase; + double time = 0.0; + double until = 0.0; + Frame frame; +}; + +/// A participant in a session: joining it sends the declaration (and the links it proposes) and waits for the welcome. +/// The ids of the entities it creates are in [id_offset, id_offset + id_count). +class Participant { + public: + /// `declaration`: its JSON (spec/schema/coupling.schema.json); its participant name, if empty, and `endpoint`, if + /// empty, from the environment. + explicit Participant(json declaration, std::string endpoint = "", json links = json::array(), + std::optional timeout = std::nullopt); + /// The next turn, or none when the coordinator stopped the session (`stopped`: its reason). + std::optional next_turn(); + /// Sends the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the + /// entities it adds or changes); in frame.update->removed, the entities it removed. + void send_update(Frame frame); + /// Tells the coordinator this participant cannot go on (the session stops). + void fail(const std::string &message); + + std::string name; + std::int64_t id_offset = 0; + std::int64_t id_count = 0; + json session; + json couplings; + std::optional stopped; + + private: + std::unique_ptr channel_; + std::optional timeout_; + std::int64_t step_ = -1; +}; + +} // namespace fspm_tf::coupling diff --git a/bindings/cpp/include/fspm_tf/model.hpp b/bindings/cpp/include/fspm_tf/model.hpp index df0e12f..4936ecd 100644 --- a/bindings/cpp/include/fspm_tf/model.hpp +++ b/bindings/cpp/include/fspm_tf/model.hpp @@ -32,7 +32,7 @@ namespace fspm_tf { using json = nlohmann::ordered_json; /// The version of the specification this library reads and writes. -inline constexpr const char *SPECIFICATION_VERSION = "0.2"; +inline constexpr const char *SPECIFICATION_VERSION = "0.3"; /// The id of no entity: the complex or parent of an entity that has none, the feature row of a vertex with none. inline constexpr std::int64_t NO_ID = -1; /// The values of edge_type: the root of its scale, MTG '<', MTG '+'. @@ -233,8 +233,17 @@ struct DataStructure { EntitySet &entity_set(const std::string &name); }; -/// A scene at one time step (time in time_unit, step), or a parameter set. unknown: the content of a read file this -/// library does not interpret (other root extensions, the root's extras), written back as it was. +/// What makes a frame partial: a coupling session's message (specification §11). participant: its sender +/// ("coordinator" for a turn's inputs); its tables hold some rows and columns only. removed: (data structure id, +/// entity set name) -> the ids of the entities removed. +struct Update { + std::string participant; + std::map, std::vector> removed; +}; + +/// A scene at one time step (time in time_unit, step), or a parameter set. update: set for a partial frame, a coupling +/// session's message. unknown: the content of a read file this library does not interpret (other root extensions, +/// the root's extras), written back as it was. struct Frame { std::vector data_structures; std::vector mappings; @@ -243,6 +252,7 @@ struct Frame { std::int64_t step = 0; json writer = json::object(); std::optional vocabulary_version; + std::optional update; json unknown = json::object(); const DataStructure &data_structure(const std::string &id) const; diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp index fe62682..ef45148 100644 --- a/bindings/cpp/src/codec.cpp +++ b/bindings/cpp/src/codec.cpp @@ -499,10 +499,27 @@ std::vector to_bytes(const Frame &frame) { mappings.push_back(entry); } + json update = nullptr; + if (frame.update) { + check_identifier(frame.update->participant, "participant"); + update = {{"participant", frame.update->participant}}; + if (!frame.update->removed.empty()) { + json removed = json::array(); + for (const auto &[key, ids] : frame.update->removed) { + Table table(key.second + "__removed"); + table.set("id", Column(ids)); + removed.push_back({{"dataStructure", key.first}, {"entitySet", key.second}, + {"propertyTable", add_table(table, key.first + "__" + key.second + "__removed")}}); + } + update["removed"] = removed; + } + } + json scene = {{"version", SPECIFICATION_VERSION}, {"frame", {{"time", frame.time}, {"timeUnit", frame.time_unit}, {"step", frame.step}}}}; if (frame.vocabulary_version) scene["vocabulary"] = *frame.vocabulary_version; if (!frame.writer.empty()) scene["writer"] = frame.writer; + if (!update.is_null()) scene["update"] = update; scene["dataStructures"] = data_structures; if (!mappings.empty()) scene["mappings"] = mappings; if (!variables.empty()) scene["variables"] = variables; @@ -858,6 +875,17 @@ Frame from_bytes(const std::vector &data) { table(entry.at("propertyTable").get(), mapping); frame.mappings.push_back(std::move(mapping)); } + if (scene.contains("update")) { + Update update; + update.participant = scene["update"].at("participant").get(); + for (const auto &item : get(scene["update"], "removed", json::array())) { + Table removed; + table(item.at("propertyTable").get(), removed); + update.removed[{item.at("dataStructure").get(), item.at("entitySet").get()}] = + removed.column("id").as(); + } + frame.update = std::move(update); + } json other_extensions = json::object(); for (const auto &item : extensions.items()) if (item.key() != METADATA && item.key() != EXTENSION && item.key() != FEATURES) diff --git a/bindings/cpp/src/coupling.cpp b/bindings/cpp/src/coupling.cpp new file mode 100644 index 0000000..77e55b9 --- /dev/null +++ b/bindings/cpp/src/coupling.cpp @@ -0,0 +1,209 @@ +// A participant's side of a coupling session: the folder transport, the participant, and the ZeroMQ transport when +// built with libzmq (FSPM_TF_WITH_ZMQ). +#include "fspm_tf/coupling.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fspm_tf/fspm_tf.hpp" + +#ifdef FSPM_TF_WITH_ZMQ +#include +#endif + +namespace fspm_tf::coupling { + +namespace { + +namespace filesystem = std::filesystem; + +std::vector read_bytes(const filesystem::path &path) { + std::ifstream file(path, std::ios::binary); + return std::vector((std::istreambuf_iterator(file)), std::istreambuf_iterator()); +} + +void write_bytes(const filesystem::path &path, const void *data, std::size_t size) { + std::ofstream file(path, std::ios::binary); + file.write(static_cast(data), static_cast(size)); +} + +std::string numbered(std::int64_t number) { + std::ostringstream text; + text << std::setw(8) << std::setfill('0') << number; + return text.str(); +} + +/// A participant's folder queues: //to_coordinator/ and to_participant/. Message n is n.json +/// (its header) and n.glb (its payload), the payload written first and the header renamed into place last. +class FolderChannel : public Channel { + public: + FolderChannel(const std::string &folder, const std::string &participant) + : incoming_(filesystem::path(folder) / participant / "to_participant"), + outgoing_(filesystem::path(folder) / participant / "to_coordinator") { + filesystem::create_directories(incoming_); + filesystem::create_directories(outgoing_); + } + + void send(const json &header, const std::vector *payload) override { + const auto stem = outgoing_ / numbered(next_write_); + json written = header; + if (payload) { + write_bytes(stem.string() + ".glb", payload->data(), payload->size()); + written["payload"] = numbered(next_write_) + ".glb"; + } + const std::string text = written.dump(); + write_bytes(stem.string() + ".json.part", text.data(), text.size()); + filesystem::rename(stem.string() + ".json.part", stem.string() + ".json"); + ++next_write_; + } + + Message receive(std::optional timeout) override { + const auto stem = incoming_ / numbered(next_read_); + const auto start = std::chrono::steady_clock::now(); + while (!filesystem::exists(stem.string() + ".json")) { + if (timeout && std::chrono::duration(std::chrono::steady_clock::now() - start).count() > *timeout) + throw Error("no message in " + std::to_string(*timeout) + " s"); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + const auto text = read_bytes(stem.string() + ".json"); + Message message{json::parse(text.begin(), text.end()), std::nullopt}; + if (message.header.contains("payload")) { + message.payload = read_bytes(incoming_ / message.header["payload"].get()); + message.header.erase("payload"); + } + ++next_read_; + return message; + } + + private: + filesystem::path incoming_, outgoing_; + std::int64_t next_read_ = 0, next_write_ = 0; +}; + +#ifdef FSPM_TF_WITH_ZMQ +/// A DEALER socket named after the participant, connected to the coordinator's ROUTER socket; a message is one or two +/// frames, its header then its payload. +class ZmqChannel : public Channel { + public: + ZmqChannel(const std::string &endpoint, const std::string &participant) { + context_ = zmq_ctx_new(); + socket_ = zmq_socket(context_, ZMQ_DEALER); + zmq_setsockopt(socket_, ZMQ_ROUTING_ID, participant.data(), participant.size()); + const int linger = 1000; + zmq_setsockopt(socket_, ZMQ_LINGER, &linger, sizeof linger); + if (zmq_connect(socket_, endpoint.c_str()) != 0) + throw Error("cannot connect to " + endpoint + ": " + zmq_strerror(zmq_errno())); + } + ~ZmqChannel() override { + zmq_close(socket_); + zmq_ctx_term(context_); + } + + void send(const json &header, const std::vector *payload) override { + const std::string text = header.dump(); + zmq_send(socket_, text.data(), text.size(), payload ? ZMQ_SNDMORE : 0); + if (payload) zmq_send(socket_, payload->data(), payload->size(), 0); + } + + Message receive(std::optional timeout) override { + zmq_pollitem_t item{socket_, 0, ZMQ_POLLIN, 0}; + if (zmq_poll(&item, 1, timeout ? static_cast(*timeout * 1000) : -1) <= 0) + throw Error("no message in " + std::to_string(timeout.value_or(0.0)) + " s"); + std::vector> parts; + int more = 1; + while (more) { + zmq_msg_t part; + zmq_msg_init(&part); + zmq_msg_recv(&part, socket_, 0); + const auto *data = static_cast(zmq_msg_data(&part)); + parts.emplace_back(data, data + zmq_msg_size(&part)); + more = zmq_msg_more(&part); + zmq_msg_close(&part); + } + Message message{json::parse(parts[0].begin(), parts[0].end()), std::nullopt}; + if (parts.size() > 1) message.payload = std::move(parts[1]); + return message; + } + + private: + void *context_ = nullptr; + void *socket_ = nullptr; +}; +#endif + +std::string environment(const char *name) { + const char *value = std::getenv(name); + return value ? value : ""; +} + +} // namespace + +bool has_zeromq() { +#ifdef FSPM_TF_WITH_ZMQ + return true; +#else + return false; +#endif +} + +std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant) { + if (endpoint.rfind("files:", 0) == 0) return std::make_unique(endpoint.substr(6), participant); +#ifdef FSPM_TF_WITH_ZMQ + return std::make_unique(endpoint, participant); +#else + throw Error("the endpoint " + endpoint + " needs ZeroMQ: build fspm_tf with libzmq, or use a folder (files:path)"); +#endif +} + +Participant::Participant(json declaration, std::string endpoint, json links, std::optional timeout) + : timeout_(timeout) { + if (endpoint.empty()) endpoint = environment(ENDPOINT_VARIABLE); + if (endpoint.empty()) throw Error(std::string("no endpoint: give one, or set ") + ENDPOINT_VARIABLE); + name = declaration.value("participant", std::string()); + if (name.empty()) { + name = environment(PARTICIPANT_VARIABLE); + declaration["participant"] = name; + } + channel_ = participant_channel(endpoint, name); + channel_->send({{"type", "hello"}, {"protocol", PROTOCOL_VERSION}, {"declaration", declaration}, + {"links", links}}); + const auto welcome = channel_->receive(timeout_); + const auto type = welcome.header.at("type").get(); + if (type == "stop") + throw Error("the coordinator refused " + name + ": " + welcome.header.value("reason", std::string())); + if (type != "welcome") throw Error("expected a welcome, got " + type); + id_offset = welcome.header.at("idOffset").get(); + id_count = welcome.header.at("idCount").get(); + session = welcome.header.at("session"); + couplings = welcome.header.value("couplings", json::array()); +} + +std::optional Participant::next_turn() { + auto message = channel_->receive(timeout_); + const auto type = message.header.at("type").get(); + if (type == "stop") { + stopped = message.header.value("reason", std::string()); + return std::nullopt; + } + if (type != "turn") throw Error("expected a turn, got " + type); + step_ = message.header.at("step").get(); + return Turn{step_, message.header.at("phase").get(), message.header.at("time").get(), + message.header.at("until").get(), from_bytes(message.payload.value())}; +} + +void Participant::send_update(Frame frame) { + if (!frame.update) frame.update = Update{}; + frame.update->participant = name; + const auto bytes = to_bytes(frame); + channel_->send({{"type", "update"}, {"step", step_}}, &bytes); +} + +void Participant::fail(const std::string &message) { channel_->send({{"type", "error"}, {"message", message}}); } + +} // namespace fspm_tf::coupling diff --git a/bindings/cpp/src/dump.cpp b/bindings/cpp/src/dump.cpp index 9f55bb7..b31fd59 100644 --- a/bindings/cpp/src/dump.cpp +++ b/bindings/cpp/src/dump.cpp @@ -135,8 +135,15 @@ json frame_to_json(const Frame &frame) { item["method"] = mapping.method ? json(*mapping.method) : json(nullptr); mappings.push_back(item); } - return {{"time", frame.time}, {"time_unit", frame.time_unit}, {"step", frame.step}, - {"data_structures", data_structures}, {"mappings", mappings}}; + json out = {{"time", frame.time}, {"time_unit", frame.time_unit}, {"step", frame.step}, + {"data_structures", data_structures}, {"mappings", mappings}}; + if (frame.update) { // a partial frame: its sender and removed entities, sorted (std::map) + json removed = json::array(); + for (const auto &[key, ids] : frame.update->removed) + removed.push_back({{"data_structure", key.first}, {"entity_set", key.second}, {"ids", ids}}); + out["update"] = {{"participant", frame.update->participant}, {"removed", removed}}; + } + return out; } std::vector compare(const json &actual, const json &expected, const std::string &path) { diff --git a/bindings/cpp/test/test_fspm_tf.cpp b/bindings/cpp/test/test_fspm_tf.cpp index 4229202..1a38fdd 100644 --- a/bindings/cpp/test/test_fspm_tf.cpp +++ b/bindings/cpp/test/test_fspm_tf.cpp @@ -42,7 +42,9 @@ template void check_throws(Function function, const std::str std::vector conformance_files(const filesystem::path &conformance) { std::vector files; for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) - if (entry.is_regular_file() && entry.path().extension() == ".glb") files.push_back(entry.path()); + if (entry.is_regular_file() && entry.path().extension() == ".glb" && + entry.path().string().find("coupling") == std::string::npos) // the coupling sessions: their own check + files.push_back(entry.path()); std::sort(files.begin(), files.end()); return files; } @@ -217,6 +219,24 @@ void test_sequence(const filesystem::path &conformance) { } // namespace +void test_partial_frame() { + // A coupling session's message: the sender and the removed entities kept, the reference checks lifted + EntitySet segments("SubOrgan", 6); + segments.set("id", Column(std::vector{10, 11})) + .set("complex", Column(std::vector{3, 3})) + .set("sugar", Column(std::vector{1.0, 2.0})); + DataStructure plant("plant"); + plant.entity_sets.push_back(segments); + Frame frame; + frame.data_structures.push_back(plant); + frame.update = Update{"vpl", {{{"plant", "SubOrgan"}, {5, 6}}}}; + const auto again = from_bytes(to_bytes(frame)); + check(again.update && again.update->participant == "vpl", "partial frame: its sender"); + check(again.update->removed.at({"plant", "SubOrgan"}) == std::vector{5, 6}, + "partial frame: its removed entities"); + check(frame_to_json(again)["update"]["removed"][0]["ids"] == json::array({5, 6}), "partial frame: its dump"); +} + int main(int argument_count, char **arguments) { if (argument_count != 2) { std::cerr << "usage: fspm_tf_tests CONFORMANCE\n"; @@ -225,7 +245,8 @@ int main(int argument_count, char **arguments) { const filesystem::path conformance = arguments[1]; const std::vector>> tests = { {"conformance", [&] { test_conformance(conformance); }}, {"columns", test_columns}, {"mesh", test_mesh}, - {"refused", test_refused}, {"sequence", [&] { test_sequence(conformance); }}}; + {"refused", test_refused}, {"sequence", [&] { test_sequence(conformance); }}, + {"partial frame", test_partial_frame}}; for (const auto &[name, test] : tests) { try { test(); diff --git a/bindings/cpp/tools/conformance_run.cpp b/bindings/cpp/tools/conformance_run.cpp index 662040e..d101562 100644 --- a/bindings/cpp/tools/conformance_run.cpp +++ b/bindings/cpp/tools/conformance_run.cpp @@ -19,6 +19,7 @@ int main(int argument_count, char **arguments) { const filesystem::path conformance = arguments[1], out = arguments[2]; for (const auto &entry : filesystem::recursive_directory_iterator(conformance)) { if (!entry.is_regular_file() || entry.path().extension() != ".glb") continue; + if (filesystem::relative(entry.path(), conformance).begin()->string() == "coupling") continue; // sessions const auto relative = filesystem::relative(entry.path(), conformance); const auto frame = fspm_tf::read_frame(entry.path().string()); auto dump = out / "dumps" / relative; diff --git a/bindings/cpp/tools/coupling_conformance.cpp b/bindings/cpp/tools/coupling_conformance.cpp new file mode 100644 index 0000000..4d5f718 --- /dev/null +++ b/bindings/cpp/tools/coupling_conformance.cpp @@ -0,0 +1,42 @@ +// The C++ participant under test in the coupling conformance sessions (conformance/coupling, specification §9): +// launched by the coordinator as `fspm_tf_coupling_conformance SESSION OUT`, it joins with the session's +// subject/declaration.json, writes what it reads from each turn to OUT/turn_.json, and sends +// subject/update_.glb as read and written again by this library. +// +// fspm-tf couple-conformance conformance/coupling --participant build/fspm_tf_coupling_conformance +#include +#include +#include +#include +#include + +#include "fspm_tf/coupling.hpp" +#include "fspm_tf/fspm_tf.hpp" + +namespace filesystem = std::filesystem; + +std::string numbered(std::int64_t step) { + std::ostringstream text; + text << std::setw(4) << std::setfill('0') << step; + return text.str(); +} + +int main(int count, char **arguments) { + if (count != 3) { + std::cerr << "usage: fspm_tf_coupling_conformance SESSION OUT\n"; + return 2; + } + const filesystem::path session = arguments[1], out = arguments[2]; + filesystem::create_directories(out); + std::ifstream declaration_file(session / "subject" / "declaration.json"); + fspm_tf::coupling::Participant participant(fspm_tf::json::parse(declaration_file), "", fspm_tf::json::array(), + 120.0); + while (auto turn = participant.next_turn()) { + std::ofstream dump(out / ("turn_" + numbered(turn->step) + ".json")); + dump << fspm_tf::frame_to_json(turn->frame).dump(1); + dump.close(); + const auto update = fspm_tf::read_frame((session / "subject" / ("update_" + numbered(turn->step) + ".glb")).string()); + participant.send_update(fspm_tf::from_bytes(fspm_tf::to_bytes(update))); + } + return 0; +} diff --git a/bindings/java/README.md b/bindings/java/README.md index 4e1e2e4..6f83f73 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -1,6 +1,6 @@ # fspm-tf (Java) -The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.2), with one dependency, +The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.3), with one dependency, Jackson. GroIMP's adapter is in `groimp/`. - `src/main/java/org/openalea/fspmtf/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB @@ -13,5 +13,8 @@ Jackson. GroIMP's adapter is in `groimp/`. mvn -f bindings/java/pom.xml verify ``` +`org.openalea.fspmtf.coupling` is a participant of coupling sessions (its ZeroMQ transport with JeroMQ, an optional +dependency). + Licence: GPL-3.0-or-later (`LICENSE`), as GroIMP; the rest of fspm-TF is CeCILL-C. See [the documentation](../../doc/java.md). diff --git a/bindings/java/groimp-plugin/test/run_in_groimp.sh b/bindings/java/groimp-plugin/test/run_in_groimp.sh index 027cf26..4cd93da 100755 --- a/bindings/java/groimp-plugin/test/run_in_groimp.sh +++ b/bindings/java/groimp-plugin/test/run_in_groimp.sh @@ -1,7 +1,9 @@ #!/bin/bash # GroIMP's fspm-TF plugin in GroIMP itself, headless: install it (and GroIMP's glTF plugin, which meshes the shapes) # into a GroIMP 3.0 install, grow one of GroIMP's examples, export its 3D view as an fspm-TF frame, and check the file: -# valid fspm-TF (fspm-tf validate), every triangle with its node. +# valid fspm-TF (fspm-tf validate), every triangle with its node. Then GroIMP's log: the example grown again, a frame +# exported after each of four runs, with their index (OUT/log/scene.fspm.json), which a coupling session replays +# (bindings/python/test/test_coupling.py, FSPM_TF_GROIMP_LOG). # # run_in_groimp.sh GROIMP GLTF_PLUGIN FSPMTF_PLUGIN OUT # GROIMP a GroIMP 3.0 install (the folder holding core.jar and plugins/) @@ -32,3 +34,27 @@ assert len(mesh.positions) > 0 and (rows >= 0).all(), "every vertex carries its assert mesh.positions[:, 2].max() > 0, "the plant stands, z up" print(f"{len(mesh.positions)} vertices, {len(np.unique(rows))} nodes drawn, of {plants.entity_sets[-1].count}") PYTHON + +# GroIMP's log: a frame after each run +log="$out/log" +mkdir -p "$log" +(printf '$loadExample simpleRapeseed\n'; sleep 10 + for step in 0 1 2 3; do printf 'run\nexport3d %s/frame_%06d.glb\n' "$log" "$step"; sleep 25; done + printf 'listFunctions\n') | + java -Xmx1g -Djava.util.prefs.userRoot="$out/home" -jar core.jar -a cli > "$out/groimp_log.log" 2>&1 +python - "$log" <<'PYTHON' +import os +import sys +from openalea.fspm_tf import read, write, write_index +folder = sys.argv[1] +files = sorted(name for name in os.listdir(folder) if name.endswith(".glb")) +assert len(files) == 4, f"four frames expected, {files}" +for step, name in enumerate(files): # GroIMP has no simulation clock: the log's time and step, its run count + frame = read(os.path.join(folder, name)) + frame.time, frame.step = float(step), step + write(os.path.join(folder, name), frame) +write_index(os.path.join(folder, "scene.fspm.json"), + [{"time": float(step), "step": step, "file": name} for step, name in enumerate(files)], time_unit="d") +print(f"GroIMP's log: {len(files)} frames in {folder}") +PYTHON +fspm-tf validate "$log/scene.fspm.json" diff --git a/bindings/java/pom.xml b/bindings/java/pom.xml index c8f763f..344d0c5 100644 --- a/bindings/java/pom.xml +++ b/bindings/java/pom.xml @@ -1,5 +1,6 @@ - @@ -36,6 +37,13 @@ jackson-databind ${jackson.version} + + + org.zeromq + jeromq + 0.6.0 + true + org.junit.jupiter junit-jupiter diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java index c10393a..044ff93 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Codec.java @@ -26,7 +26,7 @@ */ public final class Codec { /** The version of the specification this library reads and writes. */ - public static final String SPECIFICATION_VERSION = "0.2"; + public static final String SPECIFICATION_VERSION = "0.3"; public static final List EDGE_TYPES = List.of("none", "successor", "branch", "unknown"); private static final String EXTENSION = "FSPM_scene", METADATA = "EXT_structural_metadata", @@ -134,10 +134,26 @@ public static byte[] toBytes(Frame frame) { entry.put("propertyTable", addTable(mapping, "mapping__" + mapping.name, buffers, schema, tables, variables)); } + ObjectNode update = null; + if (frame.update != null) { + checkIdentifier(frame.update.participant, "participant"); + update = Glb.MAPPER.createObjectNode().put("participant", frame.update.participant); + if (!frame.update.removed.isEmpty()) { + ArrayNode removed = update.putArray("removed"); + for (Map.Entry item : frame.update.removed.entrySet()) { + String[] key = item.getKey().split("/", 2); + Table table = new Table(key[1] + "__removed"); + table.set("id", Column.of(item.getValue())); + removed.addObject().put("dataStructure", key[0]).put("entitySet", key[1]).put("propertyTable", + addTable(table, key[0] + "__" + key[1] + "__removed", buffers, schema, tables, variables)); + } + } + } ObjectNode scene = Glb.MAPPER.createObjectNode().put("version", SPECIFICATION_VERSION); scene.putObject("frame").put("time", frame.time).put("timeUnit", frame.timeUnit).put("step", frame.step); if (frame.vocabularyVersion != null) scene.put("vocabulary", frame.vocabularyVersion); if (!frame.writer.isEmpty()) scene.set("writer", frame.writer); + if (update != null) scene.set("update", update); scene.set("dataStructures", dataStructures); if (!mappings.isEmpty()) scene.set("mappings", mappings); if (!variables.isEmpty()) scene.set("variables", variables); @@ -647,6 +663,16 @@ public static Frame fromBytes(byte[] data) { decodeTable(document, binary, metadata, variables, entry.get("propertyTable").asInt(), mapping); frame.mappings.add(mapping); } + if (scene.has("update")) { + JsonNode entry = scene.get("update"); + frame.update = new Update(entry.get("participant").asText()); + for (JsonNode item : entry.path("removed")) { + Table removed = new Table(""); + decodeTable(document, binary, metadata, variables, item.get("propertyTable").asInt(), removed); + frame.update.removed.put(Update.key(item.get("dataStructure").asText(), item.get("entitySet").asText()), + removed.column("id").longs()); + } + } ObjectNode others = Glb.MAPPER.createObjectNode(); for (Iterator> items = extensions.fields(); items.hasNext(); ) { Map.Entry item = items.next(); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java b/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java index 4c8e3da..e22a359 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/ConformanceRun.java @@ -25,7 +25,8 @@ public static void main(String[] arguments) throws IOException { Path conformance = Path.of(arguments[0]), out = Path.of(arguments[1]); List files; try (Stream paths = Files.walk(conformance)) { - files = paths.filter(path -> path.toString().endsWith(".glb")).sorted().toList(); + files = paths.filter(path -> path.toString().endsWith(".glb")) + .filter(path -> !conformance.relativize(path).startsWith("coupling")).sorted().toList(); // sessions } for (Path file : files) { Path relative = conformance.relativize(file); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java index 41d270c..d286933 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Dump.java @@ -157,6 +157,16 @@ public static ObjectNode frameToJson(Frame frame) { if (mapping.method == null) item.putNull("method"); else item.put("method", mapping.method); mappings.add(item); } + if (frame.update != null) { // a partial frame: its sender and removed entities, sorted + ObjectNode update = dump.putObject("update").put("participant", frame.update.participant); + ArrayNode removed = update.putArray("removed"); + for (Map.Entry item : frame.update.removed.entrySet()) { + String[] key = item.getKey().split("/", 2); + ObjectNode entry = removed.addObject().put("data_structure", key[0]).put("entity_set", key[1]); + ArrayNode ids = entry.putArray("ids"); + for (long id : item.getValue()) ids.add(id); + } + } return dump; } diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java b/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java index 2fb2300..a791229 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Frame.java @@ -18,6 +18,8 @@ public final class Frame { public long step = 0; public ObjectNode writer = JsonNodeFactory.instance.objectNode(); public String vocabularyVersion; + /** Set for a partial frame, a coupling session's message. */ + public Update update; public ObjectNode unknown = JsonNodeFactory.instance.objectNode(); public DataStructure dataStructure(String id) { diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/Update.java b/bindings/java/src/main/java/org/openalea/fspmtf/Update.java new file mode 100644 index 0000000..ab46f72 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/Update.java @@ -0,0 +1,23 @@ +package org.openalea.fspmtf; + +import java.util.Map; +import java.util.TreeMap; + +/** + * What makes a frame partial: a coupling session's message (specification §11). participant: its sender + * ("coordinator" for a turn's inputs); its tables hold some rows and columns only. removed: "data structure/entity set" + * keys (see {@link #key}) to the ids of the entities removed, sorted by key. + */ +public final class Update { + public String participant; + public final Map removed = new TreeMap<>(); + + public Update(String participant) { + this.participant = participant; + } + + /** The key of an entity set in {@link #removed}. */ + public static String key(String dataStructure, String entitySet) { + return dataStructure + "/" + entitySet; + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java new file mode 100644 index 0000000..e904ba8 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java @@ -0,0 +1,23 @@ +package org.openalea.fspmtf.coupling; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** A participant's channel to the coordinator: a message is a JSON header and, for turns and updates, a partial frame's GLB bytes. */ +public interface Channel extends AutoCloseable { + /** A received message: its header, and its payload or null. */ + record Message(ObjectNode header, byte[] payload) {} + + void send(ObjectNode header, byte[] payload); + + /** The next message; FspmTfException after {@code timeout} seconds without one (null: wait). */ + Message receive(Double timeout); + + @Override + default void close() {} + + /** A participant's channel to {@code endpoint}: "files:path" (a folder), or a ZeroMQ endpoint (JeroMQ). */ + static Channel of(String endpoint, String participant) { + if (endpoint.startsWith("files:")) return new FolderChannel(endpoint.substring("files:".length()), participant); + return new ZmqChannel(endpoint, participant); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ConformanceParticipant.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ConformanceParticipant.java new file mode 100644 index 0000000..8e0f323 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ConformanceParticipant.java @@ -0,0 +1,38 @@ +package org.openalea.fspmtf.coupling; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.openalea.fspmtf.Codec; +import org.openalea.fspmtf.Dump; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * The Java participant under test in the coupling conformance sessions (conformance/coupling, specification §9): + * launched by the coordinator as {@code java ... ConformanceParticipant SESSION OUT}, it joins with the session's + * subject/declaration.json, writes what it reads from each turn to OUT/turn_<step>.json, and sends + * subject/update_<step>.glb as read and written again by this codec. + */ +public final class ConformanceParticipant { + private ConformanceParticipant() {} + + public static void main(String[] arguments) throws Exception { + if (arguments.length != 2) { + System.err.println("usage: ConformanceParticipant SESSION OUT"); + System.exit(2); + } + Path session = Path.of(arguments[0]), out = Path.of(arguments[1]); + Files.createDirectories(out); + ObjectNode declaration = (ObjectNode) Json.MAPPER.readTree(session.resolve("subject").resolve("declaration.json").toFile()); + try (Participant participant = new Participant(declaration, null, null, 120.0)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + String step = String.format("%04d", turn.step()); + Json.MAPPER.writerWithDefaultPrettyPrinter().writeValue(out.resolve("turn_" + step + ".json").toFile(), + Dump.frameToJson(turn.frame())); + byte[] update = Files.readAllBytes(session.resolve("subject").resolve("update_" + step + ".glb")); + participant.sendUpdate(Codec.fromBytes(Codec.toBytes(Codec.fromBytes(update)))); + } + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java new file mode 100644 index 0000000..c137741 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java @@ -0,0 +1,80 @@ +package org.openalea.fspmtf.coupling; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.openalea.fspmtf.FspmTfException; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A participant's folder queues: {@code //to_coordinator/} and {@code to_participant/}. Message n + * is {@code n.json} (its header, n as 8 digits) and {@code n.glb} (its payload), the payload written first and the + * header renamed into place last. + */ +public final class FolderChannel implements Channel { + private final Path incoming, outgoing; + private long nextRead = 0, nextWrite = 0; + + public FolderChannel(String folder, String participant) { + incoming = Path.of(folder, participant, "to_participant"); + outgoing = Path.of(folder, participant, "to_coordinator"); + try { + Files.createDirectories(incoming); + Files.createDirectories(outgoing); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + } + + private static String numbered(long number) { + return String.format("%08d", number); + } + + @Override + public void send(ObjectNode header, byte[] payload) { + String stem = numbered(nextWrite); + ObjectNode written = header.deepCopy(); + try { + if (payload != null) { + Files.write(outgoing.resolve(stem + ".glb"), payload); + written.put("payload", stem + ".glb"); + } + Path part = outgoing.resolve(stem + ".json.part"); + Files.writeString(part, written.toString()); + Files.move(part, outgoing.resolve(stem + ".json"), StandardCopyOption.ATOMIC_MOVE); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + nextWrite++; + } + + @Override + public Message receive(Double timeout) { + Path header = incoming.resolve(numbered(nextRead) + ".json"); + long start = System.nanoTime(); + try { + while (!Files.exists(header)) { + if (timeout != null && (System.nanoTime() - start) / 1e9 > timeout) + throw new FspmTfException("no message in " + timeout + " s"); + Thread.sleep(5); + } + ObjectNode read = (ObjectNode) Json.MAPPER.readTree(Files.readString(header)); + byte[] payload = null; + if (read.has("payload")) { + payload = Files.readAllBytes(incoming.resolve(read.get("payload").asText())); + read.remove("payload"); + } + nextRead++; + return new Message(read, payload); + } catch (IOException error) { + throw new UncheckedIOException(error); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new FspmTfException("interrupted while waiting for a message"); + } + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Json.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Json.java new file mode 100644 index 0000000..2c50b95 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Json.java @@ -0,0 +1,10 @@ +package org.openalea.fspmtf.coupling; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** The JSON mapper of the session's headers. */ +final class Json { + static final ObjectMapper MAPPER = new ObjectMapper(); + + private Json() {} +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java new file mode 100644 index 0000000..c20b999 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java @@ -0,0 +1,100 @@ +package org.openalea.fspmtf.coupling; + +import org.openalea.fspmtf.Codec; +import org.openalea.fspmtf.FspmTfException; +import org.openalea.fspmtf.Frame; +import org.openalea.fspmtf.Update; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A participant's side of a coupling session (specification §11). Joining sends the declaration (its JSON, see + * spec/schema/coupling.schema.json) and the links it proposes, and waits for the welcome; then + * + * 
+ *     for (Turn turn; (turn = participant.nextTurn()) != null; ) {   // until the coordinator stops the session
+ *         ... turn.frame(): its entity sets, their entities, its inputs
+ *         participant.sendUpdate(frame);                             // its outputs and topology changes
+ *     }
+ *
+ * + * The ids of the entities it creates are in [idOffset, idOffset + idCount). + */ +public final class Participant implements AutoCloseable { + public static final String PROTOCOL_VERSION = "0.3"; + public static final String ENDPOINT_VARIABLE = "FSPM_TF_ENDPOINT"; + public static final String PARTICIPANT_VARIABLE = "FSPM_TF_PARTICIPANT"; + + public final String name; + public final long idOffset; + public final long idCount; + public final JsonNode session; + public final JsonNode couplings; + public String stopped; + private final Channel channel; + private final Double timeout; + private long step = -1; + + /** Joins the session at {@code endpoint} (null: the environment's FSPM_TF_ENDPOINT); the declaration's participant + * name, if empty, from FSPM_TF_PARTICIPANT; {@code links}: null for none; {@code timeout}: seconds, null to wait. */ + public Participant(ObjectNode declaration, String endpoint, ArrayNode links, Double timeout) { + this.timeout = timeout; + if (endpoint == null) endpoint = System.getenv(ENDPOINT_VARIABLE); + if (endpoint == null || endpoint.isEmpty()) throw new FspmTfException("no endpoint: give one, or set " + ENDPOINT_VARIABLE); + String own = declaration.path("participant").asText(""); + if (own.isEmpty()) { + own = System.getenv(PARTICIPANT_VARIABLE); + declaration.put("participant", own); + } + name = own; + channel = Channel.of(endpoint, name); + ObjectNode hello = Json.MAPPER.createObjectNode().put("type", "hello").put("protocol", PROTOCOL_VERSION); + hello.set("declaration", declaration); + hello.set("links", links == null ? Json.MAPPER.createArrayNode() : links); + channel.send(hello, null); + Channel.Message welcome = channel.receive(timeout); + String type = welcome.header().path("type").asText(); + if (type.equals("stop")) + throw new FspmTfException("the coordinator refused " + name + ": " + welcome.header().path("reason").asText()); + if (!type.equals("welcome")) throw new FspmTfException("expected a welcome, got " + type); + idOffset = welcome.header().get("idOffset").asLong(); + idCount = welcome.header().get("idCount").asLong(); + session = welcome.header().get("session"); + couplings = welcome.header().path("couplings"); + } + + /** The next turn, or null when the coordinator stopped the session ({@link #stopped}: its reason). */ + public Turn nextTurn() { + Channel.Message message = channel.receive(timeout); + String type = message.header().path("type").asText(); + if (type.equals("stop")) { + stopped = message.header().path("reason").asText(""); + return null; + } + if (!type.equals("turn")) throw new FspmTfException("expected a turn, got " + type); + JsonNode header = message.header(); + step = header.get("step").asLong(); + return new Turn(step, header.get("phase").asText(), header.get("time").asDouble(), header.get("until").asDouble(), + Codec.fromBytes(message.payload())); + } + + /** Sends the turn's outputs: per entity set, the ids and outputs of the rows sent (and the topology columns of the + * entities it adds or changes); in frame.update.removed, the entities it removed. */ + public void sendUpdate(Frame frame) { + if (frame.update == null) frame.update = new Update(name); + frame.update.participant = name; + channel.send(Json.MAPPER.createObjectNode().put("type", "update").put("step", step), Codec.toBytes(frame)); + } + + /** Tells the coordinator this participant cannot go on (the session stops). */ + public void fail(String message) { + channel.send(Json.MAPPER.createObjectNode().put("type", "error").put("message", message), null); + } + + @Override + public void close() throws Exception { + channel.close(); + } +} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Turn.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Turn.java new file mode 100644 index 0000000..65ff715 --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Turn.java @@ -0,0 +1,9 @@ +package org.openalea.fspmtf.coupling; + +import org.openalea.fspmtf.Frame; + +/** + * A turn: step (0: the initial state, phase "init"), the time it starts from and the time it reaches, and its frame, + * the participant's inputs (frame.update.removed: the entities removed since its last turn). + */ +public record Turn(long step, String phase, double time, double until, Frame frame) {} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ZmqChannel.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ZmqChannel.java new file mode 100644 index 0000000..53820fb --- /dev/null +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/ZmqChannel.java @@ -0,0 +1,58 @@ +package org.openalea.fspmtf.coupling; + +import java.nio.charset.StandardCharsets; + +import org.openalea.fspmtf.FspmTfException; +import org.zeromq.SocketType; +import org.zeromq.ZContext; +import org.zeromq.ZMQ; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * A DEALER socket named after the participant (JeroMQ, an optional dependency: needed only for this transport), + * connected to the coordinator's ROUTER socket; a message is one or two frames, its header then its payload. + */ +public final class ZmqChannel implements Channel { + private final ZContext context = new ZContext(); + private final ZMQ.Socket socket; + + public ZmqChannel(String endpoint, String participant) { + socket = context.createSocket(SocketType.DEALER); + socket.setIdentity(participant.getBytes(StandardCharsets.UTF_8)); + socket.setLinger(1000); + socket.connect(endpoint); + } + + @Override + public void send(ObjectNode header, byte[] payload) { + byte[] text = header.toString().getBytes(StandardCharsets.UTF_8); + if (payload == null) socket.send(text, 0); + else { + socket.send(text, ZMQ.SNDMORE); + socket.send(payload, 0); + } + } + + @Override + public Message receive(Double timeout) { + try (ZMQ.Poller poller = context.createPoller(1)) { + poller.register(socket, ZMQ.Poller.POLLIN); + if (poller.poll(timeout == null ? -1 : (long) (timeout * 1000)) <= 0) + throw new FspmTfException("no message in " + timeout + " s"); + } + byte[] header = socket.recv(0); + byte[] payload = socket.hasReceiveMore() ? socket.recv(0) : null; + try { + return new Message((ObjectNode) Json.MAPPER.readTree(header), payload); + } catch (java.io.IOException error) { + throw new FspmTfException("an unreadable message header: " + error.getMessage()); + } + } + + @Override + public void close() { + context.close(); + } +} diff --git a/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java index 48b476d..ae69a7b 100644 --- a/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java +++ b/bindings/java/src/test/java/org/openalea/fspmtf/CodecTest.java @@ -25,7 +25,8 @@ class CodecTest { static Stream conformanceFiles() throws IOException { try (Stream paths = Files.walk(CONFORMANCE)) { - return paths.filter(path -> path.toString().endsWith(".glb")).sorted().toList().stream(); + return paths.filter(path -> path.toString().endsWith(".glb")) + .filter(path -> !CONFORMANCE.relativize(path).startsWith("coupling")).sorted().toList().stream(); } } @@ -142,6 +143,19 @@ void meshes() { assertEquals(mesh.displaySources, readBack.displaySources); } + @Test + void partialFrames() throws IOException { + EntitySet segments = new EntitySet("SubOrgan", 6L); + segments.set("id", Column.of(10L, 11L)).set("complex", Column.of(3L, 3L)).set("sugar", Column.of(1.0, 2.0)); + Frame frame = frameWith(segments); + frame.update = new Update("vpl"); + frame.update.removed.put(Update.key("plants", "SubOrgan"), new long[] {5, 6}); + Frame again = Codec.fromBytes(Codec.toBytes(frame)); + assertEquals("vpl", again.update.participant); + assertArrayEquals(new long[] {5, 6}, again.update.removed.get(Update.key("plants", "SubOrgan"))); + assertEquals(5, asJson(again).get("update").get("removed").get(0).get("ids").get(0).asLong()); + } + static Frame frameWith(EntitySet set) { DataStructure plants = new DataStructure("plants"); plants.entitySets.add(set); diff --git a/bindings/julia/FspmTF/src/FspmTF.jl b/bindings/julia/FspmTF/src/FspmTF.jl index 941317f..e81ad3a 100644 --- a/bindings/julia/FspmTF/src/FspmTF.jl +++ b/bindings/julia/FspmTF/src/FspmTF.jl @@ -24,7 +24,7 @@ using JSON using OrderedCollections: OrderedDict """The version of the specification this package reads and writes.""" -const SPECIFICATION_VERSION = "0.2" +const SPECIFICATION_VERSION = "0.3" include("glb.jl") include("model.jl") @@ -38,7 +38,7 @@ export Frame, DataStructure, EntitySet, Relation, Table, Mapping, Grid, Mesh, Va export read_frame, write_frame, from_bytes, to_bytes, read_index, read_sequence, write_sequence, write_index export frame_to_dict, write_dump, row_count, variable_names, entity_ids, entity_set, data_structure export from_mtg, to_mtg, from_graph, to_graph, add_geometry! -export Update, Declaration, EntitySetSpec, VariableSpec, Link, Participant, Turn, next_turn, turns, +export Update, Declaration, EntitySetSpec, VariableSpec, MappingSpec, Link, declaration_from_json, Participant, Turn, next_turn, turns, send_update, fail """ diff --git a/bindings/julia/FspmTF/src/coupling.jl b/bindings/julia/FspmTF/src/coupling.jl index 2801150..ac1da62 100644 --- a/bindings/julia/FspmTF/src/coupling.jl +++ b/bindings/julia/FspmTF/src/coupling.jl @@ -54,6 +54,18 @@ function VariableSpec(data_structure::AbstractString, entity_set::AbstractString standard_name, kind, optional) end +""" + MappingSpec(name, source, target) + +A mapping a participant provides (its updates carry it, as a frame's `Mapping` of that name): `source` and `target`, +each (data structure, entity set), for links between two data structures. +""" +struct MappingSpec + name::String + source::Tuple{String,String} + target::Tuple{String,String} +end + """ Declaration(participant; platform=nothing, entity_sets=EntitySetSpec[], variables=VariableSpec[]) @@ -64,6 +76,7 @@ Base.@kwdef mutable struct Declaration platform::Union{Nothing,String} = nothing entity_sets::Vector{EntitySetSpec} = EntitySetSpec[] variables::Vector{VariableSpec} = VariableSpec[] + mappings::Vector{MappingSpec} = MappingSpec[] end Declaration(participant::AbstractString; kwargs...) = Declaration(; participant = String(participant), kwargs...) @@ -92,9 +105,33 @@ function declaration_json(declaration::Declaration) value = json_object("participant" => declaration.participant, "entitySets" => entity_sets, "variables" => variables) declaration.platform === nothing || (value["platform"] = declaration.platform) + isempty(declaration.mappings) || (value["mappings"] = Any[ + json_object("name" => item.name, + "source" => json_object("dataStructure" => item.source[1], "entitySet" => item.source[2]), + "target" => json_object("dataStructure" => item.target[1], "entitySet" => item.target[2])) + for item in declaration.mappings]) return value end +""" + declaration_from_json(value) -> Declaration + +A declaration from its JSON (e.g. a file's, `JSON.parsefile`). +""" +function declaration_from_json(value) + entity_sets = [EntitySetSpec(item["dataStructure"], item["name"]; rank = get(item, "rank", nothing), + rights = get(item, "rights", "follows")) for item in get(value, "entitySets", Any[])] + variables = [VariableSpec(item["dataStructure"], item["entitySet"], item["name"], item["direction"]; + unit = get(item, "unit", "1"), standard_name = get(item, "standardName", nothing), + kind = get(item, "kind", nothing), optional = get(item, "optional", false)) + for item in get(value, "variables", Any[])] + mappings = [MappingSpec(item["name"], (item["source"]["dataStructure"], item["source"]["entitySet"]), + (item["target"]["dataStructure"], item["target"]["entitySet"])) + for item in get(value, "mappings", Any[])] + return Declaration(; participant = String(value["participant"]), platform = get(value, "platform", nothing), + entity_sets, variables, mappings) +end + """ Link(receiver, variable, provider, source; entity_set=nothing, source_entity_set=nothing, factor=1.0, aggregation=nothing, interpolation="hold") @@ -111,6 +148,7 @@ Base.@kwdef struct Link factor::Float64 = 1.0 aggregation::Union{Nothing,String} = nothing interpolation::String = "hold" + mapping::Union{Nothing,String} = nothing end Link(receiver, variable, provider, source; kwargs...) = @@ -125,6 +163,7 @@ function link_json(link::Link) link.aggregation === nothing || (value["aggregation"] = link.aggregation) link.factor == 1.0 || (value["factor"] = link.factor) link.interpolation == "hold" || (value["interpolation"] = link.interpolation) + link.mapping === nothing || (value["mapping"] = link.mapping) return value end diff --git a/bindings/julia/FspmTF/src/dump.jl b/bindings/julia/FspmTF/src/dump.jl index 6648ee5..7bbd603 100644 --- a/bindings/julia/FspmTF/src/dump.jl +++ b/bindings/julia/FspmTF/src/dump.jl @@ -85,8 +85,14 @@ function frame_to_dict(frame::Frame) "target" => collect(mapping.target), "method" => mapping.method)) for mapping in frame.mappings] - return json_object("time" => frame.time, "time_unit" => frame.time_unit, "step" => frame.step, - "data_structures" => data_structures, "mappings" => mappings) + out = json_object("time" => frame.time, "time_unit" => frame.time_unit, "step" => frame.step, + "data_structures" => data_structures, "mappings" => mappings) + if frame.update !== nothing # a partial frame: its sender and removed entities, sorted + out["update"] = json_object("participant" => frame.update.participant, + "removed" => Any[json_object("data_structure" => key[1], "entity_set" => key[2], "ids" => ids) + for (key, ids) in sort(collect(frame.update.removed); by = first)]) + end + return out end """ diff --git a/bindings/julia/FspmTF/test/runtests.jl b/bindings/julia/FspmTF/test/runtests.jl index 25cf25e..b30dc72 100644 --- a/bindings/julia/FspmTF/test/runtests.jl +++ b/bindings/julia/FspmTF/test/runtests.jl @@ -8,7 +8,8 @@ const CONFORMANCE = normpath(joinpath(@__DIR__, "..", "..", "..", "..", "conform const MTG_FILES = joinpath(pkgdir(MultiScaleTreeGraph), "test", "files") conformance_files() = sort([relpath(joinpath(folder, name), CONFORMANCE) - for (folder, _, files) in walkdir(CONFORMANCE) for name in files if endswith(name, ".glb")]) + for (folder, _, files) in walkdir(CONFORMANCE) for name in files + if endswith(name, ".glb") && !startswith(relpath(folder, CONFORMANCE), "coupling")]) """The differences between two JSON values, as `fspm-tf compare` finds them: objects by key, arrays in order, numbers exactly (1 and 1.0 equal).""" diff --git a/bindings/julia/README.md b/bindings/julia/README.md index c7edda4..1a86cf6 100644 --- a/bindings/julia/README.md +++ b/bindings/julia/README.md @@ -1,6 +1,6 @@ # FspmTF.jl -The Julia codec of fspm-TF: reading and writing frames and sequences (spec 0.2), and, as a package extension, a +The Julia codec of fspm-TF: reading and writing frames and sequences (spec 0.3), and, as a package extension, a MultiScaleTreeGraph.jl MTG to and from a `graph` data structure (`from_mtg`, `to_mtg`). - `FspmTF/src/`: the GLB container (`glb.jl`), the data model (`model.jl`), reading and writing (`codec.jl`), diff --git a/bindings/julia/conformance_run.jl b/bindings/julia/conformance_run.jl index c834af8..ee3c6f8 100644 --- a/bindings/julia/conformance_run.jl +++ b/bindings/julia/conformance_run.jl @@ -10,6 +10,7 @@ function main(conformance::AbstractString, out::AbstractString) for (folder, _, files) in walkdir(conformance) for name in sort(files) endswith(name, ".glb") || continue + startswith(relpath(folder, conformance), "coupling") && continue # the coupling sessions relative = relpath(joinpath(folder, name), conformance) frame = read_frame(joinpath(conformance, relative)) dump = joinpath(out, "dumps", relative[1:end-length(".glb")] * ".json") diff --git a/bindings/julia/coupling_conformance.jl b/bindings/julia/coupling_conformance.jl new file mode 100644 index 0000000..99cb3fa --- /dev/null +++ b/bindings/julia/coupling_conformance.jl @@ -0,0 +1,23 @@ +# FspmTF.jl's participant under test in the coupling conformance sessions (conformance/coupling, specification §9): +# launched by the coordinator as `julia --project=… coupling_conformance.jl SESSION OUT`, it joins with the session's +# subject/declaration.json, writes what it reads from each turn to OUT/turn_.json, and sends +# subject/update_.glb as read and written again by FspmTF.jl. +# +# fspm-tf couple-conformance conformance/coupling --participant "julia --project=env bindings/julia/coupling_conformance.jl" +using FspmTF +using JSON +using ZMQ + +function main(session::AbstractString, out::AbstractString) + mkpath(out) + declaration = declaration_from_json(JSON.parsefile(joinpath(session, "subject", "declaration.json"))) + participant = Participant(declaration; timeout = 120) + for turn in turns(participant) + write_dump(joinpath(out, "turn_$(lpad(turn.step, 4, '0')).json"), turn.frame) + update = from_bytes(to_bytes(read_frame(joinpath(session, "subject", "update_$(lpad(turn.step, 4, '0')).glb")))) + send_update(participant, update) + end + return 0 +end + +exit(main(ARGS[1], ARGS[2])) diff --git a/bindings/python/conformance_run.py b/bindings/python/conformance_run.py index 16e4904..a12cf91 100644 --- a/bindings/python/conformance_run.py +++ b/bindings/python/conformance_run.py @@ -16,6 +16,8 @@ def main(conformance: str, out: str) -> int: for folder, _, files in os.walk(conformance): + if os.path.relpath(folder, conformance).split(os.sep)[0] == "coupling": + continue # the coupling sessions: their own check for name in sorted(files): if not name.endswith(".glb"): continue diff --git a/bindings/python/src/openalea/fspm_tf/__init__.py b/bindings/python/src/openalea/fspm_tf/__init__.py index 621df50..c64a250 100644 --- a/bindings/python/src/openalea/fspm_tf/__init__.py +++ b/bindings/python/src/openalea/fspm_tf/__init__.py @@ -3,7 +3,7 @@ (openalea.fspm_tf.codec), the validator (openalea.fspm_tf.validate), the display copies (openalea.fspm_tf.display) and the vocabulary (openalea.fspm_tf.vocabulary). See the specification in the repository's spec/ folder. """ -SPECIFICATION_VERSION = "0.2" +SPECIFICATION_VERSION = "0.3" from openalea.fspm_tf.model import (DataStructure, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, Relation, # noqa: E402 Table, Update, Variable) diff --git a/bindings/python/src/openalea/fspm_tf/cli.py b/bindings/python/src/openalea/fspm_tf/cli.py index 14b599b..a9381ce 100644 --- a/bindings/python/src/openalea/fspm_tf/cli.py +++ b/bindings/python/src/openalea/fspm_tf/cli.py @@ -14,6 +14,10 @@ run a coupling session (specification §11): launch the participants' commands, wait for the expected ones, replay sequences; exit 1 if it stops on an error + fspm-tf couple-conformance SESSIONS --participant COMMAND [--out DIR] [--transport zmq|files] + run the coupling conformance sessions (conformance/coupling) with + COMMAND SESSION OUT as the participant under test; exit 1 if what it + read, or the session's record, differs from the expected values """ import argparse import sys @@ -131,6 +135,19 @@ def _couple(arguments) -> int: return 0 +def _couple_conformance(arguments) -> int: + import tempfile + from openalea.fspm_tf.coupling.conformance import TRANSPORTS, run_all + out = arguments.out or tempfile.mkdtemp(prefix="fspm_tf_sessions_") + transports = [arguments.transport] if arguments.transport else list(TRANSPORTS) + results = run_all(arguments.sessions, arguments.participant, out, transports) + for (name, transport), differences in results.items(): + _print_differences(f"{name} ({transport})", differences) + failed = sum(1 for differences in results.values() if differences) + print(f"{len(results) - failed} of {len(results)} sessions as expected (outputs in {out})") + return 1 if failed or not results else 0 + + def main(argv=None) -> int: parser = argparse.ArgumentParser(prog="fspm-tf", description="fspm-TF files: validate, inspect, display; couple platforms") commands = parser.add_subparsers(dest="command", required=True) @@ -171,6 +188,14 @@ def main(argv=None) -> int: couple.add_argument("--record", help="a folder: the session recorded as a sequence") couple.add_argument("--timeout", type=float, default=600., help="seconds to wait for a message") couple.set_defaults(run=_couple) + sessions = commands.add_parser("couple-conformance", help="a participant's side against the coupling " + "conformance sessions") + sessions.add_argument("sessions", help="the sessions' folder (conformance/coupling)") + sessions.add_argument("--participant", required=True, help="the command of the participant under test; it is " + "run as COMMAND SESSION OUT") + sessions.add_argument("--out", help="where the outputs go (default: a new temporary folder)") + sessions.add_argument("--transport", choices=["zmq", "files"], help="one transport only (default: both)") + sessions.set_defaults(run=_couple_conformance) arguments = parser.parse_args(argv) return arguments.run(arguments) diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/fspm_tf/codec.py index 217556e..38be5ff 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/fspm_tf/codec.py @@ -301,20 +301,23 @@ def _free_string(column: np.ndarray) -> str: def _enum_codes(name: str, column: np.ndarray, class_id: str, schema: dict) -> tuple: + values = np.asarray(column, dtype=object).tolist() + distinct = set(values) # hashed: a few names, however many rows + if None in distinct: + raise ValueError(f"{name} of '{class_id}': an enum column has no missing values (give each row a {name})") + names = sorted(distinct) + code_of = {label: code for code, label in enumerate(names)} + codes = np.fromiter(map(code_of.__getitem__, values), dtype=np.int64, count=len(values)) if name == "edge_type": - unknown = set(column) - set(EDGE_TYPES) + unknown = set(names) - set(EDGE_TYPES) if unknown: raise ValueError(f"edge_type of '{class_id}': unknown values {sorted(unknown)} (allowed: {EDGE_TYPES})") - return EDGE_TYPE_ENUM, np.array([EDGE_TYPES.index(value) for value in column], dtype=np.uint8) - if any(value is None for value in column): - raise ValueError(f"{name} of '{class_id}': an enum column has no missing values (give each row a {name})") - names = sorted(set(column)) + return EDGE_TYPE_ENUM, np.array([EDGE_TYPES.index(value) for value in names], dtype=np.uint8)[codes] enum_id = f"{class_id}__{name}" value_type = "UINT8" if len(names) <= 256 else "UINT16" schema["enums"][enum_id] = {"valueType": value_type, "values": [{"name": label, "value": code} for code, label in enumerate(names)]} - code_of = {label: code for code, label in enumerate(names)} - return enum_id, np.array([code_of[value] for value in column], dtype=value_type.lower()) + return enum_id, codes.astype(value_type.lower()) def _grid_json(grid: Grid) -> dict: @@ -454,8 +457,10 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict if prop["type"] == "ENUM": enum = schema["enums"][prop["enumType"]] codes = np.frombuffer(raw, dtype=enum.get("valueType", "UINT16").lower(), count=count) - name_of = {item["value"]: item["name"] for item in enum["values"]} - column = np.array([name_of[int(code)] for code in codes], dtype=object) + values = np.array([item["value"] for item in enum["values"]], dtype=np.int64) + names = np.empty(int(values.max()) + 1 if values.size else 0, dtype=object) + names[values] = [item["name"] for item in enum["values"]] + column = names[codes.astype(np.int64)] if count else np.zeros(0, dtype=object) elif prop["type"] == "STRING": offset_type = stored.get("stringOffsetType", "UINT32").lower() offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, diff --git a/bindings/python/src/openalea/fspm_tf/coupling/__init__.py b/bindings/python/src/openalea/fspm_tf/coupling/__init__.py index 74e145c..ef93700 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/__init__.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/__init__.py @@ -11,7 +11,7 @@ - replay: a recorded sequence as a participant (Replay). """ from openalea.fspm_tf.coupling.messages import (ID_COUNT, PROTOCOL_VERSION, CouplingError, Declaration, # noqa: F401 - EntitySetSpec, Link, VariableSpec) + EntitySetSpec, Link, MappingSpec, VariableSpec) from openalea.fspm_tf.coupling.resolution import Coupling, resolve, stages # noqa: F401 from openalea.fspm_tf.coupling.participant import Participant, Turn # noqa: F401 from openalea.fspm_tf.coupling.coordinator import Coordinator, python_command # noqa: F401 diff --git a/bindings/python/src/openalea/fspm_tf/coupling/conformance.py b/bindings/python/src/openalea/fspm_tf/coupling/conformance.py new file mode 100644 index 0000000..e8258b5 --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/coupling/conformance.py @@ -0,0 +1,113 @@ +""" +Conformance sessions (specification §9): how a codec proves its participant's side of a coupling session. + +A session's folder (conformance/coupling//) holds: + + session.json the coordinator's settings: steps, interval, timeUnit, links, stages + peer/scene.fspm.json, frames the other participant, a recorded sequence the coordinator replays ("peer") + subject/declaration.json the participant under test's declaration ("subject") + subject/update_.glb what it sends at each step (0: its initial state) + expected/turn_.json what it must read from each turn (frame_to_dict) + expected/record/frame_.json the session's record, once its updates merged + +The participant under test is a small program of the codec's language, run as + + SESSION OUT + +that joins the session with subject/declaration.json, and at each turn writes what it read (its dump, the +conformance files' layout) to OUT/turn_.json and sends subject/update_.glb as read and written again by +its codec, until the coordinator stops it. `fspm-tf couple-conformance SESSIONS --participant COMMAND` runs every +session over ZeroMQ and over a folder and compares the dumps; this module's `subject` is Python's program +(`python -m openalea.fspm_tf.coupling.conformance subject SESSION OUT`). +""" +import json +import os +import sys + +from openalea.fspm_tf.codec import from_bytes, read, read_index, to_bytes +from openalea.fspm_tf.compare import compare +from openalea.fspm_tf.coupling.coordinator import Coordinator +from openalea.fspm_tf.coupling.messages import Declaration, Link +from openalea.fspm_tf.coupling.participant import Participant +from openalea.fspm_tf.dump import frame_to_dict + +TRANSPORTS = ("zmq", "files") + + +def subject(session: str, out: str) -> int: + """The participant under test, in Python.""" + os.makedirs(out, exist_ok=True) + with open(os.path.join(session, "subject", "declaration.json"), encoding="utf-8") as file: + declaration = Declaration.from_json(json.load(file)) + participant = Participant.connect(declaration, timeout=120) + for turn in participant.turns(): + with open(os.path.join(out, f"turn_{turn.step:04d}.json"), "w", encoding="utf-8") as file: + json.dump(frame_to_dict(turn.frame), file, indent=1) + participant.send(from_bytes(to_bytes(read(os.path.join(session, "subject", + f"update_{turn.step:04d}.glb"))))) + return 0 + + +def _settings(session: str) -> dict: + with open(os.path.join(session, "session.json"), encoding="utf-8") as file: + return json.load(file) + + +def run_session(session: str, command: str, out: str, transport: str = "zmq") -> list: + """Run one session with *command* as its subject; the differences between what it read (and the record) and + the expected values, as lines (empty: it conforms).""" + settings = _settings(session) + endpoint = "tcp://127.0.0.1:*" if transport == "zmq" else "files:" + os.path.join(out, "messages") + coordinator = Coordinator(endpoint, steps=settings["steps"], interval=settings["interval"], + time_unit=settings.get("timeUnit", "d"), start=settings.get("start", 0.), + links=[Link.from_json(link) for link in settings.get("links", [])], + stages=settings.get("stages"), + replays={"peer": os.path.join(session, "peer", "scene.fspm.json")}, + record=os.path.join(out, "record"), timeout=settings.get("timeout", 120)) + turns = os.path.join(out, "turns") + coordinator.launch("subject", f"{command} {session} {turns}") + coordinator.run() + differences = [] + expected_folder = os.path.join(session, "expected") + for name in sorted(os.listdir(expected_folder)): + if name.startswith("turn_") and name.endswith(".json"): + differences += _compare(os.path.join(turns, name), os.path.join(expected_folder, name)) + for entry in read_index(os.path.join(out, "record", "scene.fspm.json"))["frames"]: + expected = os.path.join(expected_folder, "record", entry["file"].replace(".glb", ".json")) + differences += _compare_values(frame_to_dict(read(entry["path"])), expected, entry["file"]) + return differences + + +def _compare(actual_path: str, expected_path: str) -> list: + if not os.path.exists(actual_path): + return [f"{os.path.basename(actual_path)}: not written"] + with open(actual_path, encoding="utf-8") as file: + actual = json.load(file) + return _compare_values(actual, expected_path, os.path.basename(actual_path)) + + +def _compare_values(actual, expected_path: str, name: str) -> list: + with open(expected_path, encoding="utf-8") as file: + expected = json.load(file) + actual = json.loads(json.dumps(actual)) # the same JSON types as the expected file's + return [f"{name}: {line}" for line in compare(actual, expected)] + + +def run_all(sessions: str, command: str, out: str, transports=TRANSPORTS) -> dict: + """Every session of the folder *sessions*, over each transport: {(session, transport): differences}.""" + results = {} + for name in sorted(os.listdir(sessions)): + session = os.path.join(sessions, name) + if not os.path.exists(os.path.join(session, "session.json")): + continue + for transport in transports: + results[(name, transport)] = run_session(os.path.abspath(session), command, + os.path.join(out, name, transport), transport) + return results + + +if __name__ == "__main__": + if len(sys.argv) == 4 and sys.argv[1] == "subject": + sys.exit(subject(sys.argv[2], sys.argv[3])) + print("usage: python -m openalea.fspm_tf.coupling.conformance subject SESSION OUT", file=sys.stderr) + sys.exit(2) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/messages.py b/bindings/python/src/openalea/fspm_tf/coupling/messages.py index 106efb3..82928c5 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/messages.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/messages.py @@ -95,13 +95,32 @@ def from_json(cls, value: dict) -> "VariableSpec": value.get("unit", "1"), value.get("standardName"), value.get("kind"), value.get("optional", False)) +@dataclass +class MappingSpec: + """A mapping a participant provides (its updates carry it, as a frame's Mapping of that name): weights between the + entities of *source* and *target*, each (data structure, entity set), for links between two data structures.""" + name: str + source: tuple + target: tuple + + def to_json(self) -> dict: + return {"name": self.name, "source": {"dataStructure": self.source[0], "entitySet": self.source[1]}, + "target": {"dataStructure": self.target[0], "entitySet": self.target[1]}} + + @classmethod + def from_json(cls, value: dict) -> "MappingSpec": + return cls(value["name"], (value["source"]["dataStructure"], value["source"]["entitySet"]), + (value["target"]["dataStructure"], value["target"]["entitySet"])) + + @dataclass class Declaration: - """A participant's declaration: its name, platform, entity sets and variables.""" + """A participant's declaration: its name, platform, entity sets, variables, and the mappings it provides.""" participant: str platform: Optional[str] = None entity_sets: list = field(default_factory=list) variables: list = field(default_factory=list) + mappings: list = field(default_factory=list) def provides(self) -> list: return [variable for variable in self.variables if variable.direction == "provides"] @@ -132,13 +151,16 @@ def to_json(self) -> dict: "variables": [item.to_json() for item in self.variables]} if self.platform: value["platform"] = self.platform + if self.mappings: + value["mappings"] = [item.to_json() for item in self.mappings] return value @classmethod def from_json(cls, value: dict) -> "Declaration": return cls(value["participant"], value.get("platform"), [EntitySetSpec.from_json(item) for item in value.get("entitySets", [])], - [VariableSpec.from_json(item) for item in value.get("variables", [])]).complete() + [VariableSpec.from_json(item) for item in value.get("variables", [])], + [MappingSpec.from_json(item) for item in value.get("mappings", [])]).complete() @dataclass @@ -146,7 +168,9 @@ class Link: """An input's provider: the receiver's variable (*entity_set* when its name is ambiguous) fed by the provider's *source* variable, times *factor* (after the unit conversion), changed of scale by *aggregation* (sum, mean, max, min from a finer entity set; copy from a coarser one), and between a replayed sequence's times by - *interpolation* (hold, linear).""" + *interpolation* (hold, linear). Between two data structures, through the *mapping* a participant provides: + "sum" shares each source entity's value among its targets in proportion to their weights (an amount is kept), + "mean" gives each target the weighted mean of its sources.""" receiver: str variable: str provider: str @@ -156,6 +180,7 @@ class Link: factor: float = 1. aggregation: Optional[str] = None interpolation: str = "hold" + mapping: Optional[str] = None def __post_init__(self): if self.aggregation is not None and self.aggregation not in AGGREGATIONS: @@ -169,7 +194,7 @@ def to_json(self) -> dict: value = {"receiver": self.receiver, "variable": self.variable, "provider": self.provider, "source": self.source} for key, name in (("entity_set", "entitySet"), ("source_entity_set", "sourceEntitySet"), - ("aggregation", "aggregation")): + ("aggregation", "aggregation"), ("mapping", "mapping")): if getattr(self, key) is not None: value[name] = getattr(self, key) if self.factor != 1.: @@ -182,4 +207,4 @@ def to_json(self) -> dict: def from_json(cls, value: dict) -> "Link": return cls(value["receiver"], value["variable"], value["provider"], value["source"], value.get("entitySet"), value.get("sourceEntitySet"), float(value.get("factor", 1.)), - value.get("aggregation"), value.get("interpolation", "hold")) + value.get("aggregation"), value.get("interpolation", "hold"), value.get("mapping")) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/replay.py b/bindings/python/src/openalea/fspm_tf/coupling/replay.py index 8cf22de..29391ca 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/replay.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/replay.py @@ -7,8 +7,8 @@ import numpy as np from openalea.fspm_tf.codec import read, read_index -from openalea.fspm_tf.coupling.messages import Declaration, EntitySetSpec, VariableSpec -from openalea.fspm_tf.model import NO_ID, DataStructure, EntitySet, Frame, Update +from openalea.fspm_tf.coupling.messages import Declaration, EntitySetSpec, MappingSpec, VariableSpec +from openalea.fspm_tf.model import NO_ID, DataStructure, EntitySet, Frame, Mapping, Update from openalea.fspm_tf.ucum import conversion_factor _ID_COLUMNS = ("id", "complex", "parent") @@ -29,23 +29,42 @@ def __init__(self, name: str, index_path: str): self.linear = set() # (data structure, entity set, variable) interpolated self.sent = {} # (data structure, entity set): ids sent at the last turn self.id_map = {} # recorded id: session id - self.declaration = self._declare(self._frame(0)) + self.declaration = self._declare() def _frame(self, position: int) -> Frame: if position not in self._cache: self._cache = {position: read(self.paths[position])} # one frame kept return self._cache[position] - def _declare(self, frame: Frame) -> Declaration: - declaration = Declaration(self.name, platform=frame.writer.get("platform", "replay")) - for data_structure in frame.data_structures: - for entity_set in data_structure.entity_sets: - declaration.entity_sets.append(EntitySetSpec(data_structure.id, entity_set.name, entity_set.rank, - "owns")) - for name in entity_set.variable_names(): - variable = entity_set.declaration(name) - declaration.variables.append(VariableSpec(data_structure.id, entity_set.name, name, "provides", - variable.unit, variable.standard_name, variable.kind)) + def _declare(self) -> Declaration: + """Its declaration: every entity set, variable and mapping of its frames (some appear later than others).""" + declaration = None + sets, variables, mappings = set(), set(), set() + self.own = set() + for position in range(len(self.paths)): + frame = self._frame(position) + if declaration is None: + declaration = Declaration(self.name, platform=frame.writer.get("platform", "replay")) + for data_structure in frame.data_structures: + self.own.add(data_structure.id) + for entity_set in data_structure.entity_sets: + if (data_structure.id, entity_set.name) not in sets: + sets.add((data_structure.id, entity_set.name)) + declaration.entity_sets.append(EntitySetSpec(data_structure.id, entity_set.name, + entity_set.rank, "owns")) + for name in entity_set.variable_names(): + if (data_structure.id, entity_set.name, name) in variables: + continue + variables.add((data_structure.id, entity_set.name, name)) + variable = entity_set.declaration(name) + declaration.variables.append(VariableSpec(data_structure.id, entity_set.name, name, "provides", + variable.unit, variable.standard_name, + variable.kind)) + for mapping in frame.mappings: + if mapping.name not in mappings: + mappings.add(mapping.name) + declaration.mappings.append(MappingSpec(mapping.name, tuple(mapping.source), + tuple(mapping.target))) return declaration def update(self, time: float, time_unit: str) -> Frame: @@ -79,7 +98,16 @@ def update(self, time: float, time_unit: str) -> Frame: removed[key] = np.array(sorted(gone), dtype=np.int64) self.sent[key] = ids structures.append(out) - return Frame(data_structures=structures, update=Update(self.name, removed)) + mappings = [] + for mapping in frame.mappings: # ids renumbered at the ends in its own data structures + columns = dict(mapping.columns) + for end, (data_structure_id, _) in (("source", mapping.source), ("target", mapping.target)): + if data_structure_id in self.own: + columns[end] = np.array([self._session_id(int(value)) for value in columns[end].tolist()], + dtype=np.int64) + mappings.append(Mapping(mapping.name, columns=columns, source=mapping.source, target=mapping.target, + method=mapping.method)) + return Frame(data_structures=structures, mappings=mappings, update=Update(self.name, removed)) def _session_id(self, recorded: int) -> int: """The session id of a recorded id: the replay's ids renumbered from its offset, in order of appearance.""" diff --git a/bindings/python/src/openalea/fspm_tf/coupling/resolution.py b/bindings/python/src/openalea/fspm_tf/coupling/resolution.py index 6fd3dbc..0b10577 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/resolution.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/resolution.py @@ -22,6 +22,8 @@ class Coupling: aggregation: Optional[str] = None interpolation: str = "hold" how: str = "link" + mapping: Optional[str] = None + reverse: bool = False # the mapping read from its target to its source def to_json(self) -> dict: value = {"variable": self.variable.name, "entitySet": self.variable.entity_set, @@ -32,6 +34,8 @@ def to_json(self) -> dict: value["aggregation"] = self.aggregation if self.interpolation != "hold": value["interpolation"] = self.interpolation + if self.mapping: + value["mapping"] = self.mapping return value @@ -52,6 +56,13 @@ def resolve(declarations: dict, links=()) -> list: """The couplings of every input of *declarations* ({participant: Declaration}), from *links* (Link) first, then by standard name, then by name; CouplingError when an input has none (unless optional) or several.""" known_ranks = ranks(declarations) + mappings = {} + for declaration in declarations.values(): + for mapping in declaration.mappings: + if mapping.name in mappings: + raise CouplingError(f"mapping {mapping.name}: provided by {mappings[mapping.name][0]} and " + f"{declaration.participant}") + mappings[mapping.name] = (declaration.participant, mapping) owners = {} for declaration in declarations.values(): for entity_set in declaration.entity_sets: @@ -84,7 +95,8 @@ def resolve(declarations: dict, links=()) -> list: raise CouplingError(f"link {receiver}.{variable.name} <- {link.provider}.{link.source}: " f"{len(sources)} such outputs of {link.provider}") couplings.append(_coupling(receiver, variable, link.provider, sources[0], known_ranks, "link", - link.factor, link.aggregation, link.interpolation)) + link.factor, link.aggregation, link.interpolation, + mappings.get(link.mapping, (None, link.mapping))[1])) continue coupling = None for how, same in (("standard name", lambda source: variable.standard_name is not None @@ -112,14 +124,29 @@ def resolve(declarations: dict, links=()) -> list: def _coupling(receiver, variable, provider, source, known_ranks, how, factor=1., aggregation=None, - interpolation="hold") -> Coupling: - if source.data_structure != variable.data_structure: - raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: different data structures " - f"({source.data_structure}, {variable.data_structure}) need a mapping (not supported yet)") + interpolation="hold", mapping=None) -> Coupling: + where = f"{receiver}.{variable.name} <- {provider}.{source.name}" try: factor = factor * conversion_factor(source.unit, variable.unit) except UnitError as error: - raise CouplingError(f"{receiver}.{variable.name} <- {provider}.{source.name}: {error}") from None + raise CouplingError(f"{where}: {error}") from None + if mapping is not None or source.data_structure != variable.data_structure: + if mapping is None or isinstance(mapping, str): + raise CouplingError(f"{where}: different data structures ({source.data_structure}, " + f"{variable.data_structure}) need a mapping a participant provides" + + (f" ({mapping} is none)" if mapping else "")) + if (mapping.source, mapping.target) == (source.where, variable.where): + reverse = False + elif (mapping.target, mapping.source) == (source.where, variable.where): + reverse = True + else: + raise CouplingError(f"{where}: mapping {mapping.name} joins {mapping.source} and {mapping.target}, not " + f"{source.where} and {variable.where}") + aggregation = aggregation or ("sum" if source.kind == "extensive" else "mean") + if aggregation not in ("sum", "mean"): + raise CouplingError(f"{where}: through a mapping, 'sum' or 'mean'") + return Coupling(receiver, variable, provider, source, factor, aggregation, interpolation, how, mapping.name, + reverse) if source.entity_set != variable.entity_set: source_rank = known_ranks.get(source.where) target_rank = known_ranks.get(variable.where) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/store.py b/bindings/python/src/openalea/fspm_tf/coupling/store.py index 4be2245..36073b8 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/store.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/store.py @@ -57,7 +57,7 @@ class EntitySetStore: def __init__(self, data_structure: str, name: str, rank): self.data_structure, self.name, self.rank = data_structure, name, rank self.ids = np.zeros(0, dtype=np.int64) - self.row = {} + self._sorted = None # (ids sorted, their rows): the lookups by id self.topology = {"label": np.zeros(0, dtype=object), "complex": np.zeros(0, dtype=np.int64), "parent": np.zeros(0, dtype=np.int64), "edge_type": np.zeros(0, dtype=object), "sibling_order": np.zeros(0, dtype=np.int64)} @@ -68,12 +68,31 @@ def __init__(self, data_structure: str, name: str, rank): def __len__(self): return self.ids.size - def rows(self, ids) -> np.ndarray: - return np.array([self.row[int(identifier)] for identifier in ids], dtype=np.int64) + def _index(self) -> tuple: + if self._sorted is None: + order = np.argsort(self.ids, kind="stable") + self._sorted = (self.ids[order], order) + return self._sorted + + def rows(self, ids, strict: bool = True) -> np.ndarray: + """The rows of *ids* (-1 for an id not here; KeyError if *strict*).""" + ids = np.asarray(ids, dtype=np.int64).reshape(-1) + keys, order = self._index() + if keys.size == 0: + found, rows = np.zeros(ids.size, dtype=bool), np.full(ids.size, -1, dtype=np.int64) + else: + position = np.minimum(np.searchsorted(keys, ids), keys.size - 1) + found = keys[position] == ids + rows = np.where(found, order[position], -1) + if strict and not found.all(): + raise KeyError(f"ids not in {self.data_structure}/{self.name}: {ids[~found][:5].tolist()}") + return rows + + def contains(self, ids) -> np.ndarray: + return self.rows(ids, strict=False) >= 0 def add(self, ids: np.ndarray, topology: dict, creator: str) -> None: count = ids.size - start = self.ids.size self.ids = np.concatenate([self.ids, ids]) for name, default in (("label", self.name), ("complex", NO_ID), ("parent", NO_ID), ("edge_type", "none"), ("sibling_order", NO_ID)): # an entity without a label: its entity set's name @@ -85,7 +104,7 @@ def add(self, ids: np.ndarray, topology: dict, creator: str) -> None: self.creator = np.concatenate([self.creator, np.full(count, creator, dtype=object)]) for column in self.columns.values(): column.grow(count) - self.row.update({int(identifier): start + offset for offset, identifier in enumerate(ids.tolist())}) + self._sorted = None def remove(self, ids) -> None: keep = ~np.isin(self.ids, np.asarray(list(ids), dtype=np.int64)) @@ -95,7 +114,7 @@ def remove(self, ids) -> None: self.creator = self.creator[keep] for column in self.columns.values(): column.keep(keep) - self.row = {int(identifier): row for row, identifier in enumerate(self.ids.tolist())} + self._sorted = None def place(self, participant: str, name: str, rows: np.ndarray, values, variable) -> None: key = (participant, name) @@ -117,6 +136,7 @@ def __init__(self, ranks: dict): self.set_of = {} # (data structure, id): entity set name self.removed_log = [] # (data structure, entity set, ids) self.removed_ids = set() # (data structure, id) of every entity removed + self.mappings = {} # name: the last Mapping its provider sent def entity_set(self, data_structure: str, name: str, rank=None) -> EntitySetStore: key = (data_structure, name) @@ -139,6 +159,15 @@ def merge(self, participant: str, declaration, frame: Frame, id_offset: int) -> for entity_set in sorted(data_structure.entity_sets, key=lambda item: (item.rank if item.rank is not None else 0)): self._merge_set(participant, declaration, data_structure.id, entity_set, provided, id_offset, where) + declared = {mapping.name: mapping for mapping in getattr(declaration, "mappings", [])} + for mapping in frame.mappings: + spec = declared.get(mapping.name) + if spec is None: + raise CouplingError(f"{where}: mapping {mapping.name} is not one it declared") + if (tuple(mapping.source), tuple(mapping.target)) != (spec.source, spec.target): + raise CouplingError(f"{where}: mapping {mapping.name} joins {mapping.source} and {mapping.target}, " + f"declared {spec.source} and {spec.target}") + self.mappings[mapping.name] = mapping if frame.update is not None: for (data_structure, set_name), ids in sorted(frame.update.removed.items(), key=lambda item: self.ranks.get(item[0]) or 0): @@ -151,7 +180,7 @@ def _merge_set(self, participant, declaration, data_structure, entity_set, provi raise CouplingError(f"{where}: {data_structure}/{entity_set.name} has no 'id' column") ids = np.asarray(entity_set.columns["id"], dtype=np.int64) rights = declaration.rights(*key) - known = np.array([int(identifier) in store.row for identifier in ids.tolist()], dtype=bool) + known = store.contains(ids) new = ids[~known] if new.size: if rights == "follows": @@ -186,8 +215,12 @@ def _merge_set(self, participant, declaration, data_structure, entity_set, provi for name in ("complex", "parent"): if name in entity_set.columns: values = np.asarray(entity_set.columns[name], dtype=np.int64) - missing = [int(value) for value in values.tolist() if value != NO_ID - and (data_structure, int(value)) not in self.set_of] + values = values[values != NO_ID] + found = np.zeros(values.size, dtype=bool) + for (other_structure, _), other in self.sets.items(): + if other_structure == data_structure and values.size: + found |= other.contains(values) + missing = values[~found].tolist() if missing: raise CouplingError(f"{where}: {data_structure}/{entity_set.name}.{name} refers to unknown " f"entities {missing[:5]}") @@ -205,7 +238,7 @@ def _remove(self, participant, declaration, data_structure, set_name, ids, where if (data_structure, int(identifier)) not in self.removed_ids] # e.g. with their complex already if not ids: return - if store is None or any(identifier not in store.row for identifier in ids): + if store is None or not store.contains(ids).all(): raise CouplingError(f"{where}: removes entities not in {data_structure}/{set_name}") if declaration.rights(data_structure, set_name) != "owns": created = store.creator[store.rows(ids)] @@ -240,18 +273,24 @@ def remove(self, data_structure: str, set_name: str, ids) -> None: # ── a participant's inputs ───────────────────────────────────────────────────── def ancestors(self, data_structure: str, ids: np.ndarray, target: str) -> np.ndarray: """Each id's complex (or complex's complex, …) in the entity set *target*, NO_ID if none.""" + ids = np.asarray(ids, dtype=np.int64) found = np.full(ids.size, NO_ID, dtype=np.int64) - for position, identifier in enumerate(ids.tolist()): - current = identifier - while current != NO_ID: - name = self.set_of.get((data_structure, current)) - if name is None: - break + current, active = ids.copy(), ids != NO_ID + stores = [(name, store) for (structure, name), store in self.sets.items() if structure == data_structure] + for _ in range(len(stores) + 1): # one scale up per pass + if not active.any(): + break + snapshot, moved = current.copy(), np.zeros(ids.size, dtype=bool) + for name, store in stores: + rows = store.rows(snapshot, strict=False) + here = active & (rows >= 0) if name == target: - found[position] = current - break - store = self.sets[(data_structure, name)] - current = int(store.topology["complex"][store.row[current]]) + found[here] = snapshot[here] + active[here] = False + else: + current[here] = store.topology["complex"][rows[here]] + moved |= here + active &= moved & (current != NO_ID) return found def values(self, coupling) -> tuple: @@ -264,9 +303,10 @@ def values(self, coupling) -> tuple: if column is None: return np.full(count, np.nan), np.ones(count, dtype=bool) values, present = column.values, column.present + if coupling.mapping is not None: + return self._through_mapping(coupling, origin, values, present, target) if coupling.aggregation is None: - rows = np.array([origin.row.get(int(identifier), -1) for identifier in target.ids.tolist()], - dtype=np.int64) + rows = origin.rows(target.ids, strict=False) found = rows >= 0 out = _empty(values, count) out[found] = values[rows[found]] @@ -274,7 +314,7 @@ def values(self, coupling) -> tuple: missing[found] |= ~present[rows[found]] elif coupling.aggregation == "copy": above = self.ancestors(variable.data_structure, target.ids, source.entity_set) - rows = np.array([origin.row.get(int(identifier), -1) for identifier in above.tolist()], dtype=np.int64) + rows = origin.rows(above, strict=False) found = rows >= 0 out = _empty(values, count) out[found] = values[rows[found]] @@ -285,22 +325,58 @@ def values(self, coupling) -> tuple: raise CouplingError(f"{coupling.receiver}.{variable.name}: {coupling.aggregation} of non-numeric " f"values") above = self.ancestors(variable.data_structure, origin.ids, variable.entity_set) - out = np.full(count, np.nan) - missing = np.ones(count, dtype=bool) - groups = {} - for row, identifier in enumerate(above.tolist()): - if identifier != NO_ID and present[row]: - groups.setdefault(identifier, []).append(values[row]) - reduce = {"sum": np.sum, "mean": np.mean, "max": np.max, "min": np.min}[coupling.aggregation] - for identifier, group in groups.items(): - row = target.row.get(identifier) - if row is not None: - out[row] = reduce(np.asarray(group, dtype=float)) - missing[row] = False + target_rows = target.rows(above, strict=False) + used = (target_rows >= 0) & present + target_rows, x = target_rows[used], values[used].astype(float) + counts = np.bincount(target_rows, minlength=count) + missing = counts == 0 + if coupling.aggregation in ("sum", "mean"): + out = np.bincount(target_rows, weights=x, minlength=count) + if coupling.aggregation == "mean": + out = np.divide(out, counts, out=np.zeros(count), where=counts > 0) + else: + start = -np.inf if coupling.aggregation == "max" else np.inf + out = np.full(count, start) + (np.maximum if coupling.aggregation == "max" else np.minimum).at(out, target_rows, x) + out = np.where(missing, np.nan, out) if coupling.factor != 1. and out.dtype.kind in "fi": out = out.astype(np.float64) * coupling.factor return out, missing + def _through_mapping(self, coupling, origin, values, present, target) -> tuple: + """An input from another data structure through a mapping: "sum" shares each source's value among its + targets in proportion to their weights; "mean" gives each target its sources' weighted mean.""" + count = len(target) + out, missing = np.full(count, np.nan), np.ones(count, dtype=bool) + mapping = self.mappings.get(coupling.mapping) + if mapping is None: + return out, missing + if values.dtype.kind not in "fi": + raise CouplingError(f"{coupling.receiver}.{coupling.variable.name}: non-numeric values through a " + "mapping") + sources, targets = np.asarray(mapping.columns["source"]), np.asarray(mapping.columns["target"]) + if coupling.reverse: + sources, targets = targets, sources + weights = np.asarray(mapping.columns["weight"], dtype=float) + source_rows = origin.rows(sources, strict=False) + target_rows = target.rows(targets, strict=False) + used = (source_rows >= 0) & (target_rows >= 0) + used[used] &= present[source_rows[used]] + source_rows, target_rows, weights = source_rows[used], target_rows[used], weights[used] + x = values[source_rows].astype(float) + if coupling.aggregation == "sum": # each source's amount shared among its targets + totals = np.bincount(source_rows, weights=weights, minlength=len(origin)) + shares = np.divide(weights, totals[source_rows], out=np.zeros_like(weights), + where=totals[source_rows] > 0) + out_values = np.bincount(target_rows, weights=x * shares, minlength=count) + else: # each target's weighted mean + weighted = np.bincount(target_rows, weights=x * weights, minlength=count) + totals = np.bincount(target_rows, weights=weights, minlength=count) + out_values = np.divide(weighted, totals, out=np.full(count, np.nan), where=totals > 0) + reached = np.bincount(target_rows, minlength=count) > 0 + out[reached], missing[reached] = out_values[reached], False + return out * coupling.factor, missing + def inputs(self, receiver: str, declaration, couplings, removed_since: int, time: float, time_unit: str, step: int) -> Frame: """*receiver*'s turn frame: the entity sets it works on (all their entities and topology) with its inputs, @@ -361,7 +437,8 @@ def frame(self, time: float, time_unit: str, step: int) -> Frame: if (participant, variable) in store.declarations: entity_set.variables[column_name] = store.declarations[(participant, variable)] structures.setdefault(data_structure, DataStructure(data_structure)).entity_sets.append(entity_set) - return Frame(data_structures=list(structures.values()), time=time, time_unit=time_unit, step=step, + return Frame(data_structures=list(structures.values()), mappings=list(self.mappings.values()), time=time, + time_unit=time_unit, step=step, writer={"platform": "fspm-tf", "adapter": "openalea.fspm_tf.coupling"}) diff --git a/bindings/python/src/openalea/fspm_tf/dump.py b/bindings/python/src/openalea/fspm_tf/dump.py index 776f6b2..f561f3b 100644 --- a/bindings/python/src/openalea/fspm_tf/dump.py +++ b/bindings/python/src/openalea/fspm_tf/dump.py @@ -94,7 +94,12 @@ def frame_to_dict(frame: Frame) -> dict: for name, values in mesh.display.items()}} if mesh.display else {})} for mesh in data_structure.meshes] data_structures.append(entry) - return {"time": frame.time, "time_unit": frame.time_unit, "step": frame.step, - "data_structures": data_structures, - "mappings": [dict(_table(mapping), source=list(mapping.source), target=list(mapping.target), - method=mapping.method) for mapping in frame.mappings]} + out = {"time": frame.time, "time_unit": frame.time_unit, "step": frame.step, + "data_structures": data_structures, + "mappings": [dict(_table(mapping), source=list(mapping.source), target=list(mapping.target), + method=mapping.method) for mapping in frame.mappings]} + if frame.update is not None: # a partial frame: its sender and removed entities + out["update"] = {"participant": frame.update.participant, + "removed": [{"data_structure": key[0], "entity_set": key[1], "ids": ids.tolist()} + for key, ids in sorted(frame.update.removed.items())]} + return out diff --git a/bindings/python/src/openalea/fspm_tf/model.py b/bindings/python/src/openalea/fspm_tf/model.py index 6dca612..e2f339f 100644 --- a/bindings/python/src/openalea/fspm_tf/model.py +++ b/bindings/python/src/openalea/fspm_tf/model.py @@ -247,7 +247,7 @@ def _as_column(name: str, values) -> np.ndarray: if array.ndim == 0: array = array.reshape(1) if array.dtype.kind in "OU": - if not all(isinstance(value, str) or value is None for value in array.reshape(-1)): + if not set(map(type, array.reshape(-1).tolist())) <= {str, type(None)}: raise TypeError(f"column '{name}': object columns must hold strings (None: missing)") array = array.astype(object) elif np.ma.isMaskedArray(array) and array.dtype.kind == "f": diff --git a/bindings/python/test/test_coupling.py b/bindings/python/test/test_coupling.py index f79555e..d4578d9 100644 --- a/bindings/python/test/test_coupling.py +++ b/bindings/python/test/test_coupling.py @@ -266,3 +266,169 @@ def test_the_command_line(tmp_path): assert code == 0 assert validate_sequence(str(tmp_path / "again" / "scene.fspm.json")).ok assert (tmp_path / "session" / "reader" / "to_coordinator" / "00000000.json").exists() # the session's log + + +WATER = np.array([0.1, 0.2, 0.3]) # the soil's three cells + + +def run_plant(endpoint, log): + """Two organs, then three; each takes up 1 mol per step and logs the water it receives from the soil.""" + from openalea.fspm_tf.coupling import Declaration as D + declaration = D("plant", "test", [EntitySetSpec("plant", "Organ", rights="owns")], [ + VariableSpec("plant", "Organ", "uptake", "provides", "mol", kind="extensive"), + VariableSpec("plant", "Organ", "water", "requires", "m3/m3", kind="intensive")]) + participant = Participant.connect(declaration, endpoint, timeout=60) + for turn in participant.turns(): + organs = turn.frame.data_structure("plant").entity_set("Organ") if turn.frame.data_structures else None + if organs is not None and organs.count: + log[turn.step] = dict(zip(organs.columns["id"].tolist(), np.asarray(organs.columns["water"]).tolist())) + count = 2 if turn.step < 2 else 3 + ids = participant.id_offset + np.arange(count) + participant.send(fspm_tf.Frame(data_structures=[fspm_tf.DataStructure("plant", entity_sets=[ + fspm_tf.EntitySet("Organ", rank=5, columns={"id": ids, "uptake": np.ones(count)})])])) + + +def run_soil(endpoint, log): + """Three cells; organ i in cells i % 3 (weight 1) and (i + 1) % 3 (weight 3): the mapping it provides.""" + from openalea.fspm_tf.coupling import MappingSpec + declaration = Declaration("soil", "test", [EntitySetSpec("soil", "Cell", rank=1, rights="owns"), + EntitySetSpec("plant", "Organ")], [ + VariableSpec("soil", "Cell", "water_content", "provides", "m3/m3", kind="intensive"), + VariableSpec("soil", "Cell", "uptake", "requires", "mol", kind="extensive")], + mappings=[MappingSpec("organs_in_soil", ("plant", "Organ"), ("soil", "Cell"))]) + participant = Participant.connect(declaration, endpoint, timeout=60) + cells = participant.id_offset + np.arange(3) + for turn in participant.turns(): + if turn.frame.data_structures and any(item.id == "soil" for item in turn.frame.data_structures): + soil = turn.frame.data_structure("soil").entity_set("Cell") + log[turn.step] = np.asarray(soil.columns["uptake"]).tolist() + organs = [] + if any(item.id == "plant" for item in turn.frame.data_structures): + organs = turn.frame.data_structure("plant").entity_set("Organ").columns["id"].tolist() + source = np.repeat(np.array(organs, dtype=np.int64), 2) + target = np.array([cells[(i + k) % 3] for i in range(len(organs)) for k in (0, 1)], dtype=np.int64) + weight = np.tile([1., 3.], len(organs)) + frame = fspm_tf.Frame(data_structures=[fspm_tf.DataStructure("soil", kind="table", entity_sets=[ + fspm_tf.EntitySet("Cell", rank=1, columns={"id": cells, "water_content": WATER})])]) + frame.mappings.append(fspm_tf.Mapping("organs_in_soil", source=("plant", "Organ"), target=("soil", "Cell"), + columns={"source": source, "target": target, "weight": weight})) + participant.send(frame) + + +def test_a_link_between_two_data_structures(endpoint, tmp_path): + """The plant's organs and the soil's cells, two data structures of two participants, linked by the mapping the + soil provides: the organs' water, the cells' weighted mean; the cells' uptake, the organs' shared by weight.""" + links = [Link("plant", "water", "soil", "water_content", mapping="organs_in_soil"), + Link("soil", "uptake", "plant", "uptake", mapping="organs_in_soil")] + coordinator = Coordinator(endpoint, steps=4, interval=1., expect=["plant", "soil"], links=links, timeout=60, + record=str(tmp_path / "record")) + logs = {"plant": {}, "soil": {}} + threads = [threading.Thread(target=target, args=(coordinator.endpoint, logs[name]), daemon=True) + for name, target in (("plant", run_plant), ("soil", run_soil))] + for thread in threads: + thread.start() + coordinator.run() + for thread in threads: + thread.join(30) + first = 2 ** 32 + # explicit coupling, the stage [plant, soil]: at step 4 the mapping the soil sent at step 3 (from the plant's + # three organs of step 2) and the uptake the plant sent at step 3 + water = logs["plant"][4] + for i in range(3): + assert water[first + i] == pytest.approx((WATER[i % 3] + 3 * WATER[(i + 1) % 3]) / 4) + uptake = logs["soil"][4] + assert uptake == pytest.approx([1., 1., 1.]) # 1/4 of one organ and 3/4 of another, each + assert validate_sequence(str(tmp_path / "record" / "scene.fspm.json")).ok + with pytest.raises(CouplingError, match="need a mapping"): + resolve({"plant": Declaration.from_json(Declaration("plant", entity_sets=[], variables=[ + VariableSpec("plant", "Organ", "water", "requires", "1")]).to_json()), + "soil": Declaration("soil", variables=[VariableSpec("soil", "Cell", "water_content", "provides", + "1")]).complete()}, + [Link("plant", "water", "soil", "water_content")]) + + +def test_the_conformance_sessions(tmp_path): + """Python's participant against conformance/coupling, over both transports (fspm-tf couple-conformance).""" + import sys + from openalea.fspm_tf.cli import main + sessions = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "conformance", "coupling") + command = f"{sys.executable} -m openalea.fspm_tf.coupling.conformance subject" + assert main(["couple-conformance", sessions, "--participant", command, "--out", str(tmp_path)]) == 0 + + +def test_the_conformance_sessions_are_what_their_generator_writes(tmp_path): + import importlib.util + folder = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "conformance", "coupling") + spec = importlib.util.spec_from_file_location("coupling_generate", os.path.join(folder, "generate.py")) + generate = importlib.util.module_from_spec(spec) + spec.loader.exec_module(generate) + for name, session in generate.SESSIONS.items(): + generate.generate(name, session, str(tmp_path)) + for root, _, files in os.walk(tmp_path / name): + for file_name in files: + written = os.path.join(root, file_name) + committed = os.path.join(folder, os.path.relpath(written, tmp_path)) + with open(written, "rb") as left, open(committed, "rb") as right: + assert left.read() == right.read(), committed + + +# Platforms' logs replayed (their index, scene.fspm.json): CPlantBox's (bindings/cpp/cplantbox's tests write +# anagallis_log/) and GroIMP's (bindings/java/groimp-plugin/test/run_in_groimp.sh writes log/), in their CI jobs +PLATFORM_LOGS = {"conformance": os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, + "conformance", "growing_sequence", "scene.fspm.json"), + "CPlantBox": os.environ.get("FSPM_TF_CPLANTBOX_LOG"), "GroIMP": os.environ.get("FSPM_TF_GROIMP_LOG")} + + +def _sorted(values): + """The values, missing ones (NaN, masked) as None, sorted: to compare a column whatever its rows' order.""" + def plain(value): + if isinstance(value, list): + return [plain(item) for item in value] + return None if value is None or (isinstance(value, float) and value != value) else value + values = np.ma.filled(np.ma.asarray(values, dtype=object), None).tolist() if np.ma.isMaskedArray(values) \ + else np.asarray(values).tolist() + return sorted((plain(value) for value in values), key=repr) + + +@pytest.mark.parametrize("platform", sorted(PLATFORM_LOGS)) +def test_a_platform_log_replayed(platform, tmp_path): + """A platform's log as a participant: a reader following every entity set and requiring every variable gets, at + each step, the logged entities (renumbered) and values.""" + from openalea.fspm_tf.coupling import Replay + from openalea.fspm_tf.codec import read_index + index = PLATFORM_LOGS[platform] + if not index: + pytest.skip(f"no {platform} log (FSPM_TF_{platform.upper()}_LOG)") + entries = read_index(index)["frames"] + times = [entry["time"] for entry in entries] + replay = Replay("log", index) # its declaration: what the reader requires + declaration = Declaration("reader", "test", [EntitySetSpec(item.data_structure, item.name, item.rank) + for item in replay.declaration.entity_sets], + [VariableSpec(item.data_structure, item.entity_set, item.name, "requires", item.unit, + kind=item.kind) for item in replay.declaration.variables]) + log = {} + + def reader(endpoint): + participant = Participant.connect(declaration, endpoint, timeout=120) + for turn in participant.turns(): + log[turn.step] = {(data_structure.id, entity_set.name): entity_set + for data_structure in turn.frame.data_structures + for entity_set in data_structure.entity_sets} + participant.send(fspm_tf.Frame()) + + interval = times[1] - times[0] if len(times) > 1 else 1. + coordinator = Coordinator("tcp://127.0.0.1:*", steps=len(entries) - 1, interval=interval, start=times[0], + time_unit=read_index(index).get("timeUnit", "d"), expect=["reader"], + replays={"log": index}, timeout=120) + thread = threading.Thread(target=reader, args=(coordinator.endpoint,), daemon=True) + thread.start() + coordinator.run() + thread.join(60) + for step, entry in enumerate(entries): + frame = fspm_tf.read(entry["path"]) + for data_structure in frame.data_structures: + for entity_set in data_structure.entity_sets: + received = log[step][(data_structure.id, entity_set.name)] + assert received.count == entity_set.count, (step, entity_set.name) + for name in entity_set.variable_names(): + assert _sorted(received.columns[name]) == _sorted(entity_set.columns[name]), (step, name) diff --git a/conformance/README.md b/conformance/README.md index 8588750..16ab82a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -15,6 +15,14 @@ numbers, the same bytes on every run; the CI checks that the committed files are | `missing_values` | integer, text, boolean and float columns with missing values; a date column (`format`); `unknown` edge types and a `sibling_order` column; a `mixed` entity set; a labels map; a parent in another axis; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system, ids kept, and their index | +## Coupling sessions + +`coupling/` holds the coupling conformance sessions (specification §9, §11), which `coupling/generate.py` writes: each +session's peer (a sequence the coordinator replays), its subject's declaration and updates, and the expected values of +what the subject reads (`expected/turn_.json`) and of the session's record (`expected/record/`). A codec's +participant runs them with `fspm-tf couple-conformance conformance/coupling --participant COMMAND`. The frame tools +(`compare-all`, each codec's conformance run) leave this folder out. + ## The expected values `x.expected.json` is the frame `x.glb` holds, as plain JSON (written by `openalea.fspm_tf.dump.frame_to_dict`): @@ -37,7 +45,9 @@ numbers, the same bytes on every run; the CI checks that the committed files are "grid": {"kind", "shape", "origin", "spacing", "axes", "periodic", "z_up", "max_level", "cells"}, "meshes": [{"name": "...", "positions_float32": [[x, y, z], ...], "feature_rows": {"SubOrgan": [...]}, "indices": [...], "display": {"name": [...]}}]}], - "mappings": [... like entity sets, with "source", "target" ([data structure, entity set]) and "method"]} + "mappings": [... like entity sets, with "source", "target" ([data structure, entity set]) and "method"], + "update": {"participant": "...", (a partial frame only, §11.1) + "removed": [{"data_structure": "...", "entity_set": "...", "ids": [...]}, ...]}} (sorted) ``` - **Floats** are the shortest decimals giving the same double, compared exactly. diff --git a/conformance/adaptive_soil.glb b/conformance/adaptive_soil.glb index 0e5b314f5138926adbd4ac301ab8f975aa6d15b1..cbad5e3d2e9d97f5866d35dba0f9436c85f305dd 100644 GIT binary patch delta 19 acmew(^haodE2Hs7w};G(#+x-*LRbMz_XceM delta 19 acmew(^haodE2Ggyw};G(Mw>NQLRbMz.json, expected/record/frame_.json: not *.expected.json, which are the frames'). + +The subject's updates and expected values come from running the session once with a reference subject (the +behaviour each session describes, in Python), over a folder. Every run writes the same bytes. + + python conformance/coupling/generate.py +""" +import json +import os +import shutil +import sys +import tempfile +import threading + +import numpy as np + +from openalea.fspm_tf import (DataStructure, EntitySet, Frame, Mapping, NO_ID, Variable, read, read_index, to_bytes, + write_sequence) +from openalea.fspm_tf.coupling import (Coordinator, Declaration, EntitySetSpec, Link, MappingSpec, Participant, + VariableSpec) +from openalea.fspm_tf.dump import frame_to_dict + +DIRECTORY = os.path.dirname(os.path.abspath(__file__)) +STEPS = 4 +OFFSET = 2 ** 32 # the subject's id offset: the first participant expected +PEER_OFFSET = 2 * 2 ** 32 # the peer's: the replay, after it + + +def _json(value) -> str: + return json.dumps(value, indent=1, ensure_ascii=False) + "\n" + + +# ── two_scales: the subject refines the peer's organs into segments ────────────────────────────────────── +def two_scales_peer() -> list: + """The grower: one organ more each step, in a chain, each 2 cm longer than the previous; the first removed at + step 3.""" + frames, organs, next_id = [], [], 0 + for step in range(STEPS + 1): + organs.append(next_id) + next_id += 1 + if step == 3: + organs.pop(0) + count = len(organs) + organ_set = EntitySet("Organ", rank=5, columns={ + "id": np.array(organs, dtype=np.int64), "label": np.array(["Leaf"] * count, dtype=object), + "parent": np.array([NO_ID] + organs[:-1], dtype=np.int64), + "edge_type": np.array(["none"] + ["successor"] * (count - 1), dtype=object), + "length": 2. * np.arange(1, count + 1)}, + variables={"length": Variable(unit="cm", standard_name="length", kind="extensive")}) + frames.append(Frame([DataStructure("plant", entity_sets=[organ_set])], time=float(step), step=step, + writer={"platform": "conformance"})) + return frames + + +def two_scales_declaration() -> Declaration: + return Declaration("subject", "conformance", [EntitySetSpec("plant", "Organ"), + EntitySetSpec("plant", "SubOrgan", rights="refines")], [ + VariableSpec("plant", "SubOrgan", "segment_length", "requires", "m", "length"), + VariableSpec("plant", "SubOrgan", "sugar", "provides", "mol", kind="extensive")]) + + +def two_scales_subject(): + """Two segments under each new organ; each segment's sugar 1e-6 mol per m of its length.""" + segments = {} + + def answer(turn, offset): + plant = turn.frame.data_structure("plant") + for segment in turn.frame.update.removed.get(("plant", "SubOrgan"), np.zeros(0)).tolist(): + segments.pop(segment, None) + lengths = {} + if any(item.name == "SubOrgan" for item in plant.entity_sets): + known = plant.entity_set("SubOrgan") + lengths = dict(zip(known.columns["id"].tolist(), np.asarray(known.columns["segment_length"]).tolist())) + with_segments = {organ for organ, _ in segments.values()} + for organ in plant.entity_set("Organ").columns["id"].tolist(): + if organ not in with_segments: + first = offset + 1000 * turn.step + len(segments) + segments[first], segments[first + 1] = (organ, NO_ID), (organ, first) + ids = list(segments) + columns = {"id": np.array(ids, dtype=np.int64), + "complex": np.array([segments[segment][0] for segment in ids], dtype=np.int64), + "parent": np.array([segments[segment][1] for segment in ids], dtype=np.int64), + "edge_type": np.array(["none" if segments[segment][1] == NO_ID else "successor" + for segment in ids], dtype=object), + "label": np.array(["Segment"] * len(ids), dtype=object), + "sugar": np.array([1e-6 * np.nan_to_num(lengths.get(segment, 0.)) for segment in ids])} + return Frame([DataStructure("plant", entity_sets=[EntitySet("SubOrgan", rank=6, columns=columns)])]) + return answer + + +# ── two_data_structures: the subject's soil linked to the peer's plant by the mapping it provides ──────────── +def two_data_structures_peer() -> list: + """The plant: two organs, then three from step 2, each taking up 1 mol per step.""" + frames = [] + for step in range(STEPS + 1): + count = 2 if step < 2 else 3 + organ_set = EntitySet("Organ", rank=5, columns={"id": np.arange(count, dtype=np.int64), + "uptake": np.ones(count)}, + variables={"uptake": Variable(unit="mol", kind="extensive")}) + frames.append(Frame([DataStructure("plant", entity_sets=[organ_set])], time=float(step), step=step, + writer={"platform": "conformance"})) + return frames + + +def two_data_structures_declaration() -> Declaration: + return Declaration("subject", "conformance", [EntitySetSpec("soil", "Cell", rank=1, rights="owns"), + EntitySetSpec("plant", "Organ")], [ + VariableSpec("soil", "Cell", "water_content", "provides", "m3/m3", kind="intensive"), + VariableSpec("soil", "Cell", "uptake", "requires", "mol", kind="extensive")], + mappings=[MappingSpec("organs_in_soil", ("plant", "Organ"), ("soil", "Cell"))]) + + +def two_data_structures_subject(): + """Three cells; organ i in cells i % 3 (weight 1) and (i + 1) % 3 (weight 3).""" + water = np.array([0.1, 0.2, 0.3]) + + def answer(turn, offset): + cells = offset + np.arange(3) + organs = [] + if any(item.id == "plant" for item in turn.frame.data_structures): + organs = turn.frame.data_structure("plant").entity_set("Organ").columns["id"].tolist() + frame = Frame([DataStructure("soil", kind="table", entity_sets=[ + EntitySet("Cell", rank=1, columns={"id": cells, "water_content": water})])]) + frame.mappings.append(Mapping("organs_in_soil", source=("plant", "Organ"), target=("soil", "Cell"), columns={ + "source": np.repeat(np.array(organs, dtype=np.int64), 2), + "target": np.array([cells[(i + k) % 3] for i in range(len(organs)) for k in (0, 1)], dtype=np.int64), + "weight": np.tile([1., 3.], len(organs))})) + return frame + return answer + + +SESSIONS = { + "two_scales": dict( + peer=two_scales_peer, declaration=two_scales_declaration, subject=two_scales_subject, + links=[Link("subject", "segment_length", "peer", "length", factor=0.5, aggregation="copy")], + covers="a subject refining a replayed peer's organs into segments: new entities, removals cascading to the " + "subject's segments, a unit conversion and a factor, a copy to finer entities"), + "two_data_structures": dict( + peer=two_data_structures_peer, declaration=two_data_structures_declaration, + subject=two_data_structures_subject, + links=[Link("subject", "uptake", "peer", "uptake", mapping="organs_in_soil")], + covers="a subject owning another data structure, linked to the peer's by the mapping it provides: a sum " + "through a mapping, the subject's mappings in its updates"), +} + + +def generate(name: str, spec: dict, directory: str = DIRECTORY) -> None: + folder = os.path.join(directory, name) + for part in ("peer", "subject", "expected"): + if os.path.isdir(os.path.join(folder, part)): + shutil.rmtree(os.path.join(folder, part)) + os.makedirs(os.path.join(folder, "subject")) + os.makedirs(os.path.join(folder, "expected", "record")) + write_sequence(os.path.join(folder, "peer"), spec["peer"]()) + declaration = spec["declaration"]() + with open(os.path.join(folder, "subject", "declaration.json"), "w", encoding="utf-8") as file: + file.write(_json(declaration.to_json())) + settings = {"steps": STEPS, "interval": 1., "timeUnit": "d", "links": [link.to_json() for link in spec["links"]], + "covers": spec["covers"]} + with open(os.path.join(folder, "session.json"), "w", encoding="utf-8") as file: + file.write(_json(settings)) + + answer = spec["subject"]() + + def reference(endpoint): + participant = Participant.connect(spec["declaration"](), endpoint, timeout=60) + for turn in participant.turns(): + with open(os.path.join(folder, "expected", f"turn_{turn.step:04d}.json"), "w", + encoding="utf-8") as file: + file.write(_json(frame_to_dict(turn.frame))) + update = answer(turn, participant.id_offset) + update.update = None + with open(os.path.join(folder, "subject", f"update_{turn.step:04d}.glb"), "wb") as file: + file.write(to_bytes(update)) + participant.send(update) + + with tempfile.TemporaryDirectory() as temporary: + coordinator = Coordinator("files:" + os.path.join(temporary, "messages"), steps=STEPS, interval=1., + expect=["subject"], links=spec["links"], timeout=60, + replays={"peer": os.path.join(folder, "peer", "scene.fspm.json")}, + record=os.path.join(temporary, "record")) + thread = threading.Thread(target=reference, args=(coordinator.endpoint,), daemon=True) + thread.start() + coordinator.run() + thread.join(60) + for entry in read_index(os.path.join(temporary, "record", "scene.fspm.json"))["frames"]: + path = os.path.join(folder, "expected", "record", entry["file"].replace(".glb", ".json")) + with open(path, "w", encoding="utf-8") as file: + file.write(_json(frame_to_dict(read(entry["path"])))) + assert participant_offset_ok(folder) + + +def participant_offset_ok(folder: str) -> bool: + """The subject's ids in its updates are in its range (the first participant expected's).""" + for name in sorted(os.listdir(os.path.join(folder, "subject"))): + if name.endswith(".glb"): + frame = read(os.path.join(folder, "subject", name)) + for data_structure in frame.data_structures: + for entity_set in data_structure.entity_sets: + ids = entity_set.columns["id"] + if ids.size and not ((ids >= OFFSET) & (ids < OFFSET + 2 ** 32)).all(): + return False + return True + + +def main() -> int: + for name, spec in SESSIONS.items(): + generate(name, spec) + print(name) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00000.json b/conformance/coupling/two_data_structures/expected/record/frame_00000.json new file mode 100644 index 0000000..5a3d304 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00000.json @@ -0,0 +1,219 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 0, + "columns": { + "source": { + "dtype": "int64", + "values": [] + }, + "target": { + "dtype": "int64", + "values": [] + }, + "weight": { + "dtype": "float64", + "values": [] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00001.json b/conformance/coupling/two_data_structures/expected/record/frame_00001.json new file mode 100644 index 0000000..7b0e0a1 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00001.json @@ -0,0 +1,234 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 4, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00002.json b/conformance/coupling/two_data_structures/expected/record/frame_00002.json new file mode 100644 index 0000000..013a26c --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00002.json @@ -0,0 +1,240 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 4, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00003.json b/conformance/coupling/two_data_structures/expected/record/frame_00003.json new file mode 100644 index 0000000..410ddb7 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00003.json @@ -0,0 +1,246 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 6, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298, + 4294967298, + 4294967296 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/record/frame_00004.json b/conformance/coupling/two_data_structures/expected/record/frame_00004.json new file mode 100644 index 0000000..9dc2df3 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/record/frame_00004.json @@ -0,0 +1,246 @@ +{ + "time": 4.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + } + ] + }, + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "water_content": { + "dtype": "float64", + "values": [ + 0.1, + 0.2, + 0.3 + ] + } + }, + "variables": { + "water_content": { + "unit": "m3/m3", + "role": "state", + "kind": "intensive" + } + }, + "rank": 1 + } + ] + } + ], + "mappings": [ + { + "name": "organs_in_soil", + "count": 6, + "columns": { + "source": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "target": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967297, + 4294967298, + 4294967298, + 4294967296 + ] + }, + "weight": { + "dtype": "float64", + "values": [ + 1.0, + 3.0, + 1.0, + 3.0, + 1.0, + 3.0 + ] + } + }, + "source": [ + "plant", + "Organ" + ], + "target": [ + "soil", + "Cell" + ], + "method": null + } + ] +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0000.json b/conformance/coupling/two_data_structures/expected/turn_0000.json new file mode 100644 index 0000000..e3d558a --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0000.json @@ -0,0 +1,11 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0001.json b/conformance/coupling/two_data_structures/expected/turn_0001.json new file mode 100644 index 0000000..9182413 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0001.json @@ -0,0 +1,181 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + null, + null, + null + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0002.json b/conformance/coupling/two_data_structures/expected/turn_0002.json new file mode 100644 index 0000000..03e5f83 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0002.json @@ -0,0 +1,181 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 0.25, + 1.0, + 0.75 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0003.json b/conformance/coupling/two_data_structures/expected/turn_0003.json new file mode 100644 index 0000000..32fa5f4 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0003.json @@ -0,0 +1,186 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 0.25, + 1.0, + 0.75 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/expected/turn_0004.json b/conformance/coupling/two_data_structures/expected/turn_0004.json new file mode 100644 index 0000000..d87bf54 --- /dev/null +++ b/conformance/coupling/two_data_structures/expected/turn_0004.json @@ -0,0 +1,186 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "soil", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Cell", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294967298 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Cell", + "Cell", + "Cell" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + }, + "uptake": { + "dtype": "float64", + "values": [ + 1.0, + 1.0, + 1.0 + ] + } + }, + "variables": { + "uptake": { + "unit": "mol", + "role": "input", + "kind": "extensive" + } + }, + "rank": 1 + } + ] + }, + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Organ", + "Organ", + "Organ" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "none", + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_data_structures/peer/frame_000000.glb b/conformance/coupling/two_data_structures/peer/frame_000000.glb new file mode 100644 index 0000000000000000000000000000000000000000..194496bc9c68a9fd5d8e37b2c57e78fd03456cd4 GIT binary patch literal 1420 zcmah};cDA35Y7gBk8r=;lB8Q0{!_ZKLPyt>B^`qjg5sPV5nJ{oxorr3gFS2?VRw>} zI3#5PCnw#V?(Vzql;xA__nl7X?}tw3p1$wn(cK1wF=!#)z!PdCO9e3u&U+9+h6=T? zrKXeARK=N1yS?)ZXd*BP=^*X}$w2(L0aY%LrQWoeM!OxIi-K(36*KP`K_RS`FJ#<5 z??t^6-&tGHmfOa3PLYzO%oiFd@yGtOiFN_2Tqp~i3%*!XT$_0?xF9?D_IoljR@aHG zwa8}$TCosTP}SGt`yV0A15ZZiEiY1;WQm1fBS`@ci%oLIn4sB!%V{0INVT52wj>cS?1y0T(L zuK&N{`gU~1mo&5pEmapTSqB9!GMxDwh$14dF~ohKz@I6uI|~f(omQ0IRzW0bwZzm( zg2oV~f*-{FmU^y*N|t-wprxzTN|n!Que@lcF5Q6HdLrhzViy?Qn5s^AkQ6hbR1|KYISt;RF7aK6|*nXqGA|JEi>un;>3iV8mc$Wrc4Se z?Rt#%pwnC4Qr}h(f7&7Go56LFE58feXA+-;mZ43@v#{rD?eQtfoQlw8&ipr4SY}ly zF^u#R!NMzod7Y;8fYESoV7|6^iz>5Ah%W{l*lT}Dd=3`Qni3=x8IRJ`J%VAo_3aOt kfZq4Wwmrol{d%_*-2dyP-@ab{12vAj5dZ)H literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/peer/frame_000001.glb b/conformance/coupling/two_data_structures/peer/frame_000001.glb new file mode 100644 index 0000000000000000000000000000000000000000..93262d9fa1267acfb9cb032cd22a94e635c156cf GIT binary patch literal 1420 zcmah}ZHwAK5VnQ>51G$7@uiiLFUPf1xN<#@s-0w!-|s*egBHRaJfSu+R}ezK(}e&MRH%i` zHECi~WLGwBcRK^90x%KTAnewXf$(t$DqkT>-FY<+4qv1f8Tq;+7Ctb5Ojs>n$hp_) z2HgwqRnyRvo6e-COwn4VD~(k6V{cvshlE8clm)H@Uo0x_&CCq0$Pd2#o-T~lWn@b& z(nW?=tb`TR^=R_&qb~C#k`a0!YpTu$5|~@@I844yw5;x5jKc70cA|?+jIkKS6i`J_A`2Kf?ZH z*!OKw`3)rZ+8(7Qt~J?0I3G8N_auN=J3wdvl2?Ff?8KRplo(dzVk{O81bEicBJG+Z ziH&oekxP^-mIxN)y7wlfp{7 zouECL^q#jgw+)1!4oLc8@a!p+?*(p=#3!L;J*M+rIP$gh_+)uXP3Syj{u?VSv#M^f zUg;--g`WVHWgOE5M$5f}<<{aIs>H4#9P~M{xBim&TzfcgcZ0F>IF8*VNb_mN_J&MA hAA9E88=g7Ot9G02*DtsC^y{B@d%^v`Ui;72>wh_nyAuEa literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/peer/frame_000002.glb b/conformance/coupling/two_data_structures/peer/frame_000002.glb new file mode 100644 index 0000000000000000000000000000000000000000..d1d126e8b8d5aff8f6cdab1b10d73618af7f0b63 GIT binary patch literal 1436 zcmah};cDA35Y7gBk8;1=;-uLq{xiC} zI3#5P+h^UK?(Vzqloykmx4mBP&%0jlJ3U{=qx%gAW6(mffhW{PRtgdrT*MGTjtaG~ zm8PA{)aAL&&f<$pXd^Hw=^%-NWFUFmfT|YAQao*^(e97-MM<{qh?#ecpcGci7c%Z& z#8G_WIqxdEa@UylDN?$W#X=(`{@9L#^~ z7R9VYD;B~Es`_U9{aZ-$z>^Vr&5KkfU1BNNNK$~qV$*^#CTKR`a+=LXIv14hE>-^@ zL9MGAwUyqOPuhT8*9_nI^Yz{J4@xkt%DPgh?8GZ>?M_(&b(^=;#`?Qq{BCD=*urOE+M)o``wj_;-Y=!=E5L0H1+uh#$%2 zY1p@AQTiPux4V4`Oc?a5YI^jnRb9s10;_CjaB5t2PG*mEXc%ItVu&hJ!@%^ zX3derW|c0H&68lR#GwhaxH3xvC=Y^ZZ1wt*r}m>3A0Qd~G~FWmQlSx+<9e#tO@< z3MGb-ej-?SMKEu&j4m)5?hVY>7Vl8yb_vPwiUWJ?Pl?aL!dbf;jEzTG<}N|%PuF)a lWCD8MBiq67$avbG#dN=Zy1l2jUvI*f@7?|S`_aq$?>|A^ylVge literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/peer/frame_000003.glb b/conformance/coupling/two_data_structures/peer/frame_000003.glb new file mode 100644 index 0000000000000000000000000000000000000000..19929bb0c1f5f24496de9b39483f250e2886f6b9 GIT binary patch literal 1436 zcmah}|7zPn49*67kD|Zb;{4bs{%2%kfsU>zO*#f8go`6PN9?nAcWxVk-((NDN0@ZJ zBo0a0fo=IDoup5nF3qN+_nl7X&xcOuo}TZM@!b}LF=!#$!ZT_k3k4DMu0jYPMTJ_} zLeoxS%KXwM7vWVO>HthkI*39q8Hk>?po%536wc~du>Yfdk&~@EV%|6gkPEBjD;amM z!XP~JT(%W$xou4Q6e(WGY^jkFf9lTaU>~r|gtEZ7;HyQ&wV6kQOR|G+zov6zbrsu6 zi)@~w6-!|SRXv*A|L|!Zc``z8d6CM*E6fEONeXaWY@9L11kDy)PV@O#r-Jg`r`r4n zQ0k&YZDnK3CvCxQN``Opb$B~`pakP0FAIgrP8-F|-SpFd_BH;lq~`LdF02EnN-Ngn z`u`(Fx8ornQqv-|RF%7A9Td1sao*%W6cJg4KJFt0zNWZt&oiLuw4(HO3L;6XC8mmF zG=?bU{UGYL)C(HB=8^Efupb)uLn=3oTRW=HkqRog1n*_GU~9 zEA3{2_Ndc4-csMz5PjSu*))TDPp*6~aGyzh7Fzl?ozB96uT_IjUSw2+E;8o7vBEN| ze2IRfp9vOT0W7K{p$m+Ldkc$=#amRVT|qRs=D>~ir^FX;;jG;a#>V3$ahD+Vr|sJt mFaaCi6WiY4#CTR;gmk}txw)fvv)=hH-@AwP_j4m3zW)G4`@C%c literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/peer/frame_000004.glb b/conformance/coupling/two_data_structures/peer/frame_000004.glb new file mode 100644 index 0000000000000000000000000000000000000000..0ae370df94ffbe8b455b0837b9168bdca691e229 GIT binary patch literal 1436 zcmah};cDA35Y7gBk8;1=;-uXu{xiCCK-bgg5t=Ih%I}P+%^Qi$sTf#u$?|Z%8pAWs>J)ZBQ;oXLWwbD_vp(kmqDl|njI14GDRBCC2 zD-3oLTjr-OISJ3ssSe1-&`}h&lBMWzL%LWC5=gm%Qa$QLIy*JdYHE?`IBevN0=nJRXb z5!o!4PAr5IsQP+z|D#Rwz>^Vr%L^(SFJ&&+NFl&sv2n&21DXx_oMyA3Nd@xVrP}-l zRGOlc#;L}bPuh@Mmki(N>(%Ym0}_mjyezcTZrmtt?#7=kuy5q=N*OK>b;&xQs&rxn z*Z&`JeLKA3LuxFbrK{X0>!FZ~RL+_lKoOBuvW+`I;7i2ydo2T+PHUvMRe&U&me?we zrL~|`>qk+)qn;a~z;iE7v_fE7z{EOPLW858jdBvR z=1Jm`!sIZC3(XQ>kvw{_Zi}oFsg0S>si?IwQBImiws4WB4hqrCmge? zEwLTxC&0ohpm~)fxWF*n8=9}3yp=k2OF(dZvX%Q literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/peer/scene.fspm.json b/conformance/coupling/two_data_structures/peer/scene.fspm.json new file mode 100644 index 0000000..ee7ae02 --- /dev/null +++ b/conformance/coupling/two_data_structures/peer/scene.fspm.json @@ -0,0 +1,31 @@ +{ + "fspmIndex": "0.3", + "timeUnit": "d", + "frames": [ + { + "time": 0.0, + "step": 0, + "file": "frame_000000.glb" + }, + { + "time": 1.0, + "step": 1, + "file": "frame_000001.glb" + }, + { + "time": 2.0, + "step": 2, + "file": "frame_000002.glb" + }, + { + "time": 3.0, + "step": 3, + "file": "frame_000003.glb" + }, + { + "time": 4.0, + "step": 4, + "file": "frame_000004.glb" + } + ] +} diff --git a/conformance/coupling/two_data_structures/session.json b/conformance/coupling/two_data_structures/session.json new file mode 100644 index 0000000..c70b963 --- /dev/null +++ b/conformance/coupling/two_data_structures/session.json @@ -0,0 +1,15 @@ +{ + "steps": 4, + "interval": 1.0, + "timeUnit": "d", + "links": [ + { + "receiver": "subject", + "variable": "uptake", + "provider": "peer", + "source": "uptake", + "mapping": "organs_in_soil" + } + ], + "covers": "a subject owning another data structure, linked to the peer's by the mapping it provides: a sum through a mapping, the subject's mappings in its updates" +} diff --git a/conformance/coupling/two_data_structures/subject/declaration.json b/conformance/coupling/two_data_structures/subject/declaration.json new file mode 100644 index 0000000..9e2d768 --- /dev/null +++ b/conformance/coupling/two_data_structures/subject/declaration.json @@ -0,0 +1,49 @@ +{ + "participant": "subject", + "entitySets": [ + { + "dataStructure": "soil", + "name": "Cell", + "rank": 1, + "rights": "owns" + }, + { + "dataStructure": "plant", + "name": "Organ", + "rank": 5, + "rights": "follows" + } + ], + "variables": [ + { + "dataStructure": "soil", + "entitySet": "Cell", + "name": "water_content", + "direction": "provides", + "unit": "m3/m3", + "kind": "intensive" + }, + { + "dataStructure": "soil", + "entitySet": "Cell", + "name": "uptake", + "direction": "requires", + "unit": "mol", + "kind": "extensive" + } + ], + "platform": "conformance", + "mappings": [ + { + "name": "organs_in_soil", + "source": { + "dataStructure": "plant", + "entitySet": "Organ" + }, + "target": { + "dataStructure": "soil", + "entitySet": "Cell" + } + } + ] +} diff --git a/conformance/coupling/two_data_structures/subject/update_0000.glb b/conformance/coupling/two_data_structures/subject/update_0000.glb new file mode 100644 index 0000000000000000000000000000000000000000..96ba3fcd79dca4932cb7c7e7cdb2e9a94532a971 GIT binary patch literal 1988 zcmbtV-D=w~6wY?D*9f|HOa2%oH>E2pbac&F(lIE(D30QY+Oj9fYeVpZbZ@fz-RycV zwC~7vVkZslm_dnjesuo6a}-9Ci`T7I>(`rB>jCS-cyzsH+!$fmaLv98ZDgX@koAr` z%wwTYLUWsFP!=YQkL;q|IqtLEV=32n<+wL^?HU>>1BSx->2P+hXW^Rm_LyW!u zI`NIwSzt5GBR>|F&$;CY`eOX)W9el>lM-}D3oH{XMa)S^!ay_DAfgl_GZ2GIzJD&F zh)!i<+|gsHPEw(*EF|fgHM6UfQa3(7yE^*}?}8*w6D5?L6pYL3$=d~-z9VF^A2Ozdy`~3fafob z05&bxVS&TvTrGm8AcYK<{~8_igvw%94p(e34~1XYL=W&N6J>I5VCV$Nb+i)V1(X$P zP%+R2bZ(x!V>RKk$Tiq5avZ<_ zlN!1RDj8M2zgwCDSXE5$eIF|ovrFbT#`a2HnLUJ-jSml4FQJF#_Y6z&P{YH^1 z>Huz`&+DxUplzoY%_xs8w5t^mC~5rA*l^n$C6N|3hlsHvb>(B!{Nr*Slqq(7>`?Eq zJE^FvEfoNIiI$;szn%G_)sfR^(cN)5z1Cuyq8vc&>V4Ce2e z%~n=i2^HEU8xBq=h=Th?iu+Q+Nh4UWAUs+u+%r`kE8X6J5L)Q&$@T_??uCrx43 s)dw#AKiO&CZnrylhu>XZyx@36j&5OCay#bvjAZtwc zrJ_p`swKY%;r?;Y?LF{2vKjW-76v+a3RXN`N(O=7^yh`Ub*NGv6AXpjYbG!?b*IOyp%*U8=}`4mTr1>dU^T*q6^X_O9c~pRx&QGW-mRUOZGYE z5_5y3SmBa8s+shIR6ypYxldU-znq++t)c+!gk~8Rk?*H6qC)vx_@6C@CdqiT`rqt8kBK}n?J&(|%ZT}hUaSg44 zcnM_zJ*XKl1vcjLAYe+t%v5@0+_%^ZNky<~cms=b+*&;+U-8HY-8}cF*NX~T|pr%1k}`F ztI0LhgRo%E?NJ4wHQNh!luJTsdIbpNxPEYLnB%qJP!_TT5v5sX+DCBsr`0~FN^IuX zpx!07T;Nn|Yyjw`l1JREn+F!xL%V7ZZH`dXT(750xA0YM1OUq|;AdOC5&;-7XyUd- zaKW)^K+D8Vx0;%_l7+Ogm;zF)Sb#Rc!N*PCjAd7?s?9b7&@yp@QVX#rNZcxmJPhFz z1N(PF7HiEenTYfX*q(=3l;RBwdwNvZ z8?}XpW;pj)9(BTaRJh+C?3kVxhvxYP-{aR8SMcvR&#WJQr}6&dJ3c>8%J;96p?Q9v g0Lwucd}>+yjda^yeXeDY?>K*q9mhF%Wvyx@36j&5OCay#bvjAZtwc zrJ_p`swKY%;r?;Y?LF{2vKjW-76v+a3RXN`N(O=7^yh`Ub*NGv6AXpjYbG!?b*IOyp%*U8=}`4mTr1>dU^T*q6^X_O9c~pRx&QGW-mRUOZGYE z5_5y3SmBa8s+shIR6ypYxldU-znq++t)c+!gk~8Rk?*H6qC)vx_@6C@CdqiT`rqt8kBK}n?J&(|%ZT}hUaSg44 zcnM_zJ*XKl1vcjLAYe+t%v5@0+_%^ZNky<~cms=b+*&;+U-8HY-8}cF*NX~T|pr%1k}`F ztI0LhgRo%E?NJ4wHQNh!luJTsdIbpNxPEYLnB%qJP!_TT5v5sX+DCBsr`0~FN^IuX zpx!07T;Nn|Yyjw`l1JREn+F!xL%V7ZZH`dXT(750xA0YM1OUq|;AdOC5&;-7XyUd- zaKW)^K+D8Vx0;%_l7+Ogm;zF)Sb#Rc!N*PCjAd7?s?9b7&@yp@QVX#rNZcxmJPhFz z1N(PF7HiEenTYfX*q(=3l;RBwdwNvZ z8?}XpW;pj)9(BTaRJh+C?3kVxhvxYP-{aR8SMcvR&#WJQr}6&dJ3c>8%J;96p?Q9v g0Lwucd}>+yjda^yeXeDY?>K*q9mhF%Wv2SBVKO}`qQ~*3O>Qy^MMptVh(gnj^Z&u9Q%?ao<0a1~H}>G6Q1Z zlJB3gIL1?z7;|(4&d$dtcvn$?c0#j^i^%s=8Bw8pE_{>B#^(;B>%B^I z8Gz?cjR0(FDx(sIkGWa~jX^RVE&n$<&|@M`OgT)k*&rfEv^c=mHyac@QwApl2#QGU{9GxuhakHoSqw%f+=wZ$!f!7Bf?g3K**?oyVrZ zy2vpA1DIIoLa5|e`C+#-C9vw4!1rye)XYAb-x%8}d2RMSv}}Ah==#vWqc#PJun0mr5RSGj8q}TzAc?duTI+qUL%vVY-E{Y9atwt^q%r?3D;Wk3kcs zErJWSRRLO8Y*nktc`H~5D~l;0#qtGcV{Ck!^i5lK)vDZVA^6bYn;TXGS9;h zelakA*JQrZ?3{^6FM)kDz$i-gN6v045hD)K0t=+aVQBtS)v;C{cnHx_cT0BQmAX$7 z_72;^-r)l`9|5NA7#WpU36*_wmrI oKgYmwPzF~mYkzTXKPYG4U+--lq~WT+j`P>pah#o(hVj7p9~~NnTmS$7 literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_data_structures/subject/update_0004.glb b/conformance/coupling/two_data_structures/subject/update_0004.glb new file mode 100644 index 0000000000000000000000000000000000000000..26929668a5c8680ddfca6542af13e4ed4ca9a17a GIT binary patch literal 2140 zcmb7F-EPw`6b{_5$C!MbP5p_ctv6##OahG+(GDSvDsodNwU#)+cG#*a60ZSo!X39< z^CCO~=Qv5@r2E5C68rr4{C(#nil=AK9mn~+<2c`5IL_P2_+m|{QcRQ4ntWwa@l=oz z8SM9nOClziq&k&A300Qt>2SBVKO}`qQ~*3O>Qy^MMptVh(gnj^Z&u9Q%?ao<0a1~H}>G6Q1Z zlJB3gIL1?z7;|(4&d$dtcvn$?c0#j^i^%s=8Bw8pE_{>B#^(;B>%B^I z8Gz?cjR0(FDx(sIkGWa~jX^RVE&n$<&|@M`OgT)k*&rfEv^c=mHyac@QwApl2#QGU{9GxuhakHoSqw%f+=wZ$!f!7Bf?g3K**?oyVrZ zy2vpA1DIIoLa5|e`C+#-C9vw4!1rye)XYAb-x%8}d2RMSv}}Ah==#vWqc#PJun0mr5RSGj8q}TzAc?duTI+qUL%vVY-E{Y9atwt^q%r?3D;Wk3kcs zErJWSRRLO8Y*nktc`H~5D~l;0#qtGcV{Ck!^i5lK)vDZVA^6bYn;TXGS9;h zelakA*JQrZ?3{^6FM)kDz$i-gN6v045hD)K0t=+aVQBtS)v;C{cnHx_cT0BQmAX$7 z_72;^-r)l`9|5NA7#WpU36*_wmrI oKgYmwPzF~mYkzTXKPYG4U+--lq~WT+j`P>pah#o(hVj7p9~~NnTmS$7 literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_scales/expected/record/frame_00000.json b/conformance/coupling/two_scales/expected/record/frame_00000.json new file mode 100644 index 0000000..502af8f --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00000.json @@ -0,0 +1,149 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 1, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00001.json b/conformance/coupling/two_scales/expected/record/frame_00001.json new file mode 100644 index 0000000..dcbe0cf --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00001.json @@ -0,0 +1,167 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00002.json b/conformance/coupling/two_scales/expected/record/frame_00002.json new file mode 100644 index 0000000..fe1c345 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00002.json @@ -0,0 +1,185 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592, + 8589934593 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299, + 4294969300, + 4294969301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298, + -1, + 4294969300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00003.json b/conformance/coupling/two_scales/expected/record/frame_00003.json new file mode 100644 index 0000000..b39e4f5 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00003.json @@ -0,0 +1,185 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/record/frame_00004.json b/conformance/coupling/two_scales/expected/record/frame_00004.json new file mode 100644 index 0000000..5a8eda5 --- /dev/null +++ b/conformance/coupling/two_scales/expected/record/frame_00004.json @@ -0,0 +1,203 @@ +{ + "time": 4.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595, + 8589934596 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594, + 8589934595 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor", + "successor" + ] + }, + "length": { + "dtype": "float64", + "values": [ + 2.0, + 4.0, + 6.0, + 8.0 + ] + } + }, + "variables": { + "length": { + "unit": "cm", + "role": "state", + "standard_name": "length", + "kind": "extensive" + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 8, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301, + 4294971302, + 4294971303 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595, + 8589934596, + 8589934596 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300, + -1, + 4294971302 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "sugar": { + "dtype": "float64", + "values": [ + 1e-08, + 1e-08, + 2e-08, + 2e-08, + 3e-08, + 3e-08, + 0.0, + 0.0 + ] + } + }, + "variables": { + "sugar": { + "unit": "mol", + "role": "state", + "kind": "extensive" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [] +} diff --git a/conformance/coupling/two_scales/expected/turn_0000.json b/conformance/coupling/two_scales/expected/turn_0000.json new file mode 100644 index 0000000..93154bf --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0000.json @@ -0,0 +1,82 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 0, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 1, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none" + ] + } + }, + "rank": 5, + "scale": "Organ" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0001.json b/conformance/coupling/two_scales/expected/turn_0001.json new file mode 100644 index 0000000..d92b526 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0001.json @@ -0,0 +1,144 @@ +{ + "time": 0.0, + "time_unit": "d", + "step": 1, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 2, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0002.json b/conformance/coupling/two_scales/expected/turn_0002.json new file mode 100644 index 0000000..e0f05f1 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0002.json @@ -0,0 +1,161 @@ +{ + "time": 1.0, + "time_unit": "d", + "step": 2, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934593, + 8589934594 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934592, + 8589934593 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294967296, + 4294967297, + 4294968298, + 4294968299 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934592, + 8589934592, + 8589934593, + 8589934593 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294967296, + -1, + 4294968298 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0003.json b/conformance/coupling/two_scales/expected/turn_0003.json new file mode 100644 index 0000000..7c0cd9b --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0003.json @@ -0,0 +1,177 @@ +{ + "time": 2.0, + "time_unit": "d", + "step": 3, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 3, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [ + { + "data_structure": "plant", + "entity_set": "Organ", + "ids": [ + 8589934592 + ] + }, + { + "data_structure": "plant", + "entity_set": "SubOrgan", + "ids": [ + 4294967296, + 4294967297 + ] + } + ] + } +} diff --git a/conformance/coupling/two_scales/expected/turn_0004.json b/conformance/coupling/two_scales/expected/turn_0004.json new file mode 100644 index 0000000..19d69d1 --- /dev/null +++ b/conformance/coupling/two_scales/expected/turn_0004.json @@ -0,0 +1,178 @@ +{ + "time": 3.0, + "time_unit": "d", + "step": 4, + "data_structures": [ + { + "id": "plant", + "kind": "graph", + "origin": [ + [ + 1.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 1.0 + ] + ], + "entity_sets": [ + { + "name": "Organ", + "count": 4, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934594, + 8589934595, + 8589934596 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Leaf", + "Leaf", + "Leaf", + "Leaf" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + -1, + -1, + -1, + -1 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 8589934593, + 8589934594, + 8589934595 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "successor", + "successor" + ] + } + }, + "rank": 5, + "scale": "Organ" + }, + { + "name": "SubOrgan", + "count": 6, + "columns": { + "id": { + "dtype": "int64", + "values": [ + 4294968298, + 4294968299, + 4294969300, + 4294969301, + 4294970300, + 4294970301 + ] + }, + "label": { + "dtype": "string", + "values": [ + "Segment", + "Segment", + "Segment", + "Segment", + "Segment", + "Segment" + ] + }, + "complex": { + "dtype": "int64", + "values": [ + 8589934593, + 8589934593, + 8589934594, + 8589934594, + 8589934595, + 8589934595 + ] + }, + "parent": { + "dtype": "int64", + "values": [ + -1, + 4294968298, + -1, + 4294969300, + -1, + 4294970300 + ] + }, + "edge_type": { + "dtype": "string", + "values": [ + "none", + "successor", + "none", + "successor", + "none", + "successor" + ] + }, + "segment_length": { + "dtype": "float64", + "values": [ + 0.01, + 0.01, + 0.02, + 0.02, + 0.03, + 0.03 + ] + } + }, + "variables": { + "segment_length": { + "unit": "m", + "role": "input", + "standard_name": "length" + } + }, + "rank": 6, + "scale": "SubOrgan" + } + ] + } + ], + "mappings": [], + "update": { + "participant": "coordinator", + "removed": [] + } +} diff --git a/conformance/coupling/two_scales/peer/frame_000000.glb b/conformance/coupling/two_scales/peer/frame_000000.glb new file mode 100644 index 0000000000000000000000000000000000000000..1547928674f4c65e4018494be4b763ff75f7e2bb GIT binary patch literal 1908 zcma)7ZExBz5Jvm5zft+qKBQH6S%vax-PTH-C{zQcNfnAr;smRSli3a}Lj3ma&UQkw zma35e=iYqx+;c~oEp9&adOiKI*Yn}?V?LYi2~~<|670#ANyQ660y4PpiAPc^~~PPOFEVBEj( zJ^z)@d0Wvww~av#H$@ws#gakd5B=4_Yc4D^Dm1}d*t2E=Yg0?bVz49EzZap>vWj#i zX%^;8)0k=sRo~3-?_8Q4PekaH7pPRUVL3%34i|KbjWWa-pxG0XQy9)*-5p9mTK}%POM^QY9qjGova&`z%L;mhWCyXY?fIbAo`WFYaiXHaHuswl9p2SA0Uk#x)k5s^ktVei>hE^HRtzyl)81M-g7guS*2p0g z%^JdG#~E&-J|?h%}Sb6g4z$&(mNlxC%A9C?ylGgw$=(sWYD9897@;0b!gaCzaN z&)w`hyv)qnec_QuE^vfXNlWZ)A$iKpa(QJ3EP z!!{J_0#{tFNIp1jspPKJFBi%Y<;vm{y! zACIi7B!RBL;@p$Tr!$H?}3`{+P=M(&Ye!ZQ-clY`0p02d_psHd=d=oxp(;=9wpQvyVHz+^s!(JS2dpMeW;;-X_y+sV+nw!% zq?D?W0CWF*_q*R+mg)5NeW%m;`=QhE;q!em9_qJNJo4mP{~7 zbuK}Ys3JSl$tVB(g6ut_BG8e*cPB-HhaC}l%<$yT_j9lQ2XdN$tqBoYM~`GwOa27L z-E-gbU-+Cg70q+g802tMwBl(j86^JDo$tN+!Xl+Y6U>D@X(q5XHB&4GJ96`T8Y(Tz zNSBhPVa7C#sisi%?c~S3OS9#P2%Yi*m5Np@qiDq8f_AY{iWmblJ7RJQ!?9da$hS__ z{`W{B^MXmutuaR05xp%CzRB0?;q^~QFv_zc7fk4>RosoHgFeV5`%`j>NhgSSj$WSa8?Q+b%R2xj3TBK zjDYha0aRGGZ6T?MRtH_Lp)19s$T#Akyx0RdTLh>ee{-TkMw3H89|B_ijx^gTU8H6J z>-;KO83F=_n*D}2QC?!3dv%Bw?<$-Cj{}t|A@=z|lS&Hpx4T~{E-*&j-b&hs^bwQB z$RQN<8p37ckwls<0hKuJ5!?mlI1?O_Coz~P%?i^QvXNXfSXgG#bdt*qOrl)i33|nF zdEuZ>-0VBN%*@(-;gLr!aDht8kr8iBBnAPS^hhpx^lNJ z*MJs29$A!00$qW{xg(3MW<%G?K_Bz6+FzV)oQ1JAZwE9UCy9B%K--$W-2o1Z^*yrf z4%)_MuujimK(w`8lmppjE|FyRDTvQJ4lyl`0gO!~v^`li3awq5L=dKes#E z2}xV3S_(0D-<|Kr=gTsg-hAkEI)6WQIzBu4jJ`SWt_)&D?FQ@|P*fi?6I3tH3QaH;_N1A>+|*352yo>3ei|q( zi%=JmCPB(Hji{!O_08nxoeQ&Ni4{6z1u7M;SV|GZ;ed9sVS+V=((H(dDG0`LNg>`k zRQul}xy*7VHMd~&v?F?(WBDfEu7+2?Aiyw7^Gq=gOH2>)Vx>nC zx?oAe!}Vx(2RJ5*)R{J;2?C1;!<<4j4JF=tw#dF?^#;eMV9uf?3#?@w{mZjWo%c|T zmx-lbH3B}E^38C3g-MsNY(l2E&s;+7)-)-YOLhi@x`6zR%rVw7Q$RT@1(3QyAytGS zQwpwtvm*n@ux{HzQW389x^6>Ph)0oa#9rAeft)P@R3(3RphHIEy+a=yV*ZXa+bLZn z<^tC7RkSh$1P(R(4SvF`z&7`47cJgZb%J{AsZFK!XuAd;1ym$(P&=msiHrYm@1KY>rITTyG~#x9C@O z3CQ$F24{6QY(tTdV2Z;P$p_mtmE5)ZR;-F_Pk{5gp2b=%lC9iN@I!EZS}e-;Edo|pIV{rq}6f?vCi+#A2E&GpZ%N><@I QNA(U>msbDDy+2?62dM{O6#xJL literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_scales/peer/frame_000003.glb b/conformance/coupling/two_scales/peer/frame_000003.glb new file mode 100644 index 0000000000000000000000000000000000000000..d3793a30a9103daed95a6fa9ed0a88a712e02bce GIT binary patch literal 1956 zcma)7-%r~x5DweJUiY}i$ECUL$YX}PV78t4^*@l)46w#UXbpXmQ7!o+ zfRB#?KX~OiY$}@jCK%*!Qnci0EExoTcRbtr^?^l7g(esad(=!|ZfdGn3^;OmI|-GR zWu!|<(=cP2##B?t`fB{^#)a9k#0u@R0+ouEETahGa6miRD8(8>X*R^f6ow+7P`iuzch17lVu65MY#NMJ|}o6RWr$PELCum+WuJC8h^?v8_*1 zI%jFa!{u;#12`ti%$YW$2}6qq!-7IJ4JF=tw#a{A_4@m+TY@bq4txnPaSFrhsx*3LtfZLaK}+ zrW9NOXGi*wVcoX5q#|1Gbe)E-6!#*35Ibde3*>AOpep&h1067$>>LK*5c7AWSx@LZ zH5agsucDP9AaJPJZ}1c4CAPU+yJ+#QsuR>>N2N-LecsWel0yFN?pKN~`l#DmNqZ1J z)}(ReuoU$e!eQf+M4B!El{n6kE({B^oeB=YlNcaMvjQ~W>OpeNU|^Y>rjuM|0Eu#e zJLna|;e{7{?C!p+m$|ccUijpm3%tS$Nek?4F8P9+;quCudTlc8kg_9;^Ms*@HtEwD~$9c$6gO0|RYq+ID() zS*+~~Y^T=-Lu>b9yPY=nO~)svZSY%;PoD(=j%V9j_I{!e{FY? z?AS{JtsqFzedz9YzdJ9I$;~Uz^ZvZ`ybJh!8;^z?LX~2g^fzS1q~e(%ebPB^5uYrW zV3O)g!cDC5^i0R6t@AF~`b32wBmI^;Dbl~+5RpX;Pp#Q@=2!pVo~EE{Tm;t8Cn?pE zKY?=lyydqJ{Lbov`nj$Q_wZ7<;q(slm?%ivPTL%Fc{gel}8vjk^|(o`eUVHXjV(a`eW z*;JXkak&DtaQS3j#4%I_2Iq#%*P0DnDckK124uxQIeT;}M%{cKP?( literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_scales/peer/scene.fspm.json b/conformance/coupling/two_scales/peer/scene.fspm.json new file mode 100644 index 0000000..ee7ae02 --- /dev/null +++ b/conformance/coupling/two_scales/peer/scene.fspm.json @@ -0,0 +1,31 @@ +{ + "fspmIndex": "0.3", + "timeUnit": "d", + "frames": [ + { + "time": 0.0, + "step": 0, + "file": "frame_000000.glb" + }, + { + "time": 1.0, + "step": 1, + "file": "frame_000001.glb" + }, + { + "time": 2.0, + "step": 2, + "file": "frame_000002.glb" + }, + { + "time": 3.0, + "step": 3, + "file": "frame_000003.glb" + }, + { + "time": 4.0, + "step": 4, + "file": "frame_000004.glb" + } + ] +} diff --git a/conformance/coupling/two_scales/session.json b/conformance/coupling/two_scales/session.json new file mode 100644 index 0000000..193f2ce --- /dev/null +++ b/conformance/coupling/two_scales/session.json @@ -0,0 +1,16 @@ +{ + "steps": 4, + "interval": 1.0, + "timeUnit": "d", + "links": [ + { + "receiver": "subject", + "variable": "segment_length", + "provider": "peer", + "source": "length", + "aggregation": "copy", + "factor": 0.5 + } + ], + "covers": "a subject refining a replayed peer's organs into segments: new entities, removals cascading to the subject's segments, a unit conversion and a factor, a copy to finer entities" +} diff --git a/conformance/coupling/two_scales/subject/declaration.json b/conformance/coupling/two_scales/subject/declaration.json new file mode 100644 index 0000000..4a1ba1c --- /dev/null +++ b/conformance/coupling/two_scales/subject/declaration.json @@ -0,0 +1,36 @@ +{ + "participant": "subject", + "entitySets": [ + { + "dataStructure": "plant", + "name": "Organ", + "rank": 5, + "rights": "follows" + }, + { + "dataStructure": "plant", + "name": "SubOrgan", + "rank": 6, + "rights": "refines" + } + ], + "variables": [ + { + "dataStructure": "plant", + "entitySet": "SubOrgan", + "name": "segment_length", + "direction": "requires", + "unit": "m", + "standardName": "length" + }, + { + "dataStructure": "plant", + "entitySet": "SubOrgan", + "name": "sugar", + "direction": "provides", + "unit": "mol", + "kind": "extensive" + } + ], + "platform": "conformance" +} diff --git a/conformance/coupling/two_scales/subject/update_0000.glb b/conformance/coupling/two_scales/subject/update_0000.glb new file mode 100644 index 0000000000000000000000000000000000000000..388f92e54ca22045825c908f04c16e31b27ca853 GIT binary patch literal 1956 zcma)7U2obj6lHta->CeYR!}HHdF!@T>eNCtV475+$OIFjCQfEMZ4u(%?0;_OIv*qy zDk24v`{m|Y<#$I-w@on zXglo#_j%n>f363^j+OjX7%h|l@Tbmf>r@M-5toJ$7V&0;q`m2>77N6&o8J>p82PZoAo2&7}RW-#pHS8Y(7!}2jmKE zWpY@ml2j-Y<|$Nf!^}D*635@J@2`IXrk})VB84=Q+;KOY^n0)?@h1xvZ4$c^+dME5 zp!)P0GWkwixaS?@rySH;eVhuRkuDa2@bb(Gz|C;_0H{J{ae>~Fz6-^&5ub~w2Ks%@ z9+jS#ryy5n0f%U-^^THudq27++P1iIJhn-DXi(^dN0}(FPC@km@U5*-ey(tdhLjE* zj+zv8=KDfx9HkjGI^CdXLk#c?+dBi{mEFzB8aK46`nTA z5(|a+))PLD>~XG8))ZPAbHp(4> z`7RulPRi&WjFkXN;}y0>tkn)T)u{#^jWvfYmF**R6Vk+@Nn;^YP)2m-}Z>TIIIAhaXuz_2?s-;UU3q*D``^;npO_-fsOnx2{4 zuUe4zl3*j7rZlM!-FDbKTP{HYCgO(8*GAk68JHDw2R1bK{tCrw5n-cI2Z;zCEtmGo zQM{J9oj$2I&wUr$0jl~!9|m`!qLYJrgJ!TjLq||t?Vk1>)KE?CcTW(XV$y c{I*)3YB#;pHz>% literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_scales/subject/update_0001.glb b/conformance/coupling/two_scales/subject/update_0001.glb new file mode 100644 index 0000000000000000000000000000000000000000..b6b4dc589dee06e4f6e0ab8fa6b718879a084630 GIT binary patch literal 2028 zcma)7+iuf95DoCaZ&ds2R!$Qt@B^YiD}mYskpv-Ck+pF)j#YbY*6X%Wls`a-Z{!j9 z2RO5~H%^*Xv6R@Jo6np%9*6Pp{AsJz`un`qdV}A`LBF?TJkNz@-jdx2l}D*$9y>mA znZrUMh2lC@u*~u-In=WQ_o%~G4$A}hnCDhkIrFZTOr}#oSMGQt-SOKk_x>ceI1I& z#(W~;2I%`aTU7dfnS$KB2suP+pZ6eX=NJ7`qHTqv#AAlEg$9LgcwC4I>j+f81bkzw zlwT@bVj#5xFGoX)c@YF6&+#f%dCUX1cJ>p+WiVg+jvKy0Ud!}WuAQ9~M7}Mgy8FGc z4n&yHTrD0KWr>AKe2XEU#O8IjC>shjiiRAwW`JrzT98`0w$O%(YtF!pQE(;9+PtZ^ z+6p$!~)kz)Qh0zM2Hoib>#9HI!raE@v(OPrZOqn@CHxX?t+BD{JBqOxM zrU=Qk$joRNt&2gmK`Nfin_q4nhh0aK7D1?ZMw3ld6h>x6?O1lZ_S*xtG3iuCdNB}s zLwt?tKuy=oJFz1uXGyS;OKh=QjSF}I zF5w~I9LI5!?nfn68vC5@*q_h&auOw@)2EK(xGx;%^>fF0KOFQ|gi6U2@mJ)U2^nXc z_@uq>5tl@aGeK1*z%rM4x~JwZz5N5Sc8LtZM|{tWB=Ij-gl98`BX6>vxON71ngUiY z5tM;0NvRU?Er475p6lIn@6{dkaUBeH*eP7Z$xJXHe%YF=UAtkPP_77aVYiBLJewR# zHUk_v`8*1w5=E#AL6ab5iq5E_aO%_HhYLfq5s4XkC<y%TCI;=DKx^_EMh?!X$hS4#}`1A@ggDhIqAH7Uv>44IVhDw#OqLvC&Dr-JfuvGKKQzJg!z?1pcg2Wt@d zwvejR@62_`q7>_D`FOA_X2FPWIik}YiqIUw?bR zHbI@LSS^Q4HN;n|4yfs>UV^fgNE@QDOQZUr+Af(E^Eo5|I^v2`V+_X`>(mc jtg+R++NYXV`v3@k_VT}azrHO`j+)Q0nU9Yw&I#>11xmSx$`EbHY{%X;e#`b$FdoJry?$rTfM zlyc&d_MSs*5;D#Nm8k&9EYFf%Ioome_Q}d7c>p@%I(nBA_hLzSI%T+X#;dVioj^_# zuvH=a($OXfl_I(Ux%lmZVYm`DVRs`R4@R3(HgJp>cT9hToS~>ZY1NF zTRzI!6zs_H$DyA~Q3SFOH1-oF>6A(eS)X|C&Na-2Csyb_FHq&doFx>EBDkQLY!G9O zp)^aPF!{b$Oa>yP0J%b2iENUIG-E=!(V# z(^yAP>K?#XGR^#w;T#%b6!3D?sF)W)!15ekCGAICsBP8ziJ&}~uXXL3uHctEz2 z{t8IGNhFi~)>u6jCfL^sj{{+`3YvV2A)UnPb*@m>7^)Zzblm6xL<`aa+0v~7HCSwF z2DMQP*2Fa8Cf|${szKJ=Fd0JS_d!wd#8`JgSP{So46*5JDZ9qjMRv_W7yp_ z-|mr(k*6|}3y;YL`D)z(HeJ?RQ1%jGgEvlTWFK_fCX-?|gCf90T$0H`vNOg*IVW!0 z>7i|@{t&TiZKBK}5U9XaZS+-0F0i*4+nS zv<}+;Vcfl);dr&9p6~EIeRbNmem}9SN3j2+*B|hG#q&?Setlk^zv&fq_$@Xo?(2UD oqt0gis~>ZZIh%XT+1vwx@mD?WXXn?K#qnWt9qIGvurkz}f6cIl9RL6T literal 0 HcmV?d00001 diff --git a/conformance/coupling/two_scales/subject/update_0004.glb b/conformance/coupling/two_scales/subject/update_0004.glb new file mode 100644 index 0000000000000000000000000000000000000000..0faf59b6be053713b61c0e42821c6c100526eb49 GIT binary patch literal 2156 zcma)7O>Wab6gEE+NGxE_rkWkXt(>%k;zdP)Rw`-}L=uElMP}kmT&woj*yFZQlp7#U zf#4Ew3%CGFI0U>mKaQQIRa~Vr?{~g=@AGpI_4aRCmKEK#tmk(u>vgx&o)VhpOb~ZU z&RCX*2`4UTZaBmy0pl#ABFR8A&eM2ZjMtovO)|4d?t_iEj^5?OJ)RPtj2P~m!E9ic z66k3RzA}ZU0&Nmgk%brF+}LnzXUT89YN+n3&Y*{r{F5*mWek8HHwH7iJTQ$Y7X-1e z3&A+vE$`=S1b$@ieb389R`{aGXynCA&=D0BZoS`qbEIL`BC$eOMS&^zPgqRRDTD*+ zm-Qp8F_dOXBqq=67Q;>!P=H*bEmby2vm|Ai2vrE?n-Vcev54K5yNA1P0j8hCX~G#7 zJ!Lp(_gY(^&)CNz%Zxl||b>?CfIMX7DIHvPlbQ zAki6(3Z}94q14*|U+OgT6~j3+#2Db^s8BI4e4phxyh=KbxKP`&_rr{G|731!R%``7 z<;gjp8#iY_@swB!e|Ejk?Y^qlIdaIXBD=q4JwxD0yPMw?J7Ez$l-<)v#)nmmBkx0}scV zO~#q*Bk(4~iG`C!Zu>lhE0GbnoGEpO6r8~f;Q{uMD7N}k#Y>37wjqHPN+hkab$4~^Ah*L70 z2zJPLAWn$eeA+}`Wqud3bL}GCArY9syE7ikPlpywa0^-h>6ZJpwp8p&&W$GUsEX^_ z)QxA&y7%@qIGWA^>g$~^;~31=@Of~EBfNM*{ry){vfTC&3aqQX8orhgTu%Mhmj2qBO4rWu=<0%__&{(-=8OYJN31% U<^Im&@XMpe4?mQvikkob4}*Y|SpWb4 literal 0 HcmV?d00001 diff --git a/conformance/growing_sequence/frame_000000.glb b/conformance/growing_sequence/frame_000000.glb index 00afa7dc892ae9a49391e42f1d1a8615775fffc4..0d63257d867ae7cd7be97aaae7506dc3529f35d8 100644 GIT binary patch delta 19 acmcbUawlbiE2Hs7x6izc#+$A9lvDszZU*)M delta 19 acmcbUawlbiE2Ggyx6izcMw_kplvDszTn6+2 diff --git a/conformance/growing_sequence/frame_000001.glb b/conformance/growing_sequence/frame_000001.glb index cc55210f04043629f62a26a207b3aaed5fadf3f7..985e841561d46795644c2ea683b34be44e91b035 100644 GIT binary patch delta 19 acmZoEX(^fD%4odN?K3Z<@n$Q&Q`!Jag$ES? delta 19 acmZoEX(^fD%4oFF?K3Z<(Pk^YQ`!Jaa|aUu diff --git a/conformance/growing_sequence/frame_000002.glb b/conformance/growing_sequence/frame_000002.glb index f79f32aeb92904170e97a5d12da2b876b6dc94a3..631d4f9d4e9f40288669b5a0144af8b5ba053826 100644 GIT binary patch delta 21 ccmaFT#rUL)ae^zO@kX~Vyo|=1t@$*~0be%J#+&tc|M3CJMw|6`|M3Cframe + participant.send_update(frame); +} +``` + +```java +try (Participant participant = new Participant(declaration, null, null, null)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + // the model, on turn.frame() + participant.sendUpdate(frame); + } +} +``` + ## Running it The coordinator starts each participant's command, and waits for the ones started otherwise (`--expect`): @@ -79,9 +110,10 @@ fspm-tf couple --steps 30 --interval 1 --time-unit d \ `[{"receiver": "growth", "variable": "supply", "provider": "function", "source": "sugar"}]`. - `--record`: the whole session written as an ordinary sequence of frames, to read in ParaView or to replay. - `--replay name=run/scene.fspm.json`: a recorded run takes part as a participant, without its platform running. - A platform's routine log can thus feed another platform's model. + A platform's routine log can thus feed another platform's model. CPlantBox's and GroIMP's logs are tested this way. - `--endpoint files:folder`: the participants exchange files in a folder instead of ZeroMQ messages. It is slower, - but needs nothing but a file system, and the folder keeps every message. + but needs nothing but a file system, and the folder keeps every message: plan its disk space (a turn of 100,000 + segments with five variables is about 5 MB), and use a new folder for each session. - `--stages "growth;function"`: who runs first in each step (by default, the participants creating entities, then the others). Each step, a stage sees what the previous stages did in that step. @@ -93,6 +125,26 @@ metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for anothe model: a component computing nothing, whose state variables are the other model's outputs and whose inputs are sent to it. Its tests couple metafspm's root growth to a carbon model written in VirtualPlantLab. +## How fast it is + +The exchange's own cost per step, with the coordinator and two participants in separate processes, each sending five +variables on every segment: + +| Segments | ZeroMQ | A folder | +|---|---|---| +| 1,000 | 3 ms | 13 ms | +| 10,000 | 9 ms | 21 ms | +| 100,000 | 99 ms | 110 ms | + +Encoding and decoding the frames is about a tenth of it (10 ms for 100,000 segments); the rest is the coordinator's +work on the shared structure, in Python. + +## Checking a participant + +`conformance/coupling/` holds conformance sessions: a peer the coordinator replays, and what the participant under +test must read and send. `fspm-tf couple-conformance conformance/coupling --participant COMMAND` runs them over both +transports; each codec's CI runs its own participant (Python, Julia, C++, Java) this way. + ## How it behaves - The coupling is explicit: a participant's inputs at a step are what the others gave at their previous turn. diff --git a/doc/cpp.md b/doc/cpp.md index 88931b8..30d0baa 100644 --- a/doc/cpp.md +++ b/doc/cpp.md @@ -45,6 +45,15 @@ A column holds a vector of numbers (any integer or float type), `bool` or `std:: more, it holds a fixed-size vector per row. NaN and infinities are written with the specification's stand-ins and read back as they were. +## Coupling with other platforms + +`fspm_tf/coupling.hpp` is a participant of the coupling sessions fspm-TF's coordinator runs +([Coupling platforms](coupling.md)): `fspm_tf::coupling::Participant(declaration)` joins one (the declaration as its +JSON; the address and name from the environment when launched by the coordinator), `next_turn()` gives each turn's +inputs until the session stops, `send_update(frame)` answers. Its folder transport needs nothing; its ZeroMQ transport +is built when CMake finds libzmq (`FSPM_TF_WITH_ZMQ`, on by default; `has_zeromq()` tells). The tool +`fspm_tf_coupling_conformance` is its program for the conformance sessions. + ## CPlantBox Build the adapter against CPlantBox's sources and its library: @@ -105,3 +114,7 @@ CPlantBox has two scales below the plant. `to_root_system` reads frames with mor segments are the entity set with segment ends, and the organs are the set their `complex` points to. For example, metafspm's Plant, Axis and SubOrgan become CPlantBox root organs. CPlantBox has static roots only, so stems and leaves are not rebuilt. + +The adapter's tests also write CPlantBox's log of a growing root system (`anagallis_log/`: a frame a day for six +days, and its index), which the CI then replays as a participant of a coupling session: a reader receives every +logged segment and value at each step. diff --git a/doc/governance.md b/doc/governance.md index 0867f7c..0156538 100644 --- a/doc/governance.md +++ b/doc/governance.md @@ -43,6 +43,7 @@ The CI then runs every codec against the new conformance files, and shows which ## Status -Version 0.2 is a draft. 0.1 was written with metafspm as the first platform; 0.2 adds what the C++ (CPlantBox), Julia -(VirtualPlantLab) and Java (GroIMP) bindings needed (its "Changes from 0.1"). The partners' reviews come next. Until -0.2 is released, the draft may still change in ways that make earlier files invalid. +Version 0.3 is a draft. 0.1 was written with metafspm as the first platform; 0.2 adds what the C++ (CPlantBox), Julia +(VirtualPlantLab) and Java (GroIMP) bindings needed (its "Changes from 0.1"); 0.3 adds coupling sessions (§11, "Changes +from 0.2"), which the other platforms' teams are to review. Until 0.3 is released, the draft may still change in ways +that make earlier files invalid. diff --git a/doc/java.md b/doc/java.md index 930c65e..c0c5db7 100644 --- a/doc/java.md +++ b/doc/java.md @@ -32,6 +32,15 @@ the array of their width), with a `width` of 2 or more for a fixed-size vector p (`withMissing(...)`, `isMissing(row)`; NaN in a float column). NaN and infinities are written with the specification's stand-ins and read back as they were. +## Coupling with other platforms + +`org.openalea.fspmtf.coupling.Participant` is a participant of the coupling sessions fspm-TF's coordinator runs +([Coupling platforms](coupling.md)): `new Participant(declaration, endpoint, links, timeout)` joins one (the declaration +as its JSON; a null endpoint, the environment's when launched by the coordinator), `nextTurn()` gives each turn's +inputs until the session stops, `sendUpdate(frame)` answers. Its folder transport needs nothing; its ZeroMQ transport +needs JeroMQ (pure Java, an optional dependency) on the class path. `ConformanceParticipant` is its program for the +conformance sessions. + ## GroIMP `GroImp.toDataStructure(root, options)` writes the graph below a GroIMP root: @@ -67,6 +76,8 @@ writes the project as an fspm-TF frame, and the CLI's `export3d plant.glb` does GroIMP's glTF plugin, which it requires, meshes the shapes: cylinders, boxes, spheres, cones, frusta, planes, parallelograms, NURBS, meshes and polygons. Its folder (`target/plugin/fspmTF`) is copied into GroIMP's `plugins/`. `test/run_in_groimp.sh` installs both plugins in GroIMP 3.0.0, grows GroIMP's `simpleRapeseed` example headless and -exports it (456,900 vertices, each with its node), then checks the frame; CI runs it. +exports it (456,900 vertices, each with its node), then checks the frame; CI runs it. It then writes GroIMP's log, a +frame after each of four runs (GroIMP has no simulation clock: each frame's time and step are its run count), which +the CI replays as a participant of a coupling session. Not done yet: an import filter (frames into a GroIMP project), and units for GroIMP's fields, which have none. diff --git a/doc/julia.md b/doc/julia.md index 935c4c9..43feaa5 100644 --- a/doc/julia.md +++ b/doc/julia.md @@ -126,7 +126,9 @@ entity). NaN and infinities are written with the specification's stand-ins and r FspmTF.jl takes part in coupling sessions run by fspm-TF's coordinator ([Coupling platforms](coupling.md)): `Participant(declaration)` joins one (its address and name from the environment when the coordinator launches it), `turns(participant)` gives each turn's inputs, `send_update(participant, frame)` answers with its outputs. The folder -transport needs nothing more; ZeroMQ needs `using ZMQ` (the `FspmTFZMQExt` extension). +transport needs nothing more; ZeroMQ needs `using ZMQ` (the `FspmTFZMQExt` extension). A declaration can be built +(`Declaration`, `EntitySetSpec`, `VariableSpec`, `MappingSpec`) or read from its JSON (`declaration_from_json`). +`bindings/julia/coupling_conformance.jl` is its program for the conformance sessions. ## Checking it diff --git a/doc/platforms.md b/doc/platforms.md index 2c79e9f..5b394f2 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -114,6 +114,15 @@ the coordinator is fspm-TF's own (`fspm-tf couple`), so no platform implements i 4. speaks both transports: a folder of numbered JSON headers and GLB payloads (files only), and ZeroMQ (a DEALER socket named after the participant; one or two message frames). -The Python package (`openalea.fspm_tf.coupling.participant`) and FspmTF.jl (`Participant`, `turns`, `send_update`) -are the two participants so far; the Python tests run FspmTF.jl's against the coordinator over both transports. An -adapter then maps the turns to the platform's model, as metafspm's `RemoteComponent` does. +The four codecs have a participant: Python (`openalea.fspm_tf.coupling.participant`), Julia (`Participant`, `turns`, +`send_update`), C++ (`fspm_tf/coupling.hpp`) and Java (`org.openalea.fspmtf.coupling`). A new one proves itself on the +conformance sessions: a small program run as `COMMAND SESSION OUT` that joins with the session's declaration, dumps +each turn it reads and sends the session's updates, as the four codecs' programs do +(`coupling/conformance.py`, `bindings/julia/coupling_conformance.jl`, `bindings/cpp/tools/coupling_conformance.cpp`, +`ConformanceParticipant.java`): + +```bash +fspm-tf couple-conformance conformance/coupling --participant "" +``` + +An adapter then maps the turns to the platform's model, as metafspm's `RemoteComponent` does. diff --git a/implementations.md b/implementations.md index eea029e..3d58d40 100644 --- a/implementations.md +++ b/implementations.md @@ -2,19 +2,21 @@ | Platform | Language | Where | Reads | Writes | Specification version | |---|---|---|---|---|---| -| openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.2 | -| openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.2 (through openalea.fspm_tf) | -| fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.2 | -| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.2 | -| fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.2 | -| GroIMP | Java | `bindings/java/groimp/` (`GroImp`) and `bindings/java/groimp-plugin/` (a GroIMP 3.0 plugin: File > Export to `.glb`, CLI `export3d`) | yes: graphs and fields into the model's node classes (adapter) | yes: scales, topology, edge bits, managed fields, the scene's geometry by node | 0.2 | -| FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.2 | -| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.2 | -| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.2 | +| openalea.fspm_tf (reference codec) | Python | this repository, `bindings/python/` | yes | yes | 0.3 | +| openalea.metafspm | Python | `openalea.metafspm.io.fspm_tf` (branch release2026) | yes: its own frames, rebuilt bit for bit | yes | 0.3 (through openalea.fspm_tf) | +| fspm_tf (C++ codec) | C++ | this repository, `bindings/cpp/` | yes | yes | 0.3 | +| CPlantBox | C++ | `bindings/cpp/cplantbox/` (`fspm_tf::cplantbox`), to move into CPlantBox | yes: static root systems, from any graph with segment ends | yes: organs, segments, geometry, soil grid, mapping | 0.3 | +| fspm-tf (Java codec) | Java | this repository, `bindings/java/` | yes | yes | 0.3 | +| GroIMP | Java | `bindings/java/groimp/` (`GroImp`) and `bindings/java/groimp-plugin/` (a GroIMP 3.0 plugin: File > Export to `.glb`, CLI `export3d`) | yes: graphs and fields into the model's node classes (adapter) | yes: scales, topology, edge bits, managed fields, the scene's geometry by node | 0.3 | +| FspmTF.jl (Julia codec) | Julia | this repository, `bindings/julia/FspmTF/` | yes | yes | 0.3 | +| MultiScaleTreeGraph.jl (VirtualPlantLab, PlantSimEngine) | Julia | FspmTF.jl's package extension: `from_mtg`, `to_mtg` | yes: any graph as an MTG; its own MTGs rebuilt as they were | yes | 0.3 | +| VirtualPlantLab (PlantGraphs, PlantGeomTurtle) | Julia | FspmTF.jl's package extensions: `from_graph`, `to_graph`, `add_geometry!` | yes: nodes rebuilt from their fields | yes: nodes, fields, turtle geometry | 0.3 | -Coupling sessions (specification §11, draft for 0.3): the Python package holds the coordinator and a participant; -FspmTF.jl a participant (both transports); metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for -another platform's model in a metafspm model, tested with a VirtualPlantLab model (PlantGraphs). +Coupling sessions (specification §11): the Python package holds the coordinator and a participant; FspmTF.jl, the C++ +and the Java codecs a participant each. All four pass the coupling conformance sessions over both transports (ZeroMQ: +pyzmq, ZMQ.jl, libzmq, JeroMQ). metafspm's `RemoteComponent` (`openalea.metafspm.io.coupling`) stands for another +platform's model in a metafspm model, tested with a VirtualPlantLab model (PlantGraphs). CPlantBox's and GroIMP's logs +are replayed as participants in their CI jobs. Codecs (a language's reading and writing of the format) live in this repository, in `bindings//`; adapters (a platform's model to and from the codec's tables) in the platforms' repositories, or as a codec's optional diff --git a/spec/schema/coupling.schema.json b/spec/schema/coupling.schema.json index 3162ae9..acb93d0 100644 --- a/spec/schema/coupling.schema.json +++ b/spec/schema/coupling.schema.json @@ -33,10 +33,22 @@ "properties": { "participant": {"$ref": "#/$defs/identifier"}, "platform": {"type": "string"}, "entitySets": {"type": "array", "items": {"$ref": "#/$defs/entitySet"}}, - "variables": {"type": "array", "items": {"$ref": "#/$defs/variable"}} + "variables": {"type": "array", "items": {"$ref": "#/$defs/variable"}}, + "mappings": {"type": "array", "items": {"$ref": "#/$defs/mapping"}} }, "additionalProperties": false }, + "end": { + "type": "object", "required": ["dataStructure", "entitySet"], + "properties": {"dataStructure": {"$ref": "#/$defs/identifier"}, "entitySet": {"$ref": "#/$defs/name"}}, + "additionalProperties": false + }, + "mapping": { + "type": "object", "required": ["name", "source", "target"], + "properties": {"name": {"$ref": "#/$defs/identifier"}, "source": {"$ref": "#/$defs/end"}, + "target": {"$ref": "#/$defs/end"}}, + "additionalProperties": false + }, "link": { "type": "object", "required": ["receiver", "variable", "provider", "source"], "properties": { @@ -44,7 +56,7 @@ "provider": {"$ref": "#/$defs/identifier"}, "source": {"$ref": "#/$defs/name"}, "entitySet": {"$ref": "#/$defs/name"}, "sourceEntitySet": {"$ref": "#/$defs/name"}, "factor": {"type": "number"}, "aggregation": {"enum": ["sum", "mean", "max", "min", "copy"]}, - "interpolation": {"enum": ["hold", "linear"]} + "interpolation": {"enum": ["hold", "linear"]}, "mapping": {"$ref": "#/$defs/identifier"} }, "additionalProperties": false }, diff --git a/spec/schema/fspm_scene.schema.json b/spec/schema/fspm_scene.schema.json index b0bccda..08853df 100644 --- a/spec/schema/fspm_scene.schema.json +++ b/spec/schema/fspm_scene.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.2/fspm_scene.schema.json", + "$id": "https://github.com/openalea/fspm-TF/schema/0.3/fspm_scene.schema.json", "title": "FSPM_scene", - "description": "The FSPM_scene root extension of a glTF 2.0 asset (fspm-TF 0.2): one frame of a scene.", + "description": "The FSPM_scene root extension of a glTF 2.0 asset (fspm-TF 0.3): one frame of a scene.", "type": "object", "required": ["version", "frame", "dataStructures"], "properties": { diff --git a/spec/schema/index.schema.json b/spec/schema/index.schema.json index 6b90d64..dd0be5f 100644 --- a/spec/schema/index.schema.json +++ b/spec/schema/index.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/openalea/fspm-TF/schema/0.2/index.schema.json", + "$id": "https://github.com/openalea/fspm-TF/schema/0.3/index.schema.json", "title": "fspm-TF sequence index", - "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (fspm-TF 0.2).", + "description": "scene.fspm.json: the frames of a sequence, one GLB file each, in time order (fspm-TF 0.3).", "type": "object", "required": ["fspmIndex", "timeUnit", "frames"], "properties": { diff --git a/spec/specification.md b/spec/specification.md index d99eb13..f387ae6 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -1,6 +1,6 @@ -# fspm-TF 0.2 specification +# fspm-TF 0.3 specification -Status: draft (2026-11-02). 0.2 adds to 0.1 (see "Changes from 0.1" at the end); a 0.1 file is a valid 0.2 file. The key words MUST, SHOULD and MAY are to be read as in RFC 2119 (https://www.rfc-editor.org/rfc/rfc2119). +Status: draft (2026-11-03). 0.3 adds coupling sessions and partial frames to 0.2 (see "Changes from 0.2" at the end); a 0.1 or 0.2 file is a valid 0.3 file. The key words MUST, SHOULD and MAY are to be read as in RFC 2119 (https://www.rfc-editor.org/rfc/rfc2119). fspm-TF is an exchange format for whole functional-structural plant model scenes: multiscale plant graphs, their variables and parameters, their geometry, and the environment's grids, at successive time steps. A **frame** (one @@ -16,7 +16,7 @@ The language-neutral files of this version are part of the specification: | `spec/vocabulary/labels.csv` | entity labels, their scales and Plant Ontology terms (§6) | | `spec/vocabulary/enums.json` | the values of `edge_type`, `kind`, `role`, data structure and grid kinds; the base scales | | `conformance/` | reference files and their expected values (§9): the specification as test cases | -| `spec/schema/coupling.schema.json` | JSON Schema of a coupling session's messages (§11, draft for 0.3) | +| `spec/schema/coupling.schema.json` | JSON Schema of a coupling session's messages (§11) | When these sources disagree, `specification.md` prevails, then the schemas, then the conformance files; the disagreement is a defect of the lower one, to be fixed. @@ -71,7 +71,7 @@ A frame holding only data structures with parameters, and no geometry, is a **pa A frame MUST be a GLB file (glTF 2.0 binary): -1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.2. +1. All buffer views MUST be in the GLB's binary chunk. No external `.bin` files and no data URIs are allowed in 0.3. 2. Every buffer view holding table values MUST start at a multiple of 8 bytes. The JSON chunk MUST be padded with spaces and the binary chunk with zeros to 8 bytes, as `EXT_structural_metadata` requires. The buffer's `byteLength` SHOULD be the binary chunk's padded length (glTF-Validator warns of more than 3 bytes of padding @@ -92,7 +92,7 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: ```text { - "version": "0.2", + "version": "0.3", "vocabulary": "0.2", "frame": {"time": 1.0, "timeUnit": "d", "step": 24}, "writer": {"platform": "...", "model": "..."}, @@ -145,7 +145,7 @@ A column is one property of the table's class: | text | `STRING` | | labels, edge types and relation labels | `ENUM` (§4.2) | -- Variable-length arrays and the other glTF metadata types are not part of 0.2. +- Variable-length arrays and the other glTF metadata types are not part of 0.3. - **NaN.** NaN and infinities are forbidden in property tables. A float column with missing values (NaN) declares `noData`: the most negative finite value of its type, −1.7976931348623157e308 for FLOAT64 or −3.4028234663852886e38 for FLOAT32, stored where the value is NaN. Readers MUST read `noData` back as NaN. A @@ -300,7 +300,7 @@ two platforms. A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json`, following `index.schema.json`: ```text -{"fspmIndex": "0.2", "timeUnit": "d", +{"fspmIndex": "0.3", "timeUnit": "d", "frames": [{"time": 0.0, "step": 0, "file": "frame_000000.glb"}, ...]} ``` @@ -312,7 +312,7 @@ A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json` given again) keeps, across a sequence, a map from its entities to the ids it writes. - **Ids are not positions.** Ids may be sparse (e.g. offsets keeping them stable as a plant grows). Readers SHOULD NOT size arrays by the largest id. -- **Packing.** Packing a sequence into one GLB is not part of 0.2. +- **Packing.** Packing a sequence into one GLB is not part of 0.3. ## 9. Conformance @@ -330,19 +330,33 @@ sequence: | `missing_values` | integer, text and boolean columns with missing values; a date column; `unknown` edge types and a `sibling_order` column; a `mixed` entity set; a labels map; bearer relations between two pairs of sets | | `growing_sequence/` | three frames of a growing root system and their index | -An implementation conforms to 0.2 when two things hold: +An implementation conforms to 0.3 when two things hold: - **Reading:** what it reads from every conformance GLB equals the expected JSON. Floats are compared exactly; NaN and missing values are null in the JSON, +∞ and −∞ the strings `"Infinity"` and `"-Infinity"`; positions are the FLOAT32 values. - **Writing:** what it writes passes `fspm-tf validate` with no error, and Khronos' glTF-Validator with no error. +A codec's participant (§11) conforms when it runs every **coupling session** of `conformance/coupling/` as expected, +over both transports. A session's folder holds the coordinator's settings (`session.json`), a peer (a sequence the +coordinator replays), the subject's declaration and the updates it sends (`subject/`), and the expected values +(`expected/turn_.json`: what the subject reads from each turn; `expected/record/frame_.json`: the +session's record). The participant under test is a program run as `COMMAND SESSION OUT`: it joins with the +declaration, writes what it reads from each turn to `OUT/turn_.json` (in the expected values' layout), and sends +`subject/update_.glb` as its codec reads and writes it again. `fspm-tf couple-conformance conformance/coupling +--participant COMMAND` runs and compares them all. + +| Session | Covers | +|---|---| +| `two_scales` | a subject refining a replayed peer's organs into segments: new entities, removals cascading to the subject's segments, a unit conversion and a factor, a copy to finer entities | +| `two_data_structures` | a subject owning another data structure, linked to the peer's by the mapping it provides: a sum through a mapping, mappings in updates | + ## 10. Keeping what one does not read -A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.2 this covers +A reader that writes a file back SHOULD keep the content it does not interpret, as RSML requires. In 0.3 this covers the glTF root's other extensions and `extras`, when they hold no buffer views. -## 11. Coupling (draft for 0.3) +## 11. Coupling Platforms couple their models through fspm-TF in a **session**: each **participant** is a platform's model running in its own process, or a recorded sequence (§11.6), and a **coordinator** runs the session. The coordinator holds the @@ -385,12 +399,16 @@ A message is a JSON header and, for a turn or an update, a payload: a partial fr - `follows`: it reads. - `variables`: each one with `dataStructure`, `entitySet`, `name`, `direction` (`provides`: an output; `requires`: an input), `unit` (UCUM), and optionally `standardName`, `kind`, `optional`. +- `mappings` (optional): the mappings it provides, each with its `name`, `source` and `target` (`dataStructure`, + `entitySet`). Its updates carry them as the frame's mappings (§1) of those names; each one sent replaces the + previous. **Link.** A link names the input it feeds and the output that feeds it: - the input: `receiver`, `variable`, `entitySet` (when the name is ambiguous); - the output: `provider`, `source`, `sourceEntitySet`; - its conversion: `factor` (applied after the unit conversion), `aggregation` and `interpolation` (`hold` or - `linear`, §11.6). + `linear`, §11.6); +- `mapping`: between two data structures, the name of a mapping a participant provides, read in either direction. **Aggregation.** A link may change the scale: - `sum`, `mean`, `max` or `min` from a finer entity set, through the `complex` chain. The default is `sum` for an @@ -398,6 +416,13 @@ A message is a JSON header and, for a turn or an update, a payload: a partial fr - `copy` from a coarser one. An extensive source needs `copy` stated explicitly, because copying it to finer entities counts it several times. +**Between two data structures**, a link goes through its mapping (weights `w` between source entities `s` and target +entities `t`): +- `sum` (the default for an extensive source) shares each source's value among its targets in proportion to their + weights: `x_t = Σ_s x_s · w_st / Σ_t' w_st'`. The total amount is kept. +- `mean` (the default otherwise) gives each target the weighted mean of its sources: `x_t = Σ_s w_st · x_s / Σ_s w_st`. +- A target no source reaches has no value. + Formulas are not part of a link: a receiver computes them from its linked inputs. ### 11.3 Resolution @@ -413,7 +438,7 @@ It refuses the session, and stops every participant with the reason, when any of - the units are not convertible; - an entity set has two owners; - a stage holds two participants that change one entity set's topology; -- a link joins two data structures, which needs a mapping (not in this draft). +- a link joins two data structures without a mapping, or with a mapping whose ends are not the link's. ### 11.4 Steps and stages @@ -445,6 +470,10 @@ It refuses the session, and stops every participant with the reason, when any of variables on them. When an entity splits, the participant shares its own extensive quantities and dilutes its own concentrations by its rules; the entity's lineage is its `parent`. - **Labels.** An entity created without a label takes its entity set's name. +- **Scales a participant does not know.** A participant declares only the entity sets it works on. One that grows a + coarse architecture (say, organs) needs to know nothing of the finer entities another participant refines under + them (segments, cells): it is never sent them, and when it removes an organ, the coordinator removes the organ's + segments and tells their refiner. ### 11.6 Transports @@ -470,6 +499,15 @@ Both transports carry the same messages. - Between two recorded times, its values are those of the last frame at or before the session's time (`hold`), or interpolated linearly for links stating `"interpolation": "linear"`. +## Changes from 0.2 + +All additions; a 0.2 file is a valid 0.3 file. + + +- §11: coupling sessions: partial frames (`FSPM_scene.update`), the messages (`spec/schema/coupling.schema.json`), + declarations and rights, links (units, factors, changes of scale, mappings between data structures), stages, the + shared structure, two transports, replay; conformance sessions (§9). + ## Changes from 0.1 All additions; a 0.1 file is a valid 0.2 file. From ef8fec0906cab9be1c9f2bb984b2e649589a9770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 15:15:02 +0100 Subject: [PATCH 21/39] =?UTF-8?q?Frame=20views=20(specification=20=C2=A78.?= =?UTF-8?q?1,=20draft=20for=200.3):=20views.json,=20an=20optional=20compan?= =?UTF-8?q?ion=20of=20a=20sequence=20(spec/schema/views.schema.json),=20na?= =?UTF-8?q?med=20configurations=20of=20layers;=20openalea.fspm=5Ftf.views?= =?UTF-8?q?=20(reading,=20writing,=20checking=20against=20a=20frame,=20the?= =?UTF-8?q?=20selection,=20display=20options);=20the=20ParaView=20reader's?= =?UTF-8?q?=20Views=20and=20Configuration,=20one=20block=20per=20layer=20h?= =?UTF-8?q?olding=20its=20entities=20only=20(frame=5Fto=5Fvtk(frame,=20vie?= =?UTF-8?q?ws=3D,=20configuration=3D));=20the=20macros=20apply=5Ffspm=5Ftf?= =?UTF-8?q?=5Fviews.py=20and=20save=5Ffspm=5Ftf=5Fviews.py;=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 11 + .../python/src/openalea/fspm_tf/schema.py | 3 +- bindings/python/src/openalea/fspm_tf/views.py | 292 ++++++++++++++++++ bindings/python/test/test_paraview_reader.py | 64 ++++ bindings/python/test/test_views.py | 251 +++++++++++++++ doc/api.md | 8 + doc/reading_the_specification.md | 1 + doc/using.md | 43 +++ paraview/apply_fspm_tf_views.py | 99 ++++++ paraview/check_in_paraview.py | 37 ++- paraview/fspm_tf_reader.py | 165 ++++++++-- paraview/save_fspm_tf_views.py | 68 ++++ spec/schema/views.schema.json | 67 ++++ spec/specification.md | 36 +++ 14 files changed, 1121 insertions(+), 24 deletions(-) create mode 100644 bindings/python/src/openalea/fspm_tf/views.py create mode 100644 bindings/python/test/test_views.py create mode 100644 paraview/apply_fspm_tf_views.py create mode 100644 paraview/save_fspm_tf_views.py create mode 100644 spec/schema/views.schema.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 91728ee..e2f1437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased: frame views (specification §8.1, draft for 0.3) + +- Views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): named configurations + of layers, each a mesh's, segments' or grid's entities selected by `where` conditions, coloured by a variable (or a + solid colour), with a colour map, range, log scale, opacity, visibility, representation, and tubes for segments. +- `openalea.fspm_tf.views`: reading, writing and checking views (the schema, and every name against a frame), the + selection, and the display options the ParaView macros apply and save back. +- The ParaView reader: "Views" and "Configuration" (a drop-down), one block per layer holding only its entities; + `frame_to_vtk(frame, views=, configuration=)`, unchanged without views. The macros `apply_fspm_tf_views.py` (each + layer its own pipeline item) and `save_fspm_tf_views.py` (the items' display options back into `views.json`). + ## Unreleased: specification 0.3 (draft), coupling platforms - Specification 0.3: the frames say 0.3 (0.1 and 0.2 files stay valid); "Changes from 0.2"; the conformance files diff --git a/bindings/python/src/openalea/fspm_tf/schema.py b/bindings/python/src/openalea/fspm_tf/schema.py index 94f55d7..b9885ce 100644 --- a/bindings/python/src/openalea/fspm_tf/schema.py +++ b/bindings/python/src/openalea/fspm_tf/schema.py @@ -1,5 +1,6 @@ """The JSON Schemas of fspm-TF (spec/schema/): fspm_scene.schema.json (the FSPM_scene extension), -index.schema.json (a sequence's scene.fspm.json) and coupling.schema.json (a coupling session's messages).""" +index.schema.json (a sequence's scene.fspm.json), views.schema.json (a sequence's views.json) and coupling.schema.json +(a coupling session's messages).""" import json from functools import lru_cache diff --git a/bindings/python/src/openalea/fspm_tf/views.py b/bindings/python/src/openalea/fspm_tf/views.py new file mode 100644 index 0000000..9307f0a --- /dev/null +++ b/bindings/python/src/openalea/fspm_tf/views.py @@ -0,0 +1,292 @@ +""" +Frame views (specification §8.1): views.json, an optional companion of a sequence or a frame, names configurations of +layers; each layer is a part of a data structure (a mesh, segments or a grid), the entities it selects and how it is +drawn:: + + {"fspm_tf_views": 1, "active": "default", + "configurations": {"default": {"layers": [ + {"name": "roots", "source": {"data_structure": "plants", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "colormap": "Viridis (matplotlib)", + "range": "auto", "log": false, "opacity": 1, "visible": true, "representation": "Surface", + "tube": {"radius": "radius", "scale": 1}}]}}} + + read_views(path), write_views(path, views) + views_path(path, views=None) the views file of a frame or sequence: *views*, else the sibling views.json + check_views(views, frame=None) a Report: the schema, and every name against *frame* + configuration(views, name=None) (its name, its layers, each with its defaults filled) + selected(table, where) the rows of *table* where every condition holds + +ParaView's macros (paraview/apply_fspm_tf_views.py, save_fspm_tf_views.py) go through display_settings(layer) (a +layer to its display options) and layer_from_display(layer, display) (back, to save them), plain dicts both ways. +""" +import copy +import json +import os +import re + +import jsonschema +import numpy as np + +from openalea.fspm_tf import schema as schemas + +VIEWS_FILE = "views.json" +FORMAT = 1 +DEFAULTS = {"color_by": None, "solid_color": [1., 1., 1.], "colormap": "Viridis (matplotlib)", "range": "auto", + "log": False, "opacity": 1., "visible": True, "representation": "Surface"} +TUBE_RADIUS = "tube_radius" # the point data a segments layer's Tube varies its radius by +SEGMENT_ENDS = (("proximal_end_x", "x1"), ("proximal_end_y", "y1"), ("proximal_end_z", "z1"), + ("distal_end_x", "x2"), ("distal_end_y", "y2"), ("distal_end_z", "z2")) + +_CONDITION = re.compile(r"^\s*(<=|>=|==|!=|<|>)\s*([-+]?(?:\d+\.?\d*|\.\d+)(?:[eE][-+]?\d+)?)\s*$") +_COMPARISONS = {"<": np.less, "<=": np.less_equal, ">": np.greater, ">=": np.greater_equal, "==": np.equal, + "!=": np.not_equal} + + +# ── Files ──────────────────────────────────────────────────────────────── + +def views_path(path: str, views: str = None) -> str: + """The views file of the frame or sequence *path*: *views* when given (relative to *path*'s folder), else the + views.json next to it; None when that file does not exist.""" + folder = os.path.dirname(os.path.abspath(path)) if path else os.getcwd() + found = os.path.join(folder, views) if views else os.path.join(folder, VIEWS_FILE) + return found if os.path.exists(found) else None + + +def read_views(path: str) -> dict: + with open(path, encoding="utf-8") as file: + return json.load(file) + + +def write_views(path: str, views: dict) -> None: + """Write *views* to *path* (checked against the schema first).""" + report = check_views(views) + if not report.ok: + raise ValueError(str(report)) + temporary = path + ".part" + with open(temporary, "w", encoding="utf-8", newline="\n") as file: + json.dump(views, file, indent=1) + file.write("\n") + os.replace(temporary, path) # a reader never sees half a file + + +def configuration_names(views: dict) -> list: + """The configurations' names, the active one first.""" + names = list(views.get("configurations", {})) + active = views.get("active") + return ([active] if active in names else []) + [name for name in names if name != active] + + +def configuration(views: dict, name: str = None) -> tuple: + """(name, layers) of the configuration *name* (None: the active one, else the first), each layer with its + defaults filled.""" + names = configuration_names(views) + if not names: + raise ValueError("the views have no configuration") + name = name or names[0] + if name not in views["configurations"]: + raise KeyError(f"no configuration '{name}' in the views (configurations: {names})") + return name, [filled(layer) for layer in views["configurations"][name]["layers"]] + + +def filled(layer: dict) -> dict: + """*layer* with the defaults of what it leaves out (a tube's scale: 1).""" + complete = dict(copy.deepcopy(DEFAULTS), **copy.deepcopy(layer)) + if "tube" in complete: + complete["tube"].setdefault("scale", 1.) + return complete + + +# ── Checks ─────────────────────────────────────────────────────────────── + +def check_views(views: dict, frame=None): + """A validate.Report of *views*: the schema, an active configuration that exists, unique layer names, and with + *frame* every name a layer gives (data structure, mesh, entity sets, columns) found in it.""" + from openalea.fspm_tf.validate import Report + report = Report("views") + for error in jsonschema.Draft202012Validator(schemas.load("views")).iter_errors(views): + report.error(f"views/{'/'.join(str(part) for part in error.absolute_path)}", error.message) + if not report.ok: + return report + if "active" in views and views["active"] not in views["configurations"]: + report.error("views/active", f"no configuration '{views['active']}'") + for name, content in views["configurations"].items(): + seen = set() + for number, layer in enumerate(content["layers"]): + where = f"{name}/layers/{number} ({layer['name']})" + if layer["name"] in seen: + report.error(where, "another layer of the configuration has this name") + seen.add(layer["name"]) + if frame is not None: + _check_layer(layer, frame, where, report) + return report + + +def segment_columns(entity_set) -> list: + """The six segment-end columns of *entity_set* (by standard name, else x1 ... z2), or [] when it has none.""" + by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() + if variable.standard_name} + found = [] + for standard, plain in SEGMENT_ENDS: + name = by_standard.get(standard, plain) + column = entity_set.columns.get(name) + if column is None or column.dtype == object or column.dtype.kind not in "fiu": + return [] + found.append(name) + return found + + +def _sets_of(data_structure, layer: dict, where: str, report) -> tuple: + """(the entity sets the layer may select and colour by, the name of the set its entities are in), or None.""" + source = layer["source"] + sets = {entity_set.name: entity_set for entity_set in data_structure.entity_sets} + if source["kind"] == "mesh": + meshes = {mesh.name: mesh for mesh in data_structure.meshes} + if source["mesh"] not in meshes: + report.error(where, f"no mesh '{source['mesh']}' in '{data_structure.id}' (meshes: {sorted(meshes)})") + return None + return {name: sets[name] for name in meshes[source["mesh"]].feature_rows if name in sets}, None + if source["kind"] == "segments": + entity_set = sets.get(source["entity_set"]) + if entity_set is None: + report.error(where, f"no entity set '{source['entity_set']}' in '{data_structure.id}'") + return None + if not segment_columns(entity_set): + report.error(where, f"'{entity_set.name}' has no segment ends (proximal_end_x ... distal_end_z, or x1 ... " + "z2)") + return None + return {entity_set.name: entity_set}, entity_set.name + if data_structure.grid is None: + report.error(where, f"'{data_structure.id}' has no grid") + return None + return {data_structure.grid.cells: sets[data_structure.grid.cells]}, data_structure.grid.cells + + +def _check_layer(layer: dict, frame, where: str, report) -> None: + source = layer["source"] + found = {data_structure.id: data_structure for data_structure in frame.data_structures} + data_structure = found.get(source["data_structure"]) + if data_structure is None: + report.error(where, f"no data structure '{source['data_structure']}' in the frame (data structures: " + f"{sorted(found)})") + return + usable = _sets_of(data_structure, layer, where, report) + if usable is None: + return + sets, _ = usable + for part, columns in (("select", list(layer.get("select", {}).get("where", {}))), + ("color_by", [(layer.get("color_by") or {}).get("variable")]), + ("tube", [layer.get("tube", {}).get("radius")])): + names = [column for column in columns if column] + if not names: + continue + set_name = (layer[part].get("entity_set") if part != "tube" else source.get("entity_set")) + if set_name not in sets: + report.error(f"{where}/{part}", f"'{set_name}' is not an entity set this layer's entities are in " + f"({sorted(sets)})") + continue + for column in names: + if column not in sets[set_name].columns: + report.error(f"{where}/{part}", f"no column '{column}' in '{data_structure.id}/{set_name}'") + if layer.get("tube") is not None and source["kind"] != "segments": + report.error(f"{where}/tube", "a tube is for segments") + + +# ── Selection ──────────────────────────────────────────────────────────── + +def _holds(column: np.ndarray, condition) -> np.ndarray: + missing = np.ma.getmaskarray(column) if np.ma.isMaskedArray(column) else np.zeros(len(column), dtype=bool) + values = np.ma.getdata(column) + comparison = _CONDITION.match(condition) if isinstance(condition, str) else None + if comparison is not None and values.dtype != object: + operator, threshold = comparison.group(1), float(comparison.group(2)) + with np.errstate(invalid="ignore"): + result = _COMPARISONS[operator](values.astype(np.float64), threshold) + elif isinstance(condition, (list, tuple)): + result = np.isin(values, np.asarray(condition, dtype=object if values.dtype == object else None)) + else: + result = values == condition + return np.asarray(result, dtype=bool) & ~missing + + +def selected(table, where: dict) -> np.ndarray: + """The rows of *table* (an entity set) where every {column: condition} of *where* holds: a condition is a value, + a list of values (labels and other text by name), or a comparison (">=0", "!=2"); a missing value never holds.""" + result = np.ones(table.count, dtype=bool) + for column, condition in where.items(): + result &= _holds(table.columns[column], condition) + return result + + +# ── Display options (ParaView's macros) ────────────────────────────────── + +def block_selector(name: str) -> str: + """The selector of the reader's block *name* in ParaView's ExtractBlock: /Root/ and the name as ParaView makes it a + valid node name (vtkDataAssembly::MakeValidNodeName: characters other than letters, digits, '_', '-' and '.' + replaced by '_', and '_' before a first character that is not a letter or '_').""" + valid = re.sub(r"[^A-Za-z0-9_.\-]", "_", name) + if not re.match(r"[A-Za-z_]", valid): + valid = "_" + valid + return f"/Root/{valid}" + + +def colour_array(layer: dict, arrays=()) -> str: + """The cell data array a layer is coloured by: ".", or its codes ". + _code" when *arrays* (the block's arrays) hold those instead (a text variable); None for a solid colour.""" + colour = layer.get("color_by") + if not colour: + return None + name = f"{colour['entity_set']}.{colour['variable']}" + if arrays and name not in arrays and f"{name}_code" in arrays: + return f"{name}_code" + return name + + +def display_settings(layer: dict, arrays=()) -> dict: + """What ParaView's "Apply fspm-TF views" does for *layer*: the block to extract (selector), a Tube on + TUBE_RADIUS or not, the colouring (a cell array, its colour map, its range: None for the data's), the solid + colour, the opacity, the visibility and the representation.""" + layer = filled(layer) + return {"name": layer["name"], "selector": block_selector(layer["name"]), + "tube": {"radius_array": TUBE_RADIUS} if layer.get("tube") else None, + "color_array": colour_array(layer, arrays), "colormap": layer["colormap"], + "range": None if layer["range"] == "auto" else [float(value) for value in layer["range"]], + "log": bool(layer["log"]), "solid_color": [float(value) for value in layer["solid_color"]], + "opacity": float(layer["opacity"]), "visible": bool(layer["visible"]), + "representation": layer["representation"]} + + +def layer_from_display(layer: dict, display: dict) -> dict: + """*layer* updated from a ParaView item's display options, as "Save fspm-TF views" reads them: visible, + opacity, representation, color_array (None: a solid colour), solid_color, range (None: the data's), log and + colormap, each optional (absent: the layer's own). A "._code" array is the variable.""" + updated = copy.deepcopy(layer) + for key in ("visible", "opacity", "representation", "solid_color", "log", "colormap"): + if display.get(key) is not None: + updated[key] = display[key] + if "opacity" in updated: + updated["opacity"] = float(updated["opacity"]) + if "range" in display: + updated["range"] = "auto" if display["range"] is None else [float(value) for value in display["range"]] + if "color_array" in display: + array = display["color_array"] + if not array: + updated["color_by"] = None + else: + entity_set, _, variable = array.partition(".") + previous = layer.get("color_by") or {} + if variable.endswith("_code") and previous.get("variable") != variable: + variable = variable[:-len("_code")] + updated["color_by"] = {"entity_set": entity_set, "variable": variable} + return updated + + +def save_configuration(views: dict, name: str, layers: list, active: bool = True) -> dict: + """A copy of *views* with configuration *name* set to *layers* (overwritten, or added), made active when + *active*.""" + saved = copy.deepcopy(views) + saved.setdefault("fspm_tf_views", FORMAT) + saved.setdefault("configurations", {})[name] = {"layers": copy.deepcopy(list(layers))} + if active: + saved["active"] = name + return saved diff --git a/bindings/python/test/test_paraview_reader.py b/bindings/python/test/test_paraview_reader.py index 7fcfaa6..93373f1 100644 --- a/bindings/python/test/test_paraview_reader.py +++ b/bindings/python/test/test_paraview_reader.py @@ -125,3 +125,67 @@ def test_missing_values_are_nan_and_code_minus_one(): lines = blocks_of(frame)[("root_system", "SubOrgan segments")] assert np.isnan(cell_array(lines, "SubOrgan.rank")[0]) and cell_array(lines, "SubOrgan.rank")[1] == 1. assert cell_array(lines, "SubOrgan.tissue_code")[0] == -1 and cell_array(lines, "SubOrgan.tissue_code")[1] == 0 + + +# ── Views: one block per layer ─────────────────────────────────────────── + +def layers_of(frame, views, configuration=None) -> dict: + output = reader.frame_to_vtk(frame, views=views, configuration=configuration) + return {output.GetMetaData(number).Get(output.NAME()): output.GetBlock(number) + for number in range(output.GetNumberOfBlocks())} + + +def test_views_give_a_block_per_layer_with_its_entities_only(): + from test_views import roots_views + frame = generate.root_system() + source = frame.data_structures[0] + blocks = layers_of(frame, roots_views()) + assert list(blocks) == ["first order", "roots"] # the hidden layer too + mesh = blocks["first order"] + triangles = source.meshes[0].feature_rows["Axis"][::3] + first_order = source.entity_set("Axis").columns["branching_order"][triangles] == 1 + assert 0 < mesh.GetNumberOfCells() == first_order.sum() < len(triangles) + assert set(cell_array(mesh, "Axis.branching_order")) == {1.} + assert mesh.GetCellData().GetScalars().GetName() == "SubOrgan.water_potential" + segments = source.entity_set("SubOrgan") + lines = blocks["roots"] + kept = (segments.columns["label"] == "RootSegment") & (segments.columns["length"] > 0) + assert lines.GetNumberOfCells() == kept.sum() + np.testing.assert_array_equal(cell_array(lines, "SubOrgan.radius"), segments.columns["radius"][kept]) + radius = vtk_to_numpy(lines.GetPointData().GetArray("tube_radius")) + np.testing.assert_array_equal(radius, np.concatenate([segments.columns["radius"][kept]] * 2) * 2) + assert lines.GetCellData().GetScalars().GetName() == "SubOrgan.label_code" # text: its codes + plain = layers_of(frame, roots_views(), "plain")["all"] + assert plain.GetNumberOfCells() == len(triangles) and plain.GetCellData().GetScalars() is None + + +def test_grid_layers_hide_the_cells_not_selected(): + frame = generate.soil_and_roots() + layer = {"name": "wet", "source": {"data_structure": "soil", "kind": "grid"}, + "select": {"entity_set": "Voxel", "where": {"soil_water_potential": ">=-15000"}}, + "color_by": {"entity_set": "Voxel", "variable": "soil_water_potential"}, "opacity": 0.3} + image = layers_of(frame, {"fspm_tf_views": 1, "configurations": {"soil": {"layers": [layer]}}})["wet"] + ghosts = vtk_to_numpy(image.GetCellData().GetArray("vtkGhostType")) + voxels = frame.data_structures[1].entity_set("Voxel") + order = np.arange(12).reshape((2, 2, 3)).ravel(order="F") + np.testing.assert_array_equal(ghosts == 0, voxels.columns["soil_water_potential"][order] >= -15000) + assert 0 < (ghosts == 0).sum() < 12 + assert image.GetCellData().GetScalars().GetName() == "Voxel.soil_water_potential" + adaptive = generate.adaptive_soil() + layer = {"name": "fine", "source": {"data_structure": "soil", "kind": "grid"}, + "select": {"entity_set": "Cell", "where": {"level": ">0"}}} + cells = layers_of(adaptive, {"fspm_tf_views": 1, "configurations": {"soil": {"layers": [layer]}}})["fine"] + assert cells.GetNumberOfCells() == (adaptive.data_structures[0].entity_set("Cell").columns["level"] > 0).sum() + + +def test_views_from_a_file_and_wrong_names(tmp_path): + from openalea.fspm_tf import views + from test_views import roots_views + path = str(tmp_path / "views.json") + views.write_views(path, roots_views()) + assert list(layers_of(generate.root_system(), path)) == ["first order", "roots"] + empty = layers_of(generate.wheat_shoot(), path) # names the frame lacks: empty blocks + assert list(empty) == ["first order", "roots"] and all(block.GetNumberOfCells() == 0 for block in empty.values()) + with pytest.raises(ValueError, match="fspm_tf_views"): + reader.frame_to_vtk(generate.root_system(), views={"fspm_tf_views": 2, "configurations": {}}) + assert reader.frame_to_vtk(generate.root_system()).GetNumberOfBlocks() == 1 # no views: as before diff --git a/bindings/python/test/test_views.py b/bindings/python/test/test_views.py new file mode 100644 index 0000000..6a17103 --- /dev/null +++ b/bindings/python/test/test_views.py @@ -0,0 +1,251 @@ +"""Frame views (openalea.fspm_tf.views, specification §8.1): the schema, the names checked against a frame, the +selection, the display options ParaView's macros apply and save back, and the macros themselves on a stand-in for +paraview.simple.""" +import copy +import os +import sys +import types + +import numpy as np +import pytest + +import generate +from conftest import ROOT + +from openalea.fspm_tf import views + + +def roots_views() -> dict: + return {"fspm_tf_views": 1, "active": "default", "configurations": { + "default": {"layers": [ + {"name": "first order", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "select": {"entity_set": "Axis", "where": {"branching_order": [1]}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}}, + {"name": "roots", "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "label"}, "tube": {"radius": "radius", "scale": 2}, + "visible": False}]}, + "plain": {"layers": [ + {"name": "all", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "color_by": None, "solid_color": [0.5, 0.4, 0.3], "opacity": 0.3}]}}} + + +def test_a_views_document_follows_the_schema(): + assert views.check_views(roots_views()).ok + wrong = roots_views() + del wrong["configurations"]["default"]["layers"][0]["source"]["mesh"] # a mesh layer names its mesh + wrong["configurations"]["plain"]["layers"][0]["range"] = [0.] + report = views.check_views(wrong) + assert len(report.errors) == 2, str(report) + + +def test_the_active_configuration_exists_and_layer_names_are_unique(): + document = roots_views() + document["active"] = "missing" + plain = document["configurations"]["plain"]["layers"] + plain.append(copy.deepcopy(plain[0])) + report = views.check_views(document) + assert {issue.where for issue in report.errors} == {"views/active", "plain/layers/1 (all)"}, str(report) + + +def test_names_are_checked_against_a_frame(): + frame = generate.root_system() + assert views.check_views(roots_views(), frame).ok + document = roots_views() + layers = document["configurations"]["default"]["layers"] + layers[0]["source"]["mesh"] = "shoots" + layers[1]["select"]["where"]["width"] = ">0" + layers[1]["color_by"]["entity_set"] = "Axis" # segments are coloured by their own entity set + document["configurations"]["plain"]["layers"][0]["source"]["data_structure"] = "soil" + messages = [issue.message for issue in views.check_views(document, frame).errors] + assert len(messages) == 4, messages + assert any("no mesh 'shoots'" in message for message in messages) + assert any("no column 'width'" in message for message in messages) + assert any("'Axis' is not an entity set" in message for message in messages) + assert any("no data structure 'soil'" in message for message in messages) + + +def test_a_grid_and_segment_ends_are_needed(): + frame = generate.soil_and_roots() + layer = {"name": "axes", "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "Axis"}} + grid = {"name": "soil", "source": {"data_structure": "root_system", "kind": "grid"}} + report = views.check_views({"fspm_tf_views": 1, "configurations": {"c": {"layers": [layer, grid]}}}, frame) + assert len(report.errors) == 2 and "no segment ends" in report.errors[0].message + assert "has no grid" in report.errors[1].message + + +def test_selection_by_values_labels_and_comparisons(): + segments = generate.root_system().data_structures[0].entity_set("SubOrgan") + count = segments.count + assert views.selected(segments, {}).all() + np.testing.assert_array_equal(views.selected(segments, {"label": ["RootSegment"]}), + segments.columns["label"] == "RootSegment") + np.testing.assert_array_equal(views.selected(segments, {"label": "RootSegment", "length": ">=0.01"}), + (segments.columns["label"] == "RootSegment") & (segments.columns["length"] >= 0.01)) + np.testing.assert_array_equal(views.selected(segments, {"id": [10, 12]}), np.isin(segments.columns["id"], [10, 12])) + segments.columns["rank"] = np.ma.MaskedArray(np.arange(count), mask=[True] + [False] * (count - 1)) + assert not views.selected(segments, {"rank": "!=5"})[0] # a missing value never holds + assert views.selected(segments, {"rank": "!=5"}).sum() == count - 2 + + +def test_configurations_and_their_defaults(tmp_path): + document = roots_views() + assert views.configuration_names(document) == ["default", "plain"] + name, layers = views.configuration(document) + assert name == "default" and layers[0]["opacity"] == 1. and layers[0]["colormap"] == views.DEFAULTS["colormap"] + assert views.configuration(document, "plain")[1][0]["opacity"] == 0.3 + with pytest.raises(KeyError, match="nothing"): + views.configuration(document, "nothing") + path = str(tmp_path / views.VIEWS_FILE) + views.write_views(path, document) + assert views.read_views(path) == document + frame_path = str(tmp_path / "frame_000000.glb") + assert views.views_path(frame_path) == path and views.views_path(frame_path, "other.json") is None + with pytest.raises(ValueError, match="views"): + views.write_views(path, {"fspm_tf_views": 2, "configurations": {}}) + + +def test_display_settings_and_back(): + _, layers = views.configuration(roots_views()) + mesh, segments = (views.display_settings(layer) for layer in layers) + assert mesh["selector"] == "/Root/first_order" and mesh["tube"] is None and mesh["range"] is None + assert mesh["color_array"] == "SubOrgan.water_potential" and mesh["visible"] + assert segments["tube"] == {"radius_array": views.TUBE_RADIUS} and not segments["visible"] + assert views.display_settings(layers[1], {"SubOrgan.label_code"})["color_array"] == "SubOrgan.label_code" + assert views.block_selector("2 roots/p0") == "/Root/_2_roots_p0" + unchanged = views.layer_from_display(layers[0], {}) + assert unchanged == layers[0] + saved = views.layer_from_display(layers[1], {"visible": True, "opacity": 0.5, "representation": "Wireframe", + "color_array": "SubOrgan.label_code", "range": [0., 2.], + "log": True}) + assert saved["color_by"] == {"entity_set": "SubOrgan", "variable": "label"} + assert saved["range"] == [0., 2.] and saved["log"] and saved["visible"] and saved["representation"] == "Wireframe" + assert saved["tube"] == layers[1]["tube"] and saved["select"] == layers[1]["select"] + solid = views.layer_from_display(layers[0], {"color_array": None, "solid_color": [1., 0., 0.], "range": None}) + assert solid["color_by"] is None and solid["solid_color"] == [1., 0., 0.] and solid["range"] == "auto" + assert views.display_settings(solid)["color_array"] is None + document = views.save_configuration(roots_views(), "mine", [saved, solid]) + assert document["active"] == "mine" and views.check_views(document, generate.root_system()).ok + assert views.configuration(document)[1][0]["opacity"] == 0.5 + assert "mine" not in roots_views()["configurations"] # a copy + + +# ── The macros, on a stand-in for paraview.simple ──────────────────────── + +class _Item: + def __init__(self, simple, kind, name, **properties): + self.kind, self.name, self.simple = kind, name, simple + self.__dict__.update(properties) + simple.items[name] = self + + def UpdatePipeline(self, *arguments): + pass + + def UpdatePropertyInformation(self): + pass + + def GetDataInformation(self): + names = ["SubOrgan.water_potential", "SubOrgan.label_code"] + + class Arrays: + def GetNumberOfArrays(self): + return len(names) + + def GetArrayInformation(self, number): + return types.SimpleNamespace(GetName=lambda: names[number]) + + return types.SimpleNamespace(GetCellDataInformation=lambda: Arrays()) + + +class _Display: + def __init__(self): + self.Visibility, self.Opacity, self.Representation = 1, 1., "Surface" + self.ColorArrayName, self.DiffuseColor, self.AmbientColor = ["POINTS", ""], [1., 1., 1.], [1., 1., 1.] + + def SetRepresentationType(self, representation): + self.Representation = representation + + def RescaleTransferFunctionToDataRange(self, *arguments): + pass + + def SetScalarBarVisibility(self, *arguments): + pass + + +class _Colours: + def __init__(self): + self.RGBPoints, self.UseLogScale, self.AutomaticRescaleRangeMode = [0., 0, 0, 0, 1., 1, 1, 1], 0, "Never" + + def ApplyPreset(self, name, rescale): + self.preset = name + + def RescaleTransferFunction(self, low, high): + self.RGBPoints = [low, 0, 0, 0, high, 1, 1, 1] + + +def _stand_in(reader): + simple = types.ModuleType("paraview.simple") + simple.items, simple.displays, simple.colours = {}, {}, {} + simple.active = reader(simple) + simple.GetActiveSource = lambda: simple.active + simple.SetActiveSource = lambda item: None + simple.GetActiveViewOrCreate = lambda kind: "view" + simple.Render = lambda view: None + simple.FindSource = lambda name: simple.items.get(name) + simple.Delete = lambda item: simple.items.pop(item.name) + simple.ExtractBlock = lambda registrationName, Input, Selectors: _Item(simple, "block", registrationName, + Input=Input, Selectors=Selectors) + simple.Tube = lambda registrationName, Input: _Item(simple, "tube", registrationName, Input=Input) + + def show(item, view): + display = simple.displays.setdefault(item.name, _Display()) + display.Visibility = 1 + return display + + def hide(item, view): + simple.displays.setdefault(item.name, _Display()).Visibility = 0 + + def colour_by(display, value): + display.ColorArrayName = ["CELLS", value[1]] if value else ["POINTS", ""] + + simple.Show, simple.Hide, simple.ColorBy = show, hide, colour_by + simple.GetDisplayProperties = lambda item, view: simple.displays[item.name] + simple.GetColorTransferFunction = lambda name: simple.colours.setdefault(name, _Colours()) + package = types.ModuleType("paraview") + package.simple = simple + return package, simple + + +def _run_macro(name: str, simple_package, monkeypatch): + monkeypatch.setitem(sys.modules, "paraview", simple_package) + monkeypatch.setitem(sys.modules, "paraview.simple", simple_package.simple) + path = os.path.join(ROOT, "paraview", name) + with open(path, encoding="utf-8") as file: + exec(compile(file.read(), path, "exec"), {"__name__": "__main__", "__file__": path}) + + +def test_the_macros_apply_and_save_views(tmp_path, monkeypatch): + path = str(tmp_path / views.VIEWS_FILE) + views.write_views(path, roots_views()) + package, simple = _stand_in(lambda simple: _Item(simple, "reader", "scene.fspm.json", Configuration="", + FileName=str(tmp_path / "scene.fspm.json"), Views="")) + _run_macro("apply_fspm_tf_views.py", package, monkeypatch) + assert simple.items["first order"].kind == "block" and simple.items["roots"].kind == "tube" + assert simple.items["roots block"].Selectors == ["/Root/roots"] + assert simple.items["roots"].Scalars == ["POINTS", views.TUBE_RADIUS] + assert simple.displays["first order"].ColorArrayName == ["CELLS", "SubOrgan.water_potential"] + assert simple.displays["roots"].ColorArrayName == ["CELLS", "SubOrgan.label_code"] + assert simple.displays["roots"].Visibility == 0 and simple.displays["first order"].Visibility == 1 + _run_macro("apply_fspm_tf_views.py", package, monkeypatch) # applied again: replaced + assert sorted(simple.items) == ["first order", "roots", "roots block", "scene.fspm.json"] + + simple.displays["first order"].Opacity = 0.25 # changed in ParaView, then saved + simple.displays["roots"].Visibility = 1 + simple.colours["SubOrgan.label_code"].RescaleTransferFunction(0., 3.) + simple.colours["SubOrgan.label_code"].AutomaticRescaleRangeMode = "Never" + _run_macro("save_fspm_tf_views.py", package, monkeypatch) + saved = views.read_views(path) + first, roots = saved["configurations"]["default"]["layers"] + assert first["opacity"] == 0.25 and roots["visible"] and roots["range"] == [0., 3.] + assert roots["color_by"] == {"entity_set": "SubOrgan", "variable": "label"} and roots["tube"]["scale"] == 2 + assert saved["configurations"]["plain"] == roots_views()["configurations"]["plain"] diff --git a/doc/api.md b/doc/api.md index 47ecd88..414f851 100644 --- a/doc/api.md +++ b/doc/api.md @@ -31,6 +31,14 @@ the vocabulary and units. :members: add_display, remove_display ``` +## Views + +```{eval-rst} +.. automodule:: openalea.fspm_tf.views + :members: read_views, write_views, views_path, check_views, configuration, configuration_names, selected, + display_settings, layer_from_display, save_configuration, block_selector +``` + ## Vocabulary and units ```{eval-rst} diff --git a/doc/reading_the_specification.md b/doc/reading_the_specification.md index c9ed197..9277534 100644 --- a/doc/reading_the_specification.md +++ b/doc/reading_the_specification.md @@ -17,6 +17,7 @@ Everything the platforms agree on is in the repository's `spec/` folder, togethe | `spec/specification.md` | the rules, in words (MUST, SHOULD, MAY as in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)) | by the validator, `fspm-tf validate` | | `spec/schema/fspm_scene.schema.json` | the shape of the `FSPM_scene` JSON (its fields and their types) | by any JSON Schema validator | | `spec/schema/index.schema.json` | the shape of a sequence's index, `scene.fspm.json` | by any JSON Schema validator | +| `spec/schema/views.schema.json` | the shape of a sequence's optional views, `views.json` (how to look at it) | by any JSON Schema validator; names against a frame by `openalea.fspm_tf.views.check_views` | | `spec/vocabulary/standard_names.csv` | the shared quantities: canonical unit, kind, definition, and each platform's names for them | by the validator (units, kinds) | | `spec/vocabulary/labels.csv` | the entity labels, with their scale and Plant Ontology term | by the validator (warnings) | | `spec/vocabulary/enums.json` | the allowed values: edge types, kinds, roles, base scales, formats… | by the schemas and the validator | diff --git a/doc/using.md b/doc/using.md index 8fd3bf3..0ef11ea 100644 --- a/doc/using.md +++ b/doc/using.md @@ -55,6 +55,49 @@ roots = fspmTFreader(FileName="outputs/frames/scene.fspm.json") roots.UpdatePipeline(roots.TimestepValues[-1]) # the last frame ``` +### Views: layers ready to look at + +A sequence may come with a `views.json` next to its `scene.fspm.json` (metafspm writes one with +`SceneRecorder(frame_views="default")`). It names **configurations**, each a list of **layers**: a part of a data +structure (a mesh, root segments, a grid), the entities it keeps (e.g. the root segments, or the cells where the +soil is wet), the variable it is coloured by, its colour map and range, its opacity, whether it is shown at first, +and tubes for segments. The specification describes the file (§8.1). + +With a views file, the reader gives **one block per layer** of the chosen configuration instead of one per data +structure: each block holds only the layer's entities, already coloured by its variable. Hidden layers are there too, +so showing one is instant, and a layer is empty in the frames that lack what it shows (a shoot not yet drawn). Two reader options: + +- **Views:** the views file (empty: the `views.json` next to the opened file; without one, the blocks are as above). +- **Configuration:** a drop-down of the file's configurations (empty: the file's active one). + +Two macros, shipped next to the plugin in `paraview/`, turn the layers into pipeline items and back: + +1. Load them once: **Macros → Import new macro…**, choose `paraview/apply_fspm_tf_views.py`, then again for + `paraview/save_fspm_tf_views.py`. They appear in the **Macros** menu and toolbar, and stay there. +2. Open a sequence with the fspm-TF reader, select the reader in the Pipeline Browser, and run + **apply_fspm_tf_views**: each layer becomes its own item, named by the layer (an *ExtractBlock*, and a *Tube* for + segments), shown as the layer says. Toggle the eyes to show or hide layers; play the time steps as usual. +3. Change colours, ranges, opacities, then select the reader again and run **save_fspm_tf_views**: the items' display + options are written back into `views.json`, over the configuration. To keep the old one, set `CONFIGURATION` at the + top of the macro (**Macros → Edit…**) to a new name: the views are saved as that configuration, made the active + one. + +The macros import `openalea.fspm_tf`, which the plugin puts on ParaView's path: load the plugin before running them. +A colour map's preset name is not read back from ParaView, so a saved layer keeps its `colormap`; layers coloured by +the same variable share one colour map in ParaView. + +In a script, the same views without ParaView: `frame_to_vtk(frame, views="views.json", configuration="default")` from +`paraview/fspm_tf_reader.py`, and `openalea.fspm_tf.views` to read, check and write views: + +```python +from openalea import fspm_tf +from openalea.fspm_tf import views + +document = views.read_views("outputs/frames/views.json") +report = views.check_views(document, fspm_tf.read("outputs/frames/frame_000000.glb")) # every name in the frame +print(report) +``` + ## In other viewers Any glTF viewer (Blender, a web viewer, ParaView without the plugin) shows the geometry, and ignores the tables. diff --git a/paraview/apply_fspm_tf_views.py b/paraview/apply_fspm_tf_views.py new file mode 100644 index 0000000..5a2cc91 --- /dev/null +++ b/paraview/apply_fspm_tf_views.py @@ -0,0 +1,99 @@ +""" +ParaView macro "Apply fspm-TF views": each layer of the selected fspm-TF reader's configuration (its "Views" file and +"Configuration", see fspm_tf_reader.py) becomes its own pipeline item, named by the layer: an ExtractBlock of the +layer's block, a Tube on "tube_radius" for a segments layer, shown with its colour variable, colour map, range, log +scale, solid colour, opacity, representation and visibility. Applied again, it replaces the items it made. + + Macros > Import new macro..., choose this file; select the fspm-TF reader in the Pipeline Browser, then + Macros > apply_fspm_tf_views. + +What each layer gives is openalea.fspm_tf.views.display_settings(layer), tested without ParaView; this file only +hands it to paraview.simple. +""" +from paraview import simple + + +def _views(): + try: + from openalea.fspm_tf import views + except ImportError: + raise RuntimeError("openalea.fspm_tf is not on ParaView's path: load the fspm-TF reader plugin first (it adds " + "its fspm-TF folder), or install openalea.fspm_tf in ParaView's Python") from None + return views + + +def reader_views(source): + """(views module, the views file's path, its document, the configuration's name, its layers) of the fspm-TF + reader *source*.""" + views = _views() + if source is None or not hasattr(source, "Configuration"): + raise RuntimeError("select an fspm-TF reader in the Pipeline Browser first") + file_name = source.FileName if isinstance(source.FileName, str) else source.FileName[0] + path = views.views_path(file_name, source.Views) + if path is None: + raise RuntimeError(f"no views file for {file_name} (a views.json next to it, or the reader's Views)") + document = views.read_views(path) + name, layers = views.configuration(document, source.Configuration or None) + return views, path, document, name, layers + + +def _cell_arrays(item) -> set: + information = item.GetDataInformation().GetCellDataInformation() + return {information.GetArrayInformation(number).GetName() for number in range(information.GetNumberOfArrays())} + + +def _remove(name: str) -> None: + for item_name in (name, f"{name} block"): # the tube first, then the block it reads + item = simple.FindSource(item_name) + if item is not None: + simple.Delete(item) + + +def show_layer(source, layer: dict, view, views) -> None: + _remove(layer["name"]) + settings = views.display_settings(layer) + tube = settings["tube"] + block = simple.ExtractBlock(registrationName=f"{layer['name']} block" if tube else layer["name"], Input=source, + Selectors=[settings["selector"]]) + item = block + if tube: + item = simple.Tube(registrationName=layer["name"], Input=block) + item.Scalars = ["POINTS", tube["radius_array"]] + item.VaryRadius = "By Absolute Scalar" + item.UpdatePipeline() + settings = views.display_settings(layer, _cell_arrays(item)) + display = simple.Show(item, view) + display.SetRepresentationType(settings["representation"]) + if settings["color_array"]: + simple.ColorBy(display, ("CELLS", settings["color_array"])) + colours = simple.GetColorTransferFunction(settings["color_array"]) + colours.ApplyPreset(settings["colormap"], True) + if settings["range"] is None: + colours.AutomaticRescaleRangeMode = "Grow and update on 'Apply'" + display.RescaleTransferFunctionToDataRange(True, False) + else: + colours.AutomaticRescaleRangeMode = "Never" + colours.RescaleTransferFunction(*settings["range"]) + colours.UseLogScale = int(settings["log"]) + display.SetScalarBarVisibility(view, settings["visible"]) + else: + simple.ColorBy(display, None) + display.AmbientColor = display.DiffuseColor = settings["solid_color"] + display.Opacity = settings["opacity"] + if not settings["visible"]: + simple.Hide(item, view) + + +def apply_views(source=None, view=None) -> None: + source = source or simple.GetActiveSource() + views, _, _, _, layers = reader_views(source) + view = view or simple.GetActiveViewOrCreate("RenderView") + source.UpdatePipeline() + simple.Hide(source, view) + for layer in layers: + show_layer(source, layer, view, views) + simple.SetActiveSource(source) + simple.Render(view) + + +apply_views() diff --git a/paraview/check_in_paraview.py b/paraview/check_in_paraview.py index 74731c2..e6d09b5 100644 --- a/paraview/check_in_paraview.py +++ b/paraview/check_in_paraview.py @@ -1,6 +1,7 @@ """ The reader inside ParaView: loads the plugin, opens conformance files and a sequence, checks their blocks, arrays and -time steps, and renders a picture of each. +time steps, and renders a picture of each; then a frame with views (one block per layer, the configurations' +drop-down) and the macros applying and saving them. pvpython paraview/check_in_paraview.py conformance [output folder] """ @@ -85,4 +86,38 @@ def picture(source, name, colour_by, side=False): names.add(point_data.GetArrayInformation(number).GetName()) print("glTF reader", type(plain).__name__, "point data", sorted(names)) assert any("WATER_POTENTIAL" in name for name in names), sorted(names) + +# Views: a views.json next to the frame gives one block per layer; the macros make and save the layers' items +import runpy # noqa: E402 +import shutil # noqa: E402 + +from openalea.fspm_tf import views # noqa: E402 + +with tempfile.TemporaryDirectory() as directory: + shutil.copy(os.path.join(conformance, "root_system.glb"), directory) + layers = [{"name": "roots", + "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "tube": {"radius": "radius"}}, + {"name": "mesh", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "color_by": None, "opacity": 0.3, "visible": False}] + path = os.path.join(directory, views.VIEWS_FILE) + views.write_views(path, {"fspm_tf_views": 1, "active": "default", + "configurations": {"default": {"layers": layers}, "mesh only": {"layers": layers[1:]}}}) + viewed = fspmTFreader(FileName=os.path.join(directory, "root_system.glb")) # noqa: F821 + viewed.UpdatePropertyInformation() + assert list(viewed.ConfigurationNames) == ["default", "mesh only"], list(viewed.ConfigurationNames) + viewed.UpdatePipeline() + assert viewed.GetDataInformation().DataInformation.GetNumberOfLeaves() == 2 + simple.SetActiveSource(viewed) + runpy.run_path(os.path.join(HERE, "apply_fspm_tf_views.py")) + tube = simple.FindSource("roots") + assert tube is not None and simple.FindSource("mesh") is not None + simple.GetDisplayProperties(tube).Opacity = 0.5 + simple.SetActiveSource(viewed) + runpy.run_path(os.path.join(HERE, "save_fspm_tf_views.py")) + assert views.read_views(path)["configurations"]["default"]["layers"][0]["opacity"] == 0.5 + picture(tube, "root_system_views", "SubOrgan.water_potential", side=True) + viewed.Configuration = "mesh only" + viewed.UpdatePipeline() + assert viewed.GetDataInformation().DataInformation.GetNumberOfLeaves() == 1 print("fspm-TF reader: ok") diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py index 5343641..4d1ed60 100644 --- a/paraview/fspm_tf_reader.py +++ b/paraview/fspm_tf_reader.py @@ -24,6 +24,14 @@ metafspm's MPG links) are left out unless "Show Settings" is ticked. Data structures are placed by their origin. +With views (specification §8.1: a views.json next to the file, or the one "Views" names), the output is instead one +block per layer of the chosen "Configuration" (the file's active one by default), named by the layer: only its +selected entities (a mesh's triangles whose rows at the select entity set match, the segments whose rows match, the +grid's cells, the others hidden), its colour variable as the active scalars, a segments layer's tube radius as point +data "tube_radius"; hidden layers are there too, and a layer whose names a frame lacks is an empty block. The macros +apply_fspm_tf_views.py and save_fspm_tf_views.py (next to this file; Macros > Import new macro...) make each layer its +own pipeline item, and write the items' display options back into the views file. + frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests. """ import os @@ -32,8 +40,8 @@ import numpy as np from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray from vtkmodules.vtkCommonCore import vtkPoints, vtkStringArray -from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkImageData, vtkMultiBlockDataSet, - vtkPolyData, vtkUnstructuredGrid) +from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkDataSetAttributes, vtkImageData, + vtkMultiBlockDataSet, vtkPolyData, vtkUnstructuredGrid) from vtkmodules.vtkCommonExecutionModel import vtkStreamingDemandDrivenPipeline @@ -48,6 +56,12 @@ def _fspm_tf(): return fspm_tf +def _views(): + _fspm_tf() + from openalea.fspm_tf import views + return views + + SEGMENT_ENDS = (("proximal_end_x", "x1"), ("proximal_end_y", "y1"), ("proximal_end_z", "z1"), ("distal_end_x", "x2"), ("distal_end_y", "y2"), ("distal_end_z", "z2")) @@ -84,16 +98,17 @@ def _shown(entity_set, name: str, settings: bool) -> bool: return name not in LINKS and (declared is None or declared.role != "setting") -def _add_columns(data, entity_set, rows: np.ndarray, field_data, settings: bool = False) -> None: - """Cell data of *entity_set*'s columns, one value per cell from *rows* (-1: no entity, NaN).""" +def _add_columns(data, entity_set, rows: np.ndarray, field_data, settings: bool = False, forced=()) -> None: + """Cell data of *entity_set*'s columns, one value per cell from *rows* (-1: no entity, NaN); the columns + *forced* (a layer's colour variable) whatever their role.""" valid = rows >= 0 taken = np.where(valid, rows, 0) for name, column in entity_set.columns.items(): - if not _shown(entity_set, name, settings): + if not _shown(entity_set, name, settings) and name not in forced: continue label = f"{entity_set.name}.{name}" if column.dtype == object: - if name not in ("label", "relation") and len(set(column.tolist())) > 64: + if name not in ("label", "relation") and name not in forced and len(set(column.tolist())) > 64: continue # free text: not a colouring codes, names = _codes(column) values = np.where(valid, codes[taken], -1) @@ -119,13 +134,16 @@ def _transform(points: np.ndarray, origin: np.ndarray) -> np.ndarray: return points @ origin[:3, :3].T + origin[:3, 3] -def _mesh(mesh, data_structure, settings: bool = False): +def _mesh(mesh, data_structure, settings: bool = False, kept: np.ndarray = None, forced: dict = None): + """The mesh's triangles (those *kept*, a mask, when given); *forced*: {entity set: columns}, see _add_columns.""" polydata = vtkPolyData() points = vtkPoints() points.SetData(numpy_to_vtk(np.ascontiguousarray(_transform(mesh.positions, data_structure.origin)), deep=True)) polydata.SetPoints(points) corners = mesh.indices if mesh.indices is not None else np.arange(len(mesh.positions)) triangles = corners.reshape(-1, 3) + if kept is not None: + triangles = triangles[kept] offsets = np.arange(0, 3 * len(triangles) + 1, 3, dtype=np.int64) cells = vtkCellArray() cells.SetData(numpy_to_vtkIdTypeArray(offsets, deep=True), @@ -133,7 +151,7 @@ def _mesh(mesh, data_structure, settings: bool = False): polydata.SetPolys(cells) for set_name, rows in mesh.feature_rows.items(): _add_columns(polydata.GetCellData(), data_structure.entity_set(set_name), rows[triangles[:, 0]], - polydata.GetFieldData(), settings) + polydata.GetFieldData(), settings, (forced or {}).get(set_name, ())) for name, values in mesh.display.items(): polydata.GetPointData().AddArray(_array(f"display.{name}", np.asarray(values, dtype=np.float32))) return polydata @@ -151,9 +169,12 @@ def _segment_columns(entity_set) -> list: return found -def _segments(entity_set, columns: list, data_structure, settings: bool = False): - ends = np.stack([entity_set.columns[name].astype(np.float64) for name in columns], axis=1) - count = entity_set.count +def _segments(entity_set, columns: list, data_structure, settings: bool = False, rows: np.ndarray = None, + forced=(), tube_radius: np.ndarray = None): + """One line per entity (per row of *rows* when given); *tube_radius*: a radius per entity, as point data.""" + rows = np.arange(entity_set.count) if rows is None else rows + ends = np.stack([entity_set.columns[name].astype(np.float64)[rows] for name in columns], axis=1) + count = len(rows) points_array = _transform(np.concatenate([ends[:, :3], ends[:, 3:]]), data_structure.origin) polydata = vtkPolyData() points = vtkPoints() @@ -164,11 +185,15 @@ def _segments(entity_set, columns: list, data_structure, settings: bool = False) cells.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 2 * count + 1, 2, dtype=np.int64), deep=True), numpy_to_vtkIdTypeArray(connectivity, deep=True)) polydata.SetLines(cells) - _add_columns(polydata.GetCellData(), entity_set, np.arange(count), polydata.GetFieldData(), settings) + _add_columns(polydata.GetCellData(), entity_set, rows, polydata.GetFieldData(), settings, forced) + if tube_radius is not None: + radius = np.asarray(tube_radius, dtype=np.float64)[rows] + polydata.GetPointData().AddArray(_array(_views().TUBE_RADIUS, np.concatenate([radius, radius]))) return polydata -def _regular_grid(data_structure, settings: bool = False): +def _regular_grid(data_structure, settings: bool = False, kept: np.ndarray = None, forced=()): + """The grid as image data; the cells not *kept* (a mask over the cells' rows) hidden.""" grid = data_structure.grid cells = data_structure.entity_set(grid.cells) dimensions = len(grid.shape) @@ -182,15 +207,20 @@ def _regular_grid(data_structure, settings: bool = False): if not grid.z_up and dimensions == 3: image.SetDirectionMatrix(1., 0., 0., 0., 1., 0., 0., 0., -1.) # z pointing down order = np.arange(int(np.prod(shape))).reshape(shape).ravel(order="F") # VTK: the first axis varies fastest - _add_columns(image.GetCellData(), cells, order, image.GetFieldData(), settings) + _add_columns(image.GetCellData(), cells, order, image.GetFieldData(), settings, forced) + if kept is not None: + ghosts = np.where(kept[order], 0, vtkDataSetAttributes.HIDDENCELL).astype(np.uint8) + image.GetCellData().AddArray(_array(vtkDataSetAttributes.GhostArrayName(), ghosts)) return image -def _adaptive_grid(data_structure, settings: bool = False): +def _adaptive_grid(data_structure, settings: bool = False, kept: np.ndarray = None, forced=()): + """The grid's leaf cells as hexahedra (those *kept*, a mask, when given).""" grid = data_structure.grid cells = data_structure.entity_set(grid.cells) - level = cells.columns["level"].astype(np.int64) - index = cells.columns["index"].astype(np.float64) + rows = np.arange(cells.count) if kept is None else np.flatnonzero(kept) + level = cells.columns["level"].astype(np.int64)[rows] + index = cells.columns["index"].astype(np.float64)[rows] size = np.asarray(grid.spacing, dtype=np.float64)[None, :] / (2. ** level)[:, None] lower = np.asarray(grid.origin, dtype=np.float64)[None, :] + index * size upper = lower + size @@ -206,12 +236,12 @@ def _adaptive_grid(data_structure, settings: bool = False): points = vtkPoints() points.SetData(numpy_to_vtk(np.ascontiguousarray(points_array), deep=True)) unstructured.SetPoints(points) - count = cells.count + count = len(rows) cell_array = vtkCellArray() cell_array.SetData(numpy_to_vtkIdTypeArray(np.arange(0, 8 * count + 1, 8, dtype=np.int64), deep=True), numpy_to_vtkIdTypeArray(np.arange(8 * count, dtype=np.int64), deep=True)) unstructured.SetCells(VTK_HEXAHEDRON, cell_array) - _add_columns(unstructured.GetCellData(), cells, np.arange(count), unstructured.GetFieldData(), settings) + _add_columns(unstructured.GetCellData(), cells, rows, unstructured.GetFieldData(), settings, forced) return unstructured @@ -221,10 +251,64 @@ def _named(blocks: vtkMultiBlockDataSet, data_set, name: str) -> None: blocks.GetMetaData(number).Set(vtkMultiBlockDataSet.NAME(), name) -def frame_to_vtk(frame, settings: bool = False) -> vtkMultiBlockDataSet: +def _layer(frame, layer: dict, settings: bool = False): + """The data set of a layer (views.configuration's, its defaults filled): its selected entities only, its colour + variable as the active scalars; empty when the frame lacks a name it gives (e.g. a mesh drawn from a later frame + on), so that every frame of a sequence has the same blocks.""" + views = _views() + report = views.check_views({"fspm_tf_views": views.FORMAT, "configurations": {"layer": {"layers": [layer]}}}, frame) + if not report.ok: + return vtkPolyData() + source = layer["source"] + data_structure = frame.data_structure(source["data_structure"]) + select = layer.get("select") + chosen = (data_structure.entity_set(select["entity_set"]), select["where"]) if select else None + colour = layer.get("color_by") + forced = {colour["entity_set"]: (colour["variable"],)} if colour else {} + if source["kind"] == "mesh": + mesh = next(mesh for mesh in data_structure.meshes if mesh.name == source["mesh"]) + kept = None + if chosen is not None: + corners = mesh.indices if mesh.indices is not None else np.arange(len(mesh.positions)) + rows = mesh.feature_rows[chosen[0].name][corners.reshape(-1, 3)[:, 0]] + matching = views.selected(*chosen) + kept = (rows >= 0) & matching[np.where(rows >= 0, rows, 0)] + data_set = _mesh(mesh, data_structure, settings, kept, forced) + elif source["kind"] == "segments": + entity_set = data_structure.entity_set(source["entity_set"]) + rows = np.flatnonzero(views.selected(*chosen)) if chosen is not None else None + tube = layer.get("tube") + radius = entity_set.columns[tube["radius"]].astype(np.float64) * float(tube.get("scale", 1.)) if tube else None + data_set = _segments(entity_set, _segment_columns(entity_set), data_structure, settings, rows, + forced.get(entity_set.name, ()), radius) + else: + kept = views.selected(*chosen) if chosen is not None else None + build = _regular_grid if data_structure.grid.kind == "regular" else _adaptive_grid + data_set = build(data_structure, settings, kept, forced.get(data_structure.grid.cells, ())) + array = views.colour_array(layer, {data_set.GetCellData().GetArrayName(number) + for number in range(data_set.GetCellData().GetNumberOfArrays())}) + if array is not None: + data_set.GetCellData().SetActiveScalars(array) + return data_set + + +def frame_to_vtk(frame, settings: bool = False, views=None, configuration: str = None) -> vtkMultiBlockDataSet: """The frame as a multiblock data set: one block per data structure, holding its meshes, its entity sets with segment ends as lines, and its grid. *settings*: also the graph's links (complex, parent, edge_type) and the - columns of role "setting".""" + columns of role "setting". + + *views* (a views document, or the path of a views.json): instead, one block per layer of the *configuration* + (None: the active one), named by the layer, hidden layers included (see _layer).""" + if views is not None: + views_module = _views() + document = views_module.read_views(views) if isinstance(views, (str, os.PathLike)) else views + report = views_module.check_views(document) + if not report.ok: + raise ValueError(str(report)) + output = vtkMultiBlockDataSet() + for layer in views_module.configuration(document, configuration)[1]: + _named(output, _layer(frame, layer, settings), layer["name"]) + return output output = vtkMultiBlockDataSet() for data_structure in frame.data_structures: blocks = vtkMultiBlockDataSet() @@ -276,12 +360,18 @@ def frame_at(self, time: float = None): VTKPythonAlgorithmBase = None if VTKPythonAlgorithmBase is not None: + try: + _fspm_tf() # on ParaView's path now: the views macros import it too + except ImportError: + pass + @smproxy.reader(name="fspmTFReader", label="fspm-TF reader", extensions="glb json", file_description="fspm-TF frames (.glb) and sequences (scene.fspm.json)") class FspmTFReader(VTKPythonAlgorithmBase): def __init__(self): VTKPythonAlgorithmBase.__init__(self, nInputPorts=0, nOutputPorts=1, outputType="vtkMultiBlockDataSet") self._file_name, self._source, self._settings = None, None, False + self._views, self._configuration = "", "" @smproperty.stringvector(name="FileName") @smdomain.filelist() @@ -302,6 +392,35 @@ def SetShowSettings(self, value): self._settings = bool(value) self.Modified() + @smproperty.stringvector(name="Views", default_values="") + @smdomain.filelist() + @smhint.filechooser(extensions="json", file_description="fspm-TF views (views.json)") + def SetViews(self, name): + """The views file (empty: the views.json next to the file, when there is one; none: today's blocks).""" + if (name or "") != self._views: + self._views = name or "" + self.Modified() + + def _views_path(self): + return _views().views_path(self._file_name, self._views) if self._file_name else None + + @smproperty.stringvector(name="ConfigurationNames", information_only="1") + def GetConfigurationNames(self): + path = self._views_path() + return _views().configuration_names(_views().read_views(path)) if path else [] + + @smproperty.stringvector(name="Configuration", number_of_elements="1", default_values="") + @smdomain.xml(""" + + + + """) + def SetConfiguration(self, name): + """The configuration shown (empty: the views' active one).""" + if (name or "") != self._configuration: + self._configuration = name or "" + self.Modified() + def _opened(self): if self._source is None and self._file_name: self._source = Source(self._file_name) @@ -332,7 +451,9 @@ def RequestData(self, request, in_info, out_info): else None frame = self._opened().frame_at(time) output = vtkMultiBlockDataSet.GetData(out_info, 0) - output.ShallowCopy(frame_to_vtk(frame, settings=self._settings)) + path = self._views_path() + output.ShallowCopy(frame_to_vtk(frame, settings=self._settings, views=path, + configuration=self._configuration or None)) if time is not None: output.GetInformation().Set(output.DATA_TIME_STEP(), frame.time) return 1 diff --git a/paraview/save_fspm_tf_views.py b/paraview/save_fspm_tf_views.py new file mode 100644 index 0000000..07d9967 --- /dev/null +++ b/paraview/save_fspm_tf_views.py @@ -0,0 +1,68 @@ +""" +ParaView macro "Save fspm-TF views": the display options of the items "Apply fspm-TF views" made (visibility, +opacity, representation, colour variable or solid colour, range, log scale) written back into the reader's views +file, over its configuration, or into a new configuration named by CONFIGURATION below (Macros > Edit to set it), +made the active one. A layer without its item keeps what it had. + + Macros > Import new macro..., choose this file; select the fspm-TF reader in the Pipeline Browser, then + Macros > save_fspm_tf_views. + +The colour map's preset is not read back (ParaView does not keep a preset's name): a layer keeps its colormap. +What a display gives back is openalea.fspm_tf.views.layer_from_display(layer, display), tested without ParaView. +""" +from paraview import simple + +CONFIGURATION = None # None: overwrite the reader's configuration; a name: save as that configuration + + +def _views(): + try: + from openalea.fspm_tf import views + except ImportError: + raise RuntimeError("openalea.fspm_tf is not on ParaView's path: load the fspm-TF reader plugin first (it adds " + "its fspm-TF folder), or install openalea.fspm_tf in ParaView's Python") from None + return views + + +def display_of(item, view) -> dict: + """The display options of a pipeline item, as layer_from_display takes them.""" + display = simple.GetDisplayProperties(item, view) + array = display.ColorArrayName[1] if display.ColorArrayName and display.ColorArrayName[1] else None + found = {"visible": bool(display.Visibility), "opacity": float(display.Opacity), + "representation": str(display.Representation), "color_array": array, + "solid_color": [float(value) for value in display.DiffuseColor]} + if array: + colours = simple.GetColorTransferFunction(array) + points = list(colours.RGBPoints) + found["range"] = None if colours.AutomaticRescaleRangeMode != "Never" else [points[0], points[-4]] + found["log"] = bool(colours.UseLogScale) + return found + + +def save_views(source=None, view=None, configuration: str = None) -> str: + """Write the items' display options into the views file of *source* (default: the active one); its path.""" + views = _views() + source = source or simple.GetActiveSource() + if source is None or not hasattr(source, "Configuration"): + raise RuntimeError("select an fspm-TF reader in the Pipeline Browser first") + file_name = source.FileName if isinstance(source.FileName, str) else source.FileName[0] + path = views.views_path(file_name, source.Views) + if path is None: + raise RuntimeError(f"no views file for {file_name} (a views.json next to it, or the reader's Views)") + document = views.read_views(path) + name, _ = views.configuration(document, source.Configuration or None) + view = view or simple.GetActiveViewOrCreate("RenderView") + layers = [] + for layer in document["configurations"][name]["layers"]: + item = simple.FindSource(layer["name"]) + layers.append(views.layer_from_display(layer, display_of(item, view)) if item is not None else layer) + target = configuration or name + views.write_views(path, views.save_configuration(document, target, layers)) + if target != name: + source.UpdatePropertyInformation() # the new configuration in the drop-down + source.Configuration = target + print(f"fspm-TF views: configuration '{target}' saved in {path}") + return path + + +save_views(configuration=CONFIGURATION) diff --git a/spec/schema/views.schema.json b/spec/schema/views.schema.json new file mode 100644 index 0000000..e5dfe64 --- /dev/null +++ b/spec/schema/views.schema.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/openalea/fspm-TF/schema/0.3/views.schema.json", + "title": "fspm-TF frame views", + "description": "views.json, an optional companion of a sequence (next to its scene.fspm.json) or of a frame: named configurations of layers, each layer a part of a data structure (a mesh, segments or a grid), the entities it selects and how it is drawn (fspm-TF 0.3, §8.1).", + "type": "object", + "required": ["fspm_tf_views", "configurations"], + "properties": { + "fspm_tf_views": {"const": 1}, + "active": {"type": "string", "minLength": 1}, + "configurations": {"type": "object", "minProperties": 1, + "additionalProperties": {"$ref": "#/$defs/configuration"}}, + "extras": {} + }, + "additionalProperties": false, + "$defs": { + "configuration": { + "type": "object", "required": ["layers"], + "properties": {"layers": {"type": "array", "items": {"$ref": "#/$defs/layer"}}, "extras": {}}, + "additionalProperties": false}, + "layer": { + "type": "object", "required": ["name", "source"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "source": {"$ref": "#/$defs/source"}, + "select": {"type": "object", "required": ["entity_set", "where"], + "properties": {"entity_set": {"type": "string", "minLength": 1}, + "where": {"type": "object", "additionalProperties": {"$ref": "#/$defs/condition"}}}, + "additionalProperties": false}, + "color_by": {"oneOf": [{"type": "null"}, + {"type": "object", "required": ["entity_set", "variable"], + "properties": {"entity_set": {"type": "string", "minLength": 1}, + "variable": {"type": "string", "minLength": 1}}, + "additionalProperties": false}]}, + "solid_color": {"type": "array", "items": {"type": "number", "minimum": 0, "maximum": 1}, + "minItems": 3, "maxItems": 3}, + "colormap": {"type": "string", "minLength": 1}, + "range": {"oneOf": [{"const": "auto"}, + {"type": "array", "items": {"type": "number"}, "minItems": 2, "maxItems": 2}]}, + "log": {"type": "boolean"}, + "opacity": {"type": "number", "minimum": 0, "maximum": 1}, + "visible": {"type": "boolean"}, + "representation": {"enum": ["Surface", "Surface With Edges", "Wireframe", "Points", "Outline"]}, + "tube": {"type": "object", "required": ["radius"], + "properties": {"radius": {"type": "string", "minLength": 1}, + "scale": {"type": "number", "exclusiveMinimum": 0}}, + "additionalProperties": false}, + "extras": {} + }, + "additionalProperties": false}, + "source": { + "type": "object", "required": ["data_structure", "kind"], + "properties": {"data_structure": {"type": "string", "minLength": 1}, + "kind": {"enum": ["mesh", "segments", "grid"]}, + "mesh": {"type": "string", "minLength": 1}, + "entity_set": {"type": "string", "minLength": 1}}, + "additionalProperties": false, + "allOf": [ + {"if": {"properties": {"kind": {"const": "mesh"}}}, "then": {"required": ["mesh"]}}, + {"if": {"properties": {"kind": {"const": "segments"}}}, "then": {"required": ["entity_set"]}}]}, + "condition": { + "description": "A value, a list of values (labels and other text by name), or a comparison such as \">=0\" or \"!=2\".", + "oneOf": [{"type": ["number", "boolean"]}, + {"type": "string", "minLength": 1}, + {"type": "array", "items": {"type": ["string", "number", "boolean"]}}]} + } +} diff --git a/spec/specification.md b/spec/specification.md index f387ae6..4f54052 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -12,6 +12,7 @@ The language-neutral files of this version are part of the specification: |---|---| | `spec/schema/fspm_scene.schema.json` | JSON Schema of the `FSPM_scene` extension (§3) | | `spec/schema/index.schema.json` | JSON Schema of a sequence's index (§8) | +| `spec/schema/views.schema.json` | JSON Schema of a sequence's optional views, `views.json` (§8.1) | | `spec/vocabulary/standard_names.csv` | the standard names, their canonical units and kinds, and each platform's names for them (§6) | | `spec/vocabulary/labels.csv` | entity labels, their scales and Plant Ontology terms (§6) | | `spec/vocabulary/enums.json` | the values of `edge_type`, `kind`, `role`, data structure and grid kinds; the base scales | @@ -314,6 +315,39 @@ A sequence is a folder holding one GLB per frame and an index, `scene.fspm.json` NOT size arrays by the largest id. - **Packing.** Packing a sequence into one GLB is not part of 0.3. +### 8.1 Views + +A sequence (or a frame) MAY come with a companion `views.json`, next to its `scene.fspm.json` (or its GLB), following +`views.schema.json`. It says how to look at the frames, not what they hold: readers MAY ignore it, and a sequence is +valid without it. It is platform-independent; any viewer may apply it. + +```text +{"fspm_tf_views": 1, "active": "default", + "configurations": {"default": {"layers": [ + {"name": "roots", "source": {"data_structure": "plants", "kind": "segments", "entity_set": "SubOrgan"}, + "select": {"entity_set": "SubOrgan", "where": {"label": ["RootSegment"], "length": ">0"}}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "colormap": "Viridis (matplotlib)", + "range": "auto", "log": false, "opacity": 1, "visible": true, "representation": "Surface", + "tube": {"radius": "radius", "scale": 1}}]}}} +``` + +- **Configurations.** Named lists of layers; `active` names the one shown first (else the first). +- **A layer** draws one part of a data structure: `source.kind` `mesh` (with the mesh's `name`), `segments` (an entity + set with segment ends, `entity_set`) or `grid` (the grid's cells). Its name is unique in its configuration. +- **Selection.** `select` keeps the entities whose row in `select.entity_set` holds every condition of `where`: a + value, a list of values (labels and other text by name), or a comparison (`">=0"`, `"!=2"`); a missing value never + holds. For a mesh, the entity set is any the mesh represents (a triangle is kept when its feature row in that set + is); for segments and grids, the source's own set. +- **Drawing.** `color_by` names a variable of an entity set the layer's entities are in (as `select`), or is null + (then `solid_color`, RGB in [0, 1]); `colormap` is a colour map's name (ParaView's preset names), `range` `"auto"` + or `[min, max]`, `log` a log scale; `opacity`, `visible` (when opened), `representation`; `tube` (segments only) + draws them as tubes of radius the `radius` column times `scale`. +- **Defaults.** `colormap` `"Viridis (matplotlib)"`, `range` `"auto"`, `log` false, `opacity` 1, `visible` true, + `representation` `"Surface"`, `solid_color` white, a tube's `scale` 1. +- Every name a layer gives (data structure, mesh, entity sets, columns) SHOULD be in the frames it goes with; + the Python reference implementation checks them (`openalea.fspm_tf.views.check_views(views, frame)`). In a frame + that lacks one (e.g. a mesh drawn from a later frame on), a viewer draws the layer empty. + ## 9. Conformance `conformance/` holds reference scenes, each a GLB with its expected values in `*.expected.json`, and a growing @@ -503,6 +537,8 @@ Both transports carry the same messages. All additions; a 0.2 file is a valid 0.3 file. +- §8.1: views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): configurations of + layers, each a selection of a mesh's, segments' or grid's entities and how they are drawn. - §11: coupling sessions: partial frames (`FSPM_scene.update`), the messages (`spec/schema/coupling.schema.json`), declarations and rights, links (units, factors, changes of scale, mappings between data structures), stages, the From fcd262099e34efcfd98362b5f937701dde1c0143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 15:21:10 +0100 Subject: [PATCH 22/39] Lazy reading (read(path, lazy=True)): the file mapped in memory, its JSON parsed, each column and mesh decoded on first access (LazyColumns, LazyList; decoded columns are copies), frame.close() or a with block; the ParaView reader and replays use it (a replay decodes only the columns links use); the folder transport deletes each message once read unless the session keeps them (keepMessages in the welcome, Coordinator(keep_messages=), --keep-messages), in the Python, Julia, C++ and Java participants, the conformance sessions checking the folder is left empty; tests --- CHANGELOG.md | 7 + bindings/cpp/include/fspm_tf/coupling.hpp | 2 + bindings/cpp/src/coupling.cpp | 16 +- .../org/openalea/fspmtf/coupling/Channel.java | 3 + .../fspmtf/coupling/FolderChannel.java | 20 ++- .../openalea/fspmtf/coupling/Participant.java | 1 + bindings/julia/FspmTF/src/coupling.jl | 24 ++- bindings/python/src/openalea/fspm_tf/cli.py | 5 +- bindings/python/src/openalea/fspm_tf/codec.py | 158 +++++++++++------- .../src/openalea/fspm_tf/coupling/channels.py | 39 ++++- .../openalea/fspm_tf/coupling/conformance.py | 6 + .../openalea/fspm_tf/coupling/coordinator.py | 12 +- .../openalea/fspm_tf/coupling/participant.py | 2 + .../src/openalea/fspm_tf/coupling/replay.py | 21 ++- bindings/python/src/openalea/fspm_tf/glb.py | 5 +- bindings/python/src/openalea/fspm_tf/model.py | 97 +++++++++++ bindings/python/test/test_coupling.py | 4 +- bindings/python/test/test_lazy.py | 50 ++++++ doc/coupling.md | 14 +- doc/python.md | 16 ++ paraview/fspm_tf_reader.py | 9 +- spec/schema/coupling.schema.json | 3 +- spec/specification.md | 7 +- 23 files changed, 428 insertions(+), 93 deletions(-) create mode 100644 bindings/python/test/test_lazy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f1437..abb7502 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ ## Unreleased: specification 0.3 (draft), coupling platforms +- Lazy reading (`read(path, lazy=True)`): the file mapped in memory, its JSON parsed, each column and mesh decoded on + first access (`LazyColumns`, `LazyList`), `frame.close()` or a `with` block; used by the ParaView reader and by + replays (only the columns links use). +- The folder transport deletes each message once read, unless the session keeps them (`keepMessages`, + `--keep-messages`); every participant (Python, Julia, C++, Java) applies it, and the conformance sessions check + that the folder is left empty. + - Specification 0.3: the frames say 0.3 (0.1 and 0.2 files stay valid); "Changes from 0.2"; the conformance files regenerated (their expected values unchanged). - Links between two data structures, through a mapping a participant provides (declared, sent in its updates): diff --git a/bindings/cpp/include/fspm_tf/coupling.hpp b/bindings/cpp/include/fspm_tf/coupling.hpp index 63cfb6b..f5e306c 100644 --- a/bindings/cpp/include/fspm_tf/coupling.hpp +++ b/bindings/cpp/include/fspm_tf/coupling.hpp @@ -41,6 +41,8 @@ class Channel { virtual void send(const json &header, const std::vector *payload = nullptr) = 0; /// The next message; throws Error after `timeout` seconds without one (none: wait). virtual Message receive(std::optional timeout) = 0; + /// As the session says (its welcome's keepMessages): keep the messages read, or delete them (the folder transport). + virtual void keep_messages(bool) {} }; /// A participant's channel to `endpoint` ("files:path", or a ZeroMQ endpoint). diff --git a/bindings/cpp/src/coupling.cpp b/bindings/cpp/src/coupling.cpp index 77e55b9..a5da489 100644 --- a/bindings/cpp/src/coupling.cpp +++ b/bindings/cpp/src/coupling.cpp @@ -73,17 +73,30 @@ class FolderChannel : public Channel { } const auto text = read_bytes(stem.string() + ".json"); Message message{json::parse(text.begin(), text.end()), std::nullopt}; + std::vector files{stem.string() + ".json"}; if (message.header.contains("payload")) { - message.payload = read_bytes(incoming_ / message.header["payload"].get()); + files.push_back(incoming_ / message.header["payload"].get()); + message.payload = read_bytes(files.back()); message.header.erase("payload"); } ++next_read_; + if (keep_) read_files_.insert(read_files_.end(), files.begin(), files.end()); + else for (const auto &path : files) filesystem::remove(path); return message; } + void keep_messages(bool keep) override { + if (keep) return; + keep_ = false; + for (const auto &path : read_files_) filesystem::remove(path); + read_files_.clear(); + } + private: filesystem::path incoming_, outgoing_; std::int64_t next_read_ = 0, next_write_ = 0; + bool keep_ = true; // false: a message's files deleted once read + std::vector read_files_; // those of the messages read and kept }; #ifdef FSPM_TF_WITH_ZMQ @@ -181,6 +194,7 @@ Participant::Participant(json declaration, std::string endpoint, json links, std id_offset = welcome.header.at("idOffset").get(); id_count = welcome.header.at("idCount").get(); session = welcome.header.at("session"); + channel_->keep_messages(session.value("keepMessages", true)); couplings = welcome.header.value("couplings", json::array()); } diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java index e904ba8..2c51c31 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Channel.java @@ -12,6 +12,9 @@ record Message(ObjectNode header, byte[] payload) {} /** The next message; FspmTfException after {@code timeout} seconds without one (null: wait). */ Message receive(Double timeout); + /** As the session says (its welcome's keepMessages): keep the messages read, or delete them (the folder transport). */ + default void keepMessages(boolean keep) {} + @Override default void close() {} diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java index c137741..ceff8cf 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/FolderChannel.java @@ -18,6 +18,8 @@ public final class FolderChannel implements Channel { private final Path incoming, outgoing; private long nextRead = 0, nextWrite = 0; + private boolean keep = true; // false: a message's files deleted once read + private final java.util.List readFiles = new java.util.ArrayList<>(); // those read and kept public FolderChannel(String folder, String participant) { incoming = Path.of(folder, participant, "to_participant"); @@ -52,6 +54,18 @@ public void send(ObjectNode header, byte[] payload) { nextWrite++; } + @Override + public void keepMessages(boolean keep) { + if (keep) return; + this.keep = false; + try { + for (Path file : readFiles) Files.deleteIfExists(file); + } catch (IOException error) { + throw new UncheckedIOException(error); + } + readFiles.clear(); + } + @Override public Message receive(Double timeout) { Path header = incoming.resolve(numbered(nextRead) + ".json"); @@ -63,12 +77,16 @@ public Message receive(Double timeout) { Thread.sleep(5); } ObjectNode read = (ObjectNode) Json.MAPPER.readTree(Files.readString(header)); + java.util.List files = new java.util.ArrayList<>(java.util.List.of(header)); byte[] payload = null; if (read.has("payload")) { - payload = Files.readAllBytes(incoming.resolve(read.get("payload").asText())); + files.add(incoming.resolve(read.get("payload").asText())); + payload = Files.readAllBytes(files.get(1)); read.remove("payload"); } nextRead++; + if (keep) readFiles.addAll(files); + else for (Path file : files) Files.delete(file); return new Message(read, payload); } catch (IOException error) { throw new UncheckedIOException(error); diff --git a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java index c20b999..3a50371 100644 --- a/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java +++ b/bindings/java/src/main/java/org/openalea/fspmtf/coupling/Participant.java @@ -62,6 +62,7 @@ public Participant(ObjectNode declaration, String endpoint, ArrayNode links, Dou idOffset = welcome.header().get("idOffset").asLong(); idCount = welcome.header().get("idCount").asLong(); session = welcome.header().get("session"); + channel.keepMessages(session.path("keepMessages").asBoolean(true)); couplings = welcome.header().path("couplings"); } diff --git a/bindings/julia/FspmTF/src/coupling.jl b/bindings/julia/FspmTF/src/coupling.jl index ac1da62..dd235c7 100644 --- a/bindings/julia/FspmTF/src/coupling.jl +++ b/bindings/julia/FspmTF/src/coupling.jl @@ -177,8 +177,13 @@ mutable struct FolderChannel <: CouplingChannel outgoing::String next_read::Int next_write::Int + keep::Bool # false: a message's files deleted once read + read_files::Vector{String} # those of the messages read and kept end +FolderChannel(incoming, outgoing, next_read, next_write) = + FolderChannel(incoming, outgoing, next_read, next_write, true, String[]) + function FolderChannel(folder::AbstractString, participant::AbstractString) incoming = joinpath(folder, participant, "to_participant") outgoing = joinpath(folder, participant, "to_coordinator") @@ -210,13 +215,29 @@ function receive_message(channel::FolderChannel, timeout = nothing) sleep(0.005) end header = JSON.parse(read(path * ".json", String)) - payload = haskey(header, "payload") ? read(joinpath(channel.incoming, pop!(header, "payload"))) : nothing + files = [path * ".json"] + payload = nothing + if haskey(header, "payload") + push!(files, joinpath(channel.incoming, pop!(header, "payload"))) + payload = read(files[end]) + end channel.next_read += 1 + channel.keep ? append!(channel.read_files, files) : foreach(rm, files) return header, payload end close_channel(::CouplingChannel) = nothing +"""As the session says (its welcome's `keepMessages`): keep the messages read, or delete them, those read so far too.""" +keep_messages!(::CouplingChannel, keep::Bool) = nothing +function keep_messages!(channel::FolderChannel, keep::Bool) + keep && return nothing + channel.keep = false + foreach(path -> isfile(path) && rm(path), channel.read_files) + empty!(channel.read_files) + return nothing +end + """ zmq_channel(endpoint, participant) -> CouplingChannel @@ -281,6 +302,7 @@ function Participant(declaration::Declaration; endpoint = get(ENV, ENDPOINT_VARI header, _ = receive_message(channel, timeout) header["type"] == "stop" && error("the coordinator refused $(declaration.participant): $(get(header, "reason", ""))") header["type"] == "welcome" || error("expected a welcome, got $(header["type"])") + keep_messages!(channel, get(header["session"], "keepMessages", true)) return Participant(channel, declaration, Int64(header["idOffset"]), Int64(header["idCount"]), header["session"], collect(get(header, "couplings", Any[])), -1, nothing, timeout === nothing ? nothing : Float64(timeout)) diff --git a/bindings/python/src/openalea/fspm_tf/cli.py b/bindings/python/src/openalea/fspm_tf/cli.py index a9381ce..45340e9 100644 --- a/bindings/python/src/openalea/fspm_tf/cli.py +++ b/bindings/python/src/openalea/fspm_tf/cli.py @@ -117,7 +117,8 @@ def _couple(arguments) -> int: coordinator = Coordinator(arguments.endpoint, steps=arguments.steps, interval=arguments.interval, time_unit=arguments.time_unit, start=arguments.start, links=links, stages=stages, expect=arguments.expect or [], replays=_pairs(arguments.replay, "replay"), - record=arguments.record, timeout=arguments.timeout) + record=arguments.record, timeout=arguments.timeout, + keep_messages=arguments.keep_messages) print(f"coordinator at {coordinator.endpoint}", flush=True) for name, command in _pairs(arguments.participant, "participant").items(): coordinator.launch(name, command) @@ -187,6 +188,8 @@ def main(argv=None) -> int: couple.add_argument("--endpoint", default="tcp://127.0.0.1:*", help="a ZeroMQ endpoint, or files:FOLDER") couple.add_argument("--record", help="a folder: the session recorded as a sequence") couple.add_argument("--timeout", type=float, default=600., help="seconds to wait for a message") + couple.add_argument("--keep-messages", action="store_true", help="over a folder, keep every message (else each " + "is deleted once read)") couple.set_defaults(run=_couple) sessions = commands.add_parser("couple-conformance", help="a participant's side against the coupling " "conformance sessions") diff --git a/bindings/python/src/openalea/fspm_tf/codec.py b/bindings/python/src/openalea/fspm_tf/codec.py index 38be5ff..8760a16 100644 --- a/bindings/python/src/openalea/fspm_tf/codec.py +++ b/bindings/python/src/openalea/fspm_tf/codec.py @@ -19,8 +19,9 @@ from openalea.fspm_tf import SPECIFICATION_VERSION from openalea.fspm_tf.glb import BufferBuilder, pack, unpack, view_bytes -from openalea.fspm_tf.model import (DataStructure, EDGE_TYPES, ENUM_COLUMNS, EntitySet, Frame, Grid, Mapping, Mesh, NO_ID, - Relation, Table, Update, Variable) +from openalea.fspm_tf.model import (DataStructure, EDGE_TYPES, ENUM_COLUMNS, EntitySet, Frame, Grid, ID_COLUMNS, + LazyColumns, LazyList, Mapping, Mesh, NO_ID, Relation, Table, Update, Variable, + _as_column) EXTENSION = "FSPM_scene" METADATA = "EXT_structural_metadata" @@ -383,6 +384,10 @@ def accessor(array, accessor_type, component_type, target, with_bounds=False): def from_bytes(data: bytes) -> Frame: """The Frame of a GLB file's bytes.""" document, binary = unpack(data) + return _frame_of(document, binary, lazy=False) + + +def _frame_of(document: dict, binary, lazy: bool) -> Frame: extensions = document.get("extensions", {}) if EXTENSION not in extensions: raise ValueError(f"not an fspm-TF file: no {EXTENSION} extension") @@ -392,7 +397,7 @@ def from_bytes(data: bytes) -> Frame: def table(index: int, kind, **fields): description = metadata["propertyTables"][index] class_id = description["class"] - columns, declarations = _decode_table(document, binary, description, metadata["schema"], variables) + columns, declarations = _decode_table(document, binary, description, metadata["schema"], variables, lazy) return kind(name=description.get("name", class_id), columns=columns, variables=declarations, **fields) frame_info = scene.get("frame", {}) @@ -418,8 +423,10 @@ def table(index: int, kind, **fields): periodic=tuple(grid["periodic"]), z_up=grid.get("zUp", True), max_level=grid.get("maxLevel", 0), cells=grid["cells"]) tables = {item["propertyTable"]: item["name"] for item in entry.get("entitySets", [])} - for mesh_entry in entry.get("meshes", []): - data_structure.meshes.append(_decode_mesh(document, binary, mesh_entry, tables, data_structure)) + meshes = [(lambda mesh_entry=mesh_entry, data_structure=data_structure, tables=tables: + _decode_mesh(document, binary, mesh_entry, tables, data_structure)) + for mesh_entry in entry.get("meshes", [])] + data_structure.meshes = LazyList(meshes) if lazy else [mesh() for mesh in meshes] frame.data_structures.append(data_structure) for entry in scene.get("mappings", []): frame.mappings.append(table(entry["propertyTable"], Mapping, @@ -429,7 +436,7 @@ def table(index: int, kind, **fields): if "update" in scene: update = scene["update"] frame.update = Update(participant=update["participant"], removed={ - (item["dataStructure"], item["entitySet"]): table(item["propertyTable"], Table).columns["id"] + (item["dataStructure"], item["entitySet"]): np.asarray(table(item["propertyTable"], Table).columns["id"]) for item in update.get("removed", [])}) unknown_extensions = {name: value for name, value in extensions.items() if name not in (METADATA, EXTENSION, FEATURES)} @@ -440,61 +447,50 @@ def table(index: int, kind, **fields): return frame -def read(path: str) -> Frame: - """The Frame of the GLB file *path*.""" +def read(path: str, lazy: bool = False) -> Frame: + """The Frame of the GLB file *path*. *lazy*: the file is mapped in memory and only its JSON parsed; each column + (LazyColumns) and each mesh (LazyList) is decoded when first read, so that a reader using a few columns of a large + frame decodes only those. A lazy frame holds the file open until frame.close() (or the end of a with block).""" + if not lazy: + with open(path, "rb") as file: + return from_bytes(file.read()) + import mmap with open(path, "rb") as file: - return from_bytes(file.read()) + mapped = mmap.mmap(file.fileno(), 0, access=mmap.ACCESS_READ) + document, binary = unpack(memoryview(mapped)) + frame = _frame_of(document, binary, lazy=True) + frame._source = (mapped, binary) + return frame -def _decode_table(document: dict, binary: bytes, description: dict, schema: dict, variables: dict) -> tuple: +def _decode_table(document: dict, binary: bytes, description: dict, schema: dict, variables: dict, + lazy: bool = False) -> tuple: + """(columns, declarations) of a property table; *lazy*: the columns a LazyColumns, each decoded when first read.""" class_id = description["class"] - properties = schema["classes"][class_id]["properties"] count = description["count"] - columns, declarations = {}, {} - for name, stored in description["properties"].items(): + if lazy: + columns = LazyColumns({name: (lambda name=name, stored=stored: _checked( + name, _decode_column(document, binary, schema, class_id, name, stored, count))) + for name, stored in description["properties"].items()}, count) + else: + columns = {name: _decode_column(document, binary, schema, class_id, name, stored, count) + for name, stored in description["properties"].items()} + return columns, _decode_declarations(schema, class_id, description, variables) + + +def _checked(name: str, column): + """A column decoded lazily, as a table checks it (ids int64).""" + column = _as_column(name, column) + if name in ID_COLUMNS and column.dtype.kind in "iu" and not np.ma.isMaskedArray(column): + column = column.astype(np.int64) + return column + + +def _decode_declarations(schema: dict, class_id: str, description: dict, variables: dict) -> dict: + properties = schema["classes"][class_id]["properties"] + declarations = {} + for name in description["properties"]: prop = properties[name] - raw = view_bytes(document, binary, stored["values"]) - if prop["type"] == "ENUM": - enum = schema["enums"][prop["enumType"]] - codes = np.frombuffer(raw, dtype=enum.get("valueType", "UINT16").lower(), count=count) - values = np.array([item["value"] for item in enum["values"]], dtype=np.int64) - names = np.empty(int(values.max()) + 1 if values.size else 0, dtype=object) - names[values] = [item["name"] for item in enum["values"]] - column = names[codes.astype(np.int64)] if count else np.zeros(0, dtype=object) - elif prop["type"] == "STRING": - offset_type = stored.get("stringOffsetType", "UINT32").lower() - offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, - count=count + 1) - column = np.array([raw[offsets[i]:offsets[i + 1]].decode("utf-8") for i in range(count)], dtype=object) - if "noData" in prop: # missing text - column = np.array([None if value == prop["noData"] else value for value in column], dtype=object) - elif prop["type"] == "BOOLEAN": - column = np.unpackbits(np.frombuffer(raw, dtype=np.uint8), bitorder="little", count=count).astype(bool) - elif prop["type"] == "SCALAR": - dtype = DTYPES[prop["componentType"]] - width = prop.get("count", 1) if prop.get("array") else 1 - if prop.get("array") and "count" not in prop: - raise ValueError(f"'{class_id}.{name}': variable-length arrays are not part of fspm-TF 0.1") - column = np.frombuffer(raw, dtype=dtype.newbyteorder("<"), count=count * width).astype(dtype) - if width > 1: - column = column.reshape(count, width) - if dtype.kind == "f" and "noData" in prop: - column = np.where(column == np.asarray(prop["noData"], dtype=dtype), np.nan, column).astype(dtype) - infinity = prop.get("extras", {}).get("FSPM_infinity") - if dtype.kind == "f" and infinity: - column = np.where(column == np.asarray(infinity["positive"], dtype=dtype), np.inf, - np.where(column == np.asarray(infinity["negative"], dtype=dtype), -np.inf, column) - ).astype(dtype) - if prop.get("extras", {}).get("FSPM_boolean"): # booleans with missing values - missing = column == prop.get("noData", BOOLEAN_NO_DATA) - column = np.ma.MaskedArray(column == 1, mask=missing) if missing.any() else column == 1 - elif dtype.kind in "iu" and "noData" in prop and name not in ("complex", "parent"): # missing integers - missing = column == prop["noData"] - if missing.any(): - column = np.ma.MaskedArray(column, mask=missing) - else: - raise ValueError(f"'{class_id}.{name}': property type {prop['type']} is not part of fspm-TF 0.1") - columns[name] = column declared = variables.get(class_id, {}).get(name) if declared is not None or "description" in prop: declared = declared or {} @@ -504,7 +500,55 @@ def _decode_table(document: dict, binary: bytes, description: dict, schema: dict minimum=declared.get("minimum"), maximum=declared.get("maximum"), references=tuple(declared.get("references", ())), doi=tuple(declared.get("doi", ())), format=declared.get("format")) - return columns, declarations + return declarations + + +def _decode_column(document: dict, binary, schema: dict, class_id: str, name: str, stored: dict, count: int): + """One column of a property table, decoded from the binary chunk (a copy, not a view of it).""" + prop = schema["classes"][class_id]["properties"][name] + raw = view_bytes(document, binary, stored["values"]) + if prop["type"] == "ENUM": + enum = schema["enums"][prop["enumType"]] + codes = np.frombuffer(raw, dtype=enum.get("valueType", "UINT16").lower(), count=count) + values = np.array([item["value"] for item in enum["values"]], dtype=np.int64) + names = np.empty(int(values.max()) + 1 if values.size else 0, dtype=object) + names[values] = [item["name"] for item in enum["values"]] + return names[codes.astype(np.int64)] if count else np.zeros(0, dtype=object) + if prop["type"] == "STRING": + offset_type = stored.get("stringOffsetType", "UINT32").lower() + offsets = np.frombuffer(view_bytes(document, binary, stored["stringOffsets"]), dtype=offset_type, + count=count + 1) + raw = bytes(raw) + column = np.array([raw[offsets[i]:offsets[i + 1]].decode("utf-8") for i in range(count)], dtype=object) + if "noData" in prop: # missing text + column = np.array([None if value == prop["noData"] else value for value in column], dtype=object) + return column + if prop["type"] == "BOOLEAN": + return np.unpackbits(np.frombuffer(raw, dtype=np.uint8), bitorder="little", count=count).astype(bool) + if prop["type"] != "SCALAR": + raise ValueError(f"'{class_id}.{name}': property type {prop['type']} is not part of fspm-TF") + dtype = DTYPES[prop["componentType"]] + width = prop.get("count", 1) if prop.get("array") else 1 + if prop.get("array") and "count" not in prop: + raise ValueError(f"'{class_id}.{name}': variable-length arrays are not part of fspm-TF") + column = np.frombuffer(raw, dtype=dtype.newbyteorder("<"), count=count * width).astype(dtype) + if width > 1: + column = column.reshape(count, width) + if dtype.kind == "f" and "noData" in prop: + column = np.where(column == np.asarray(prop["noData"], dtype=dtype), np.nan, column).astype(dtype) + infinity = prop.get("extras", {}).get("FSPM_infinity") + if dtype.kind == "f" and infinity: + column = np.where(column == np.asarray(infinity["positive"], dtype=dtype), np.inf, + np.where(column == np.asarray(infinity["negative"], dtype=dtype), -np.inf, column) + ).astype(dtype) + if prop.get("extras", {}).get("FSPM_boolean"): # booleans with missing values + missing = column == prop.get("noData", BOOLEAN_NO_DATA) + return np.ma.MaskedArray(column == 1, mask=missing) if missing.any() else column == 1 + if dtype.kind in "iu" and "noData" in prop and name not in ("complex", "parent"): # missing integers + missing = column == prop["noData"] + if missing.any(): + return np.ma.MaskedArray(column, mask=missing) + return column def _accessor_array(document: dict, binary: bytes, index: int) -> np.ndarray: @@ -538,7 +582,7 @@ def _decode_mesh(document: dict, binary: bytes, mesh_entry: dict, tables: dict, feature_rows[set_name] = rows display, sources = {}, {} for item in mesh_entry.get("display", []): - display[item["variable"]] = _accessor_array(document, binary, attributes[item["attribute"]]) + display[item["variable"]] = _accessor_array(document, binary, attributes[item["attribute"]]).copy() if item.get("entitySet") is not None: sources[item["variable"]] = item["entitySet"] indices = (_accessor_array(document, binary, primitive["indices"]).astype(np.int64) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/channels.py b/bindings/python/src/openalea/fspm_tf/coupling/channels.py index 9cc7730..bd2b9f1 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/channels.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/channels.py @@ -7,7 +7,8 @@ - a folder (endpoint "files:path"): each participant has a folder // with two message queues, to_coordinator/ and to_participant/; message n is n.json (its header) and n.glb (its payload), the payload written first and the header renamed into place last (a header present: a complete message). Nothing else is needed than - files, so any platform can take part, and the folder is the session's log. + files, so any platform can take part. Each message is deleted by its reader once read, unless the session keeps + them (keepMessages: the folder is then the session's log; a record, Coordinator(record=), is the usual one). Both sides poll; every wait has a timeout (seconds, None: none). """ @@ -90,6 +91,16 @@ def __init__(self, folder: str): os.makedirs(folder, exist_ok=True) self.next_read = 0 self.next_write = 0 + self.keep = True # False: a message's files deleted once read + self.read_files = [] # those of the messages read and kept + + def discard_read(self) -> None: + """Delete the files of the messages read so far, and from now on each message once read.""" + self.keep = False + for path in self.read_files: + if os.path.exists(path): + os.remove(path) + self.read_files = [] def write(self, header: dict, payload: bytes = None) -> None: stem = os.path.join(self.folder, f"{self.next_write:08d}") @@ -109,28 +120,37 @@ def read(self): return None with open(stem + ".json", "rb") as file: header = _header(file.read()) - payload = None + payload, files = None, [stem + ".json"] if "payload" in header: - with open(os.path.join(self.folder, header.pop("payload")), "rb") as file: + files.append(os.path.join(self.folder, header.pop("payload"))) + with open(files[-1], "rb") as file: payload = file.read() self.next_read += 1 + if self.keep: + self.read_files += files + else: + for path in files: + os.remove(path) return header, payload class FolderCoordinatorChannel: - def __init__(self, folder: str): + def __init__(self, folder: str, keep: bool = False): self.folder = os.path.abspath(folder) os.makedirs(self.folder, exist_ok=True) for name in os.listdir(self.folder): # a session's folder is the log of that session only if os.path.exists(os.path.join(self.folder, name, "to_participant", f"{0:08d}.json")): raise ValueError(f"{self.folder} holds a previous session ({name}): use a new folder") self.endpoint = "files:" + self.folder + self.keep = keep self.incoming, self.outgoing = {}, {} def _discover(self) -> None: for name in sorted(os.listdir(self.folder)): if name not in self.incoming and os.path.isdir(os.path.join(self.folder, name, "to_coordinator")): self.incoming[name] = _Queue(os.path.join(self.folder, name, "to_coordinator")) + if not self.keep: + self.incoming[name].discard_read() self.outgoing[name] = _Queue(os.path.join(self.folder, name, "to_participant")) def receive(self, timeout=None): @@ -172,15 +192,20 @@ def receive(self, timeout=None): def send(self, header: dict, payload: bytes = None) -> None: self.outgoing.write(header, payload) + def keep_messages(self, keep: bool) -> None: + """As the session says (its welcome): keep the messages read, or delete them (those read so far too).""" + if not keep: + self.incoming.discard_read() + def close(self) -> None: pass -def coordinator_channel(endpoint: str): +def coordinator_channel(endpoint: str, keep_messages: bool = False): """The coordinator's side of *endpoint*: "files:path", or a ZeroMQ endpoint to bind ("tcp://127.0.0.1:*": a - free port).""" + free port). *keep_messages*: the folder transport keeps every message (the session's log).""" if endpoint.startswith("files:"): - return FolderCoordinatorChannel(endpoint[len("files:"):]) + return FolderCoordinatorChannel(endpoint[len("files:"):], keep_messages) return ZmqCoordinatorChannel(endpoint) diff --git a/bindings/python/src/openalea/fspm_tf/coupling/conformance.py b/bindings/python/src/openalea/fspm_tf/coupling/conformance.py index e8258b5..62cfa84 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/conformance.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/conformance.py @@ -68,6 +68,12 @@ def run_session(session: str, command: str, out: str, transport: str = "zmq") -> coordinator.launch("subject", f"{command} {session} {turns}") coordinator.run() differences = [] + if transport == "files": # each message deleted by its reader once read + left = [os.path.join(root, name) for root, _, names in os.walk(os.path.join(out, "messages")) + for name in names] + if left: + differences.append(f"{len(left)} message file(s) left in the folder, e.g. " + f"{os.path.relpath(left[0], out)}: a reader deletes each message once read") expected_folder = os.path.join(session, "expected") for name in sorted(os.listdir(expected_folder)): if name.startswith("turn_") and name.endswith(".json"): diff --git a/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py b/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py index dab53cb..77ab289 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/coordinator.py @@ -14,7 +14,7 @@ 5. Stop. A participant's error, or its update the store refuses, stops the session: every participant is told why. *record*: a folder where the session is written as a sequence of full frames (every entity set and output), after the -initial state and each step. +initial state and each step. *keep_messages*: over a folder, every message kept (else each deleted once read). """ import os import subprocess @@ -33,8 +33,9 @@ class Coordinator: def __init__(self, endpoint: str = "tcp://127.0.0.1:*", steps: int = 1, interval: float = 1., time_unit: str = "d", start: float = 0., links=(), stages=None, expect=(), replays=None, - record: str = None, timeout: float = 600.): - self.channel = coordinator_channel(endpoint) + record: str = None, timeout: float = 600., keep_messages: bool = False): + self.channel = coordinator_channel(endpoint, keep_messages) + self.keep_messages = keep_messages self.endpoint = self.channel.endpoint self.steps, self.interval, self.time_unit, self.start = steps, interval, time_unit, start self.links = [link if isinstance(link, Link) else Link.from_json(link) for link in links] @@ -130,8 +131,11 @@ def _join(self) -> None: replay.linear = {(coupling.source.data_structure, coupling.source.entity_set, coupling.source.name) for coupling in self.couplings if coupling.provider == name and coupling.interpolation == "linear"} + replay.needed = {(coupling.source.data_structure, coupling.source.entity_set, coupling.source.name) + for coupling in self.couplings if coupling.provider == name} session = {"start": self.start, "interval": self.interval, "timeUnit": self.time_unit, "steps": self.steps, - "stages": self.stages, "participants": list(self.declarations)} + "stages": self.stages, "participants": list(self.declarations), + "keepMessages": self.keep_messages} for name in self.declarations: if name in self.replays: continue diff --git a/bindings/python/src/openalea/fspm_tf/coupling/participant.py b/bindings/python/src/openalea/fspm_tf/coupling/participant.py index 99526d6..42cf8ee 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/participant.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/participant.py @@ -62,6 +62,8 @@ def connect(cls, declaration: Declaration, endpoint: str = None, links=(), timeo raise CouplingError(f"expected a welcome, got {header['type']}") participant.id_offset = header["idOffset"] participant.session = header["session"] + if hasattr(participant.channel, "keep_messages"): # the folder transport + participant.channel.keep_messages(header["session"].get("keepMessages", True)) participant.couplings = header.get("couplings", []) return participant diff --git a/bindings/python/src/openalea/fspm_tf/coupling/replay.py b/bindings/python/src/openalea/fspm_tf/coupling/replay.py index 29391ca..2a96330 100644 --- a/bindings/python/src/openalea/fspm_tf/coupling/replay.py +++ b/bindings/python/src/openalea/fspm_tf/coupling/replay.py @@ -27,13 +27,16 @@ def __init__(self, name: str, index_path: str): self._cache = {} self.id_offset = 0 self.linear = set() # (data structure, entity set, variable) interpolated + self.needed = None # (data structure, entity set, variable) some link uses (None: all) self.sent = {} # (data structure, entity set): ids sent at the last turn self.id_map = {} # recorded id: session id self.declaration = self._declare() def _frame(self, position: int) -> Frame: - if position not in self._cache: - self._cache = {position: read(self.paths[position])} # one frame kept + if position not in self._cache: # one frame kept, read lazily: only the columns used + for frame in self._cache.values(): + frame.close() + self._cache = {position: read(self.paths[position], lazy=True)} return self._cache[position] def _declare(self) -> Declaration: @@ -75,14 +78,18 @@ def update(self, time: float, time_unit: str) -> Frame: frame = self._frame(position) following = None if self.linear and position + 1 < len(self.times) and recorded_time > self.times[position]: - following = read(self.paths[position + 1]) + following = read(self.paths[position + 1], lazy=True) weight = (recorded_time - self.times[position]) / (self.times[position + 1] - self.times[position]) structures, removed = [], {} for data_structure in frame.data_structures: out = DataStructure(data_structure.id) for entity_set in data_structure.entity_sets: columns = {} - for name, values in entity_set.columns.items(): + for name in entity_set.columns: + if name not in entity_set.reserved and self.needed is not None \ + and (data_structure.id, entity_set.name, name) not in self.needed: + continue # a variable no link uses: not decoded + values = entity_set.columns[name] if name in _ID_COLUMNS: values = np.array([self._session_id(int(value)) for value in np.asarray(values).tolist()], dtype=np.int64) @@ -90,7 +97,8 @@ def update(self, time: float, time_unit: str) -> Frame: if following is not None: self._interpolate(data_structure.id, entity_set, following, weight, columns) out.entity_sets.append(EntitySet(entity_set.name, columns=columns, rank=entity_set.rank, - variables=dict(entity_set.variables))) + variables={name: variable for name, variable in + entity_set.variables.items() if name in columns})) key = (data_structure.id, entity_set.name) ids = set(columns["id"].tolist()) if "id" in columns else set() gone = self.sent.get(key, set()) - ids @@ -126,7 +134,8 @@ def _interpolate(self, data_structure_id, entity_set, following, weight, columns rows = np.array([later_rows.get(identifier, -1) for identifier in entity_set.columns["id"].tolist()]) found = rows >= 0 for name in entity_set.variable_names(): - if (data_structure_id, entity_set.name, name) not in self.linear or name not in later.columns: + if (data_structure_id, entity_set.name, name) not in self.linear or name not in later.columns \ + or name not in columns: continue values = np.asarray(columns[name]) if values.dtype.kind != "f": diff --git a/bindings/python/src/openalea/fspm_tf/glb.py b/bindings/python/src/openalea/fspm_tf/glb.py index 69ac098..1ee70c8 100644 --- a/bindings/python/src/openalea/fspm_tf/glb.py +++ b/bindings/python/src/openalea/fspm_tf/glb.py @@ -24,7 +24,8 @@ def pack(document: dict, binary: bytes) -> bytes: def unpack(data: bytes) -> tuple: - """(JSON document, binary buffer) of a GLB file.""" + """(JSON document, binary buffer) of a GLB file (*data*: bytes, or a memoryview: the binary chunk then a view of + it, not a copy).""" if len(data) < 20: raise ValueError("not a GLB file: too short") magic, version, length = struct.unpack_from(" tuple: chunk_length, chunk_type = struct.unpack_from(" list: + """The names of the columns decoded so far.""" + return [name for name in self._order if name in self._values] + + +class LazyList(MutableSequence): + """A list whose items (a data structure's meshes, read lazily) are decoded on first access.""" + + def __init__(self, loaders=()): + self._items = [(False, loader) for loader in loaders] + + def __getitem__(self, index): + if isinstance(index, slice): + return [self[position] for position in range(*index.indices(len(self)))] + done, item = self._items[index] + if not done: + item = item() + self._items[index] = (True, item) + return item + + def __setitem__(self, index, value): + self._items[index] = (True, value) + + def __delitem__(self, index): + del self._items[index] + + def __len__(self): + return len(self._items) + + def insert(self, index, value): + self._items.insert(index, (True, value)) + + @dataclass class Variable: """The declaration of a variable (a column): its unit (a UCUM code), its standard name in the vocabulary, its @@ -70,6 +143,8 @@ class Table: table_kind = "scalars" def __post_init__(self): + if isinstance(self.columns, LazyColumns): # read lazily: each column checked as it is decoded + return self.columns = {name: _as_column(name, values) for name, values in self.columns.items()} for name in set(self.columns) & set(ID_COLUMNS) & set(self.reserved): # ids are int64, whatever the platform's if self.columns[name].dtype.kind in "iu": # default integer @@ -82,6 +157,8 @@ def __post_init__(self): @property def count(self) -> int: + if isinstance(self.columns, LazyColumns): + return self.columns.count return len(next(iter(self.columns.values()))) if self.columns else 0 @property @@ -238,6 +315,26 @@ def data_structure(self, id: str) -> DataStructure: return data_structure raise KeyError(f"the frame has no data structure '{id}'") + def close(self) -> None: + """Release the file of a frame read lazily (codec.read(path, lazy=True)); its columns not decoded yet can no + longer be read. Nothing to do for a frame read whole.""" + source = getattr(self, "_source", None) + if source is None: + return + mapped, binary = source + self._source = None + try: + binary.release() + mapped.close() + except BufferError: # a view of the file still alive: closed when it goes + pass + + def __enter__(self): + return self + + def __exit__(self, *exception): + self.close() + def _as_column(name: str, values) -> np.ndarray: if np.ma.isMaskedArray(values): # integers or booleans with missing values diff --git a/bindings/python/test/test_coupling.py b/bindings/python/test/test_coupling.py index d4578d9..7cdf5f9 100644 --- a/bindings/python/test/test_coupling.py +++ b/bindings/python/test/test_coupling.py @@ -169,7 +169,7 @@ def test_the_messages_follow_their_schema(tmp_path): import json import jsonschema from openalea.fspm_tf.schema import load - run_session(f"files:{tmp_path / 'session'}") + run_session(f"files:{tmp_path / 'session'}", keep_messages=True) validator = jsonschema.Draft202012Validator(load("coupling")) headers = sorted(glob.glob(str(tmp_path / "session" / "*" / "*" / "*.json"))) types = set() @@ -265,7 +265,7 @@ def test_the_command_line(tmp_path): "--timeout", "60"]) assert code == 0 assert validate_sequence(str(tmp_path / "again" / "scene.fspm.json")).ok - assert (tmp_path / "session" / "reader" / "to_coordinator" / "00000000.json").exists() # the session's log + assert not list((tmp_path / "session" / "reader" / "to_coordinator").iterdir()) # each message read, deleted WATER = np.array([0.1, 0.2, 0.3]) # the soil's three cells diff --git a/bindings/python/test/test_lazy.py b/bindings/python/test/test_lazy.py new file mode 100644 index 0000000..23ad249 --- /dev/null +++ b/bindings/python/test/test_lazy.py @@ -0,0 +1,50 @@ +""" +Frames read lazily (read(path, lazy=True)): the same frame as read whole, each column decoded only when read, the meshes +too, written back the same, and the file released by close() or a with block. +""" +import os + +import pytest + +from openalea import fspm_tf +from openalea.fspm_tf.dump import frame_to_dict +from openalea.fspm_tf.model import LazyColumns + +from conftest import CONFORMANCE + +FILES = sorted(os.path.join(folder, name) for folder, _, names in os.walk(CONFORMANCE) for name in names + if name.endswith(".glb") and "coupling" not in os.path.relpath(folder, CONFORMANCE)) + + +@pytest.mark.parametrize("path", FILES, ids=[os.path.relpath(path, CONFORMANCE) for path in FILES]) +def test_read_lazily_the_same(path): + with fspm_tf.read(path, lazy=True) as frame: + assert frame_to_dict(frame) == frame_to_dict(fspm_tf.read(path)) + assert fspm_tf.to_bytes(frame) == fspm_tf.to_bytes(fspm_tf.read(path)) # written back the same + + +def test_only_what_is_read_is_decoded(): + frame = fspm_tf.read(os.path.join(CONFORMANCE, "root_system.glb"), lazy=True) + segments = frame.data_structure("root_system").entity_set("SubOrgan") + assert isinstance(segments.columns, LazyColumns) + assert segments.count == 10 and "length" in segments.columns and segments.columns.decoded() == [] + assert segments.variables["length"].unit == "m" # declarations: from the JSON + lengths = segments.columns["length"] + assert segments.columns.decoded() == ["length"] and lengths.size == 10 + meshes = frame.data_structure("root_system").meshes + assert meshes._items[0][0] is False # not decoded until read + assert len(meshes[0].positions) > 0 + frame.close() + with pytest.raises(ValueError): # the file released + segments.columns["radius"] + assert lengths.sum() > 0 # what was read stays + + +def test_a_lazy_column_set_and_deleted(): + frame = fspm_tf.read(os.path.join(CONFORMANCE, "root_system.glb"), lazy=True) + segments = frame.data_structure("root_system").entity_set("SubOrgan") + segments.columns["doubled"] = 2 * segments.columns["length"] + del segments.columns["radius"] + again = fspm_tf.from_bytes(fspm_tf.to_bytes(frame)).data_structure("root_system").entity_set("SubOrgan") + assert "radius" not in again.columns and (again.columns["doubled"] == 2 * again.columns["length"]).all() + frame.close() diff --git a/doc/coupling.md b/doc/coupling.md index 52ef354..095da78 100644 --- a/doc/coupling.md +++ b/doc/coupling.md @@ -112,8 +112,9 @@ fspm-tf couple --steps 30 --interval 1 --time-unit d \ - `--replay name=run/scene.fspm.json`: a recorded run takes part as a participant, without its platform running. A platform's routine log can thus feed another platform's model. CPlantBox's and GroIMP's logs are tested this way. - `--endpoint files:folder`: the participants exchange files in a folder instead of ZeroMQ messages. It is slower, - but needs nothing but a file system, and the folder keeps every message: plan its disk space (a turn of 100,000 - segments with five variables is about 5 MB), and use a new folder for each session. + but needs nothing but a file system. Each message is deleted once read; `--keep-messages` keeps them all, as the + session's log (a turn of 100,000 segments with five variables is about 5 MB: plan the disk space). The usual log + is `--record`. - `--stages "growth;function"`: who runs first in each step (by default, the participants creating entities, then the others). Each step, a stage sees what the previous stages did in that step. @@ -132,9 +133,12 @@ variables on every segment: | Segments | ZeroMQ | A folder | |---|---|---| -| 1,000 | 3 ms | 13 ms | -| 10,000 | 9 ms | 21 ms | -| 100,000 | 99 ms | 110 ms | +| 1,000 | 3 ms | 12 ms | +| 10,000 | 9–11 ms | 21 ms | +| 100,000 | 99–108 ms | 110–122 ms | + +ZeroMQ here goes through the local network interface (`tcp://127.0.0.1`), its messages held in memory; the folder's +messages are files. The folder costs about 10 ms more per step (its polling and file writes). Encoding and decoding the frames is about a tenth of it (10 ms for 100,000 segments); the rest is the coordinator's work on the shared structure, in Python. diff --git a/doc/python.md b/doc/python.md index d0b06d1..fde2534 100644 --- a/doc/python.md +++ b/doc/python.md @@ -71,6 +71,22 @@ for frame in fspm_tf.read_sequence("outputs/frames/scene.fspm.json"): ... ``` +## Large frames: reading lazily + +`fspm_tf.read(path, lazy=True)` maps the file in memory and parses only its JSON: names, declarations, ranks and +counts are there at once, and each column (and each mesh) is decoded when first read. A reader using a few columns of +a large frame decodes only those (a GroIMP frame of 7.4 MB with a 456,000-vertex mesh: 17 ms read whole, 0.8 ms for +one column). The frame holds the file until it is closed: + +```python +with fspm_tf.read("plant.glb", lazy=True) as frame: + segments = frame.data_structure("plant").entity_set("SubOrgan") + lengths = segments.columns["length"] # decoded now; the other columns are not +``` + +The ParaView reader and a coupling session's replay read frames this way (a replay decodes only the columns some +link uses). + ## The vocabulary ```python diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py index 4d1ed60..2e9b327 100644 --- a/paraview/fspm_tf_reader.py +++ b/paraview/fspm_tf_reader.py @@ -338,7 +338,8 @@ def __init__(self, path: str): self.times = [float(entry["time"]) for entry in index["frames"]] else: self.paths = [path] - self.times = [float(fspm_tf.read(path).time)] + with fspm_tf.read(path, lazy=True) as frame: # its time, from the file's JSON alone + self.times = [float(frame.time)] self._cache = {} def frame_at(self, time: float = None): @@ -347,8 +348,10 @@ def frame_at(self, time: float = None): if time is not None: earlier = [n for n, frame_time in enumerate(self.times) if frame_time <= time + 1e-12 * max(1., abs(time))] number = earlier[-1] if earlier else 0 - if number not in self._cache: - self._cache = {number: _fspm_tf().read(self.paths[number])} + if number not in self._cache: # read lazily: only what the display reads is decoded + for frame in self._cache.values(): + frame.close() + self._cache = {number: _fspm_tf().read(self.paths[number], lazy=True)} return self._cache[number] diff --git a/spec/schema/coupling.schema.json b/spec/schema/coupling.schema.json index acb93d0..58ab619 100644 --- a/spec/schema/coupling.schema.json +++ b/spec/schema/coupling.schema.json @@ -79,7 +79,8 @@ "start": {"type": "number"}, "interval": {"type": "number", "exclusiveMinimum": 0}, "timeUnit": {"type": "string"}, "steps": {"type": "integer", "minimum": 0}, "stages": {"type": "array", "items": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}}, - "participants": {"type": "array", "items": {"$ref": "#/$defs/identifier"}} + "participants": {"type": "array", "items": {"$ref": "#/$defs/identifier"}}, + "keepMessages": {"description": "Over a folder: every message kept (else each deleted by its reader once read).", "type": "boolean"} } }, "couplings": {"type": "array", "items": {"type": "object"}}, "payload": {"$ref": "#/$defs/payload"} diff --git a/spec/specification.md b/spec/specification.md index 4f54052..4898757 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -418,7 +418,7 @@ A message is a JSON header and, for a turn or an update, a payload: a partial fr | Message | From | Header | Payload | |---|---|---|---| | hello | participant | `type`, `protocol` (`"0.3"`), `declaration`, `links` (the links it proposes, may be empty) | – | -| welcome | coordinator | `type`, `idOffset`, `idCount`, `session` (`start`, `interval`, `timeUnit`, `steps`, `stages`, `participants`), `couplings` (its inputs' providers) | – | +| welcome | coordinator | `type`, `idOffset`, `idCount`, `session` (`start`, `interval`, `timeUnit`, `steps`, `stages`, `participants`, `keepMessages`), `couplings` (its inputs' providers) | – | | turn | coordinator | `type`, `step`, `phase` (`"init"` at step 0, else `"step"`), `time`, `until` | its inputs | | update | participant | `type`, `step` | its outputs | | error | participant | `type`, `message` | – | @@ -523,7 +523,10 @@ Both transports carry the same messages. - Its payload is `n.glb`, named in the header's `payload` key. - The payload is written first, and the header is renamed into place last, so a header present means a complete message. - - The folder is the session's log; a session needs a new folder. + - Its reader deletes a message's files once read, unless the session keeps them (`keepMessages` true in the + welcome's `session`; absent: kept). A participant applies it from the welcome on, deleting the welcome too; the + coordinator, from the start. Kept, the folder is the session's log; a session needs a new folder. The usual log of + a session is its record, a sequence of full frames (§8) the coordinator writes when asked. - **Launching.** A participant launched by the coordinator finds the endpoint in the environment variable `FSPM_TF_ENDPOINT` and its name in `FSPM_TF_PARTICIPANT`. - **Replay.** A recorded sequence (§8) may be a participant, so a platform's log feeds another platform's inputs without From 4f355ebd6a29391bc7d795407ac4c23234217486 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 15:37:17 +0100 Subject: [PATCH 23/39] =?UTF-8?q?Docs=20in=20line=20with=200.3:=20the=20sp?= =?UTF-8?q?ec's=20=C2=A73=20names=20update=20(partial=20frames);=20the=20r?= =?UTF-8?q?eading=20guide's=20example=20at=200.3,=20the=20coupling=20schem?= =?UTF-8?q?a=20and=20=C2=A711=20in=20its=20path;=20the=20README=20(couplin?= =?UTF-8?q?g=20sessions,=20schemas,=20conformance=20sessions,=20what=20eac?= =?UTF-8?q?h=20binding=20holds);=20the=20index=20page;=20the=20design=20no?= =?UTF-8?q?te's=20status,=20=C2=A74.6=20and=20=C2=A75.2=20updated=20to=20w?= =?UTF-8?q?hat=20was=20decided=20(GLB=20partial=20frames=20over=20ZeroMQ?= =?UTF-8?q?=20or=20a=20folder,=20not=20Arrow;=20codecs=20in=20this=20repos?= =?UTF-8?q?itory);=20P4=20and=20P5=20done;=20'draft=20for=200.3'=20wording?= =?UTF-8?q?=20removed;=20the=20API=20page=20lists=20LazyColumns,=20LazyLis?= =?UTF-8?q?t=20and=20the=20conformance=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +++++----- doc/api.md | 5 ++++- doc/design_note.md | 22 ++++++++++++++++++---- doc/index.md | 3 ++- doc/platforms.md | 2 +- doc/reading_the_specification.md | 5 ++++- spec/schema/coupling.schema.json | 2 +- spec/specification.md | 2 ++ 8 files changed, 37 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 70e0705..cf83c7e 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ An exchange format for whole functional-structural plant model (FSPM) scenes, ac - their geometry; - the environment's grids. -Each time step is one glTF file, and the variables share one vocabulary. It also aims to serve as an in-memory -interface for coupling components across platforms. +Each time step is one glTF file, and the variables share one vocabulary. It also runs models of several platforms +together, step by step (coupling sessions: a coordinator and each platform's participant, over ZeroMQ or a folder). Status: specification 0.3, draft (0.1 and 0.2 files stay valid): 0.3 adds coupling platforms (§11). @@ -28,10 +28,10 @@ change there, released as a new version. Everything else implements, checks or d | Folder | Content | Discussed with the platforms | |---|---|---| | `spec/specification.md` | the specification: data model, glTF layout, sequences, conformance | **yes** | -| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** | +| `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension, of a sequence's index and views, and of a coupling session's messages | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | -| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` | -| `bindings//` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with a MultiScaleTreeGraph.jl extension; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter; `bindings/java/`: the Java codec, with GroIMP's adapter | no | +| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`); `coupling/`: the coupling sessions a participant runs (`fspm-tf couple-conformance`) | follows `spec/` | +| `bindings//` | each language's codec and coupling participant, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec with lazy reading, validator, display copies, views, the coupling's coordinator, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with MultiScaleTreeGraph.jl, VirtualPlantLab and ZeroMQ extensions; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter; `bindings/java/`: the Java codec, with GroIMP's adapter and plugin | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | | `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | diff --git a/doc/api.md b/doc/api.md index 414f851..32cccd4 100644 --- a/doc/api.md +++ b/doc/api.md @@ -14,7 +14,7 @@ the vocabulary and units. ```{eval-rst} .. automodule:: openalea.fspm_tf.model - :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable, Update + :members: Frame, DataStructure, EntitySet, Relation, Table, Grid, Mesh, Mapping, Variable, Update, LazyColumns, LazyList ``` ## Validation @@ -80,6 +80,9 @@ the vocabulary and units. .. automodule:: openalea.fspm_tf.coupling.replay :members: Replay + +.. automodule:: openalea.fspm_tf.coupling.conformance + :members: run_session, run_all, subject ``` ## Command line diff --git a/doc/design_note.md b/doc/design_note.md index 3ac46c2..d64d4bb 100644 --- a/doc/design_note.md +++ b/doc/design_note.md @@ -2,8 +2,9 @@ *An exchange format for whole FSPM scenes, across platforms and languages.* -Status (2026-10-29): the decisions of §7 are taken on the metafspm side; P1 to P3 are done (§6). The specification -itself is `spec/specification.md`, version 0.1, a draft until the partner platforms have reviewed it. This note keeps +Status (2026-10-29; updated 2026-11-03): the decisions of §7 are taken on the metafspm side; P1 to P5 are done (§6). +The specification itself is `spec/specification.md`, now version 0.3, a draft until the partner platforms have +reviewed it. This note keeps the review of the existing formats and the reasons for the choices; where the two differ, the specification is the reference. Points marked **[to verify]** have not been checked on real files or code yet. @@ -405,6 +406,14 @@ A coupling **protocol** shaped like BMI: This is a later phase (§6). The data model is specified first, so that the coupling inherits it. +**Update (2026-11-03): built differently.** The coupling is the specification's §11. A coordinator (fspm-TF's, in +Python) holds the shared structure and applies the links; each platform's participant declares what it works on, +provides and requires, then answers turns with updates. The messages carry **partial frames**, the same GLB as a time +step, not Arrow batches: every codec already reads and writes GLB, Arrow would have been a large new dependency in +all four (heavy in Java and C++), and measurements showed the format is not the cost (encoding 10 ms of a 100 ms step +at 100,000 segments; the rest is the coordinator's work). Transports: ZeroMQ, and a folder of files. A BMI-like +in-process interface is not part of it. + ### 4.7 VTKHDF (dropped) A VTKHDF export was considered for viewing long simulations in ParaView. It is dropped, to keep fspm-TF free of an @@ -552,6 +561,11 @@ That is a few hundred lines in any language, and existing libraries can help: These codecs live in the platforms' repositories, maintained by their teams. `implementations.md` lists them with the specification version each one passes. fspm-TF itself maintains only Python. +**Update (2026-11-03): decided otherwise.** The codecs live in this repository (`bindings//`: Python, Julia, +C++, Java), each passing the conformance files; the adapters (a platform's model to and from the codec's tables) +belong to the platforms, as Julia package extensions where the platform's library is a plain dependency. +`implementations.md` lists them. + ## 6. Plan | Phase | Content | Done when | Status | @@ -560,8 +574,8 @@ the specification version each one passes. fspm-TF itself maintains only Python. | P1 | Specification v0.1: data model, glTF binding, JSON Schema, first vocabulary; the `openalea.fspm_tf` package (codec, validator, display tool), the conformance files, CI | the validator and glTF-Validator accept the conformance files | done | | P2 | metafspm reference implementation: `export_frame` and `import_frame` for MPG, Array and AdaptiveGrid DataStructures, mappings and triangles | round-trip tests bit-exact on the ported models' scenes (ArchiSimple, Adel, Caribu, GRANAP and MECHA, soil) | done, with `Scene(frame_period=)`; the continuation after an import is exact too | | P3 | The ParaView reader plugin (Python, on the `openalea.fspm_tf` codec: every variable as cell data, frames as time steps); viewer checks of display copies: ParaView without the plugin, three.js/3DTilesRenderer, Blender | the example files viewed with their variables | ParaView done (6.0.1, the plugin and ParaView's own glTF reader); three.js and Blender skipped | -| P4 | A second platform: CPlantBox through its Python layer (the easiest), or VPL through PlantGeom.jl | a file written by one platform and read by the other, with the same values | not started | -| P5 | Arrow binding and coupling protocol; a two-platform coupled run | one component in metafspm and one in another platform exchanging tables at each step | not started | +| P4 | A second platform: CPlantBox through its Python layer (the easiest), or VPL through PlantGeom.jl | a file written by one platform and read by the other, with the same values | done, and more: the Julia (VirtualPlantLab, MTG.jl), C++ (CPlantBox) and Java (GroIMP) codecs and adapters ([Implementations](implementations.md)) | +| P5 | Arrow binding and coupling protocol; a two-platform coupled run | one component in metafspm and one in another platform exchanging tables at each step | done without Arrow (§4.6's update): the coupling sessions of the specification's §11; metafspm coupled with VirtualPlantLab | ## 7. Open decisions diff --git a/doc/index.md b/doc/index.md index fb35514..7184131 100644 --- a/doc/index.md +++ b/doc/index.md @@ -12,7 +12,8 @@ scene at one time step: - **the environment:** soil grids, and which soil cells each root element lies in. A simulation writes one file per time step. Any platform can read them back. ParaView, Blender or any glTF viewer -can display them. +can display them. Platforms can also run their models together, step by step, exchanging their values through the +same format ([Coupling platforms](coupling.md)). ```{image} _static/images/archisimple.png :alt: An ArchiSimple root system read from an fspm-TF file in ParaView, coloured by root order diff --git a/doc/platforms.md b/doc/platforms.md index 5b394f2..2c8a36a 100644 --- a/doc/platforms.md +++ b/doc/platforms.md @@ -102,7 +102,7 @@ A codec that passes is listed in [Implementations](implementations.md) with the ## Taking part in a coupling session -To couple its models with other platforms' (specification §11, draft for 0.3), a codec adds a participant's side; +To couple its models with other platforms' (specification §11), a codec adds a participant's side; the coordinator is fspm-TF's own (`fspm-tf couple`), so no platform implements it. A participant: 1. reads and writes **partial frames**: `FSPM_scene.update` (its sender, and a property table of the removed ids per diff --git a/doc/reading_the_specification.md b/doc/reading_the_specification.md index 9277534..0b4df58 100644 --- a/doc/reading_the_specification.md +++ b/doc/reading_the_specification.md @@ -17,6 +17,7 @@ Everything the platforms agree on is in the repository's `spec/` folder, togethe | `spec/specification.md` | the rules, in words (MUST, SHOULD, MAY as in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119)) | by the validator, `fspm-tf validate` | | `spec/schema/fspm_scene.schema.json` | the shape of the `FSPM_scene` JSON (its fields and their types) | by any JSON Schema validator | | `spec/schema/index.schema.json` | the shape of a sequence's index, `scene.fspm.json` | by any JSON Schema validator | +| `spec/schema/coupling.schema.json` | the shape of a coupling session's messages (their JSON headers, §11) | by any JSON Schema validator | | `spec/schema/views.schema.json` | the shape of a sequence's optional views, `views.json` (how to look at it) | by any JSON Schema validator; names against a frame by `openalea.fspm_tf.views.check_views` | | `spec/vocabulary/standard_names.csv` | the shared quantities: canonical unit, kind, definition, and each platform's names for them | by the validator (units, kinds) | | `spec/vocabulary/labels.csv` | the entity labels, with their scale and Plant Ontology term | by the validator (warnings) | @@ -37,6 +38,8 @@ defect of the lower one. A change of the format is a pull request to `spec/`, wi recognise the same quantity. 4. **§2, §3, §4.1, §7, the encoding.** These come with the example below; a codec's writer needs them in full. 5. **§8 and §9, sequences and conformance.** +6. **§11, coupling**, for a platform whose models run with other platforms' in a session: partial frames, the + messages, the shared structure, the transports. Its conformance sessions are in §9. ## The three layers of a file @@ -54,7 +57,7 @@ Here is the JSON part of `conformance/root_system.glb`, abridged: ```text "extensions": { "FSPM_scene": { - "version": "0.2", "vocabulary": "0.2", + "version": "0.3", "vocabulary": "0.2", "frame": {"time": 0.0, "timeUnit": "d", "step": 0}, "dataStructures": [{ "id": "root_system", "kind": "graph", "origin": [1, 0, 0, 0, ...], a 4×4 matrix, column by column diff --git a/spec/schema/coupling.schema.json b/spec/schema/coupling.schema.json index 58ab619..ac25c10 100644 --- a/spec/schema/coupling.schema.json +++ b/spec/schema/coupling.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/openalea/fspm-TF/spec/schema/coupling.schema.json", "title": "fspm-TF coupling messages", - "description": "The JSON header of a coupling session's message (specification §11, draft for 0.3). Turns and updates also carry a payload: a partial frame's GLB bytes.", + "description": "The JSON header of a coupling session's message (specification §11). Turns and updates also carry a payload: a partial frame's GLB bytes.", "oneOf": [ {"$ref": "#/$defs/hello"}, {"$ref": "#/$defs/welcome"}, {"$ref": "#/$defs/turn"}, {"$ref": "#/$defs/update"}, {"$ref": "#/$defs/error"}, {"$ref": "#/$defs/stop"} diff --git a/spec/specification.md b/spec/specification.md index 4898757..c78c8b7 100644 --- a/spec/specification.md +++ b/spec/specification.md @@ -131,6 +131,8 @@ The glTF root's `extensions.FSPM_scene` MUST follow `fspm_scene.schema.json`: - **Property tables.** Every table of §1 is one `EXT_structural_metadata` property table of its own class, and no property table serves two tables. Writers SHOULD name the classes `__
` (mappings: `mapping__`). Readers MUST find tables through the `propertyTable` indices, not through class names. +- **Update.** A frame with an `update` object is a partial frame, a coupling session's message (§11.1); a frame + written as a time step has none. ## 4. Tables From 754f4003eb45b525fc4ce0d792506277ee8958f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 15:56:22 +0100 Subject: [PATCH 24/39] Selection by scale in ParaView: the macros select_at_scale (SCALE, MODE: isolate in the view or a new panel with its own camera) and clear_fspm_tf_selection, the picked cells grown to their entity at the scale with what it contains across layers, hidden layers shown for the selection only; the reader gives each cell its containing entities' ids and each block its data structure and scales; views checked in ParaView 6.2 (pvbatch, check_in_paraview.py --design, manual): no jsonschema needed, block names, cell data, colour map presets, ColorBy without an array, the representation saved, configurations switched; tests --- .../python/src/openalea/fspm_tf/validate.py | 11 +- bindings/python/src/openalea/fspm_tf/views.py | 183 ++++++++- bindings/python/test/test_paraview_reader.py | 24 ++ bindings/python/test/test_views.py | 69 +++- doc/using.md | 31 +- paraview/apply_fspm_tf_views.py | 32 +- paraview/check_in_paraview.py | 365 +++++++++++++----- paraview/clear_fspm_tf_selection.py | 12 + paraview/fspm_tf_reader.py | 39 ++ paraview/fspm_tf_selection.py | 296 ++++++++++++++ paraview/save_fspm_tf_views.py | 8 +- paraview/select_at_scale.py | 25 ++ 12 files changed, 966 insertions(+), 129 deletions(-) create mode 100644 paraview/clear_fspm_tf_selection.py create mode 100644 paraview/fspm_tf_selection.py create mode 100644 paraview/select_at_scale.py diff --git a/bindings/python/src/openalea/fspm_tf/validate.py b/bindings/python/src/openalea/fspm_tf/validate.py index 72d6201..df60931 100644 --- a/bindings/python/src/openalea/fspm_tf/validate.py +++ b/bindings/python/src/openalea/fspm_tf/validate.py @@ -13,7 +13,6 @@ import json import os -import jsonschema import numpy as np from openalea.fspm_tf import schema as schemas, ucum, vocabulary @@ -68,6 +67,12 @@ def __str__(self): # ── One frame ──────────────────────────────────────────────────────────── +def _jsonschema(): + """jsonschema, imported when a file is validated (the reading code, e.g. ParaView's reader, works without it).""" + import jsonschema + return jsonschema + + def validate_bytes(data: bytes, name: str = "") -> Report: report = Report(name) try: @@ -107,7 +112,7 @@ def _check_document(document: dict, binary: bytes, report: Report) -> bool: report.error("extensions", f"no {EXTENSION} extension: not an fspm-TF file") return False scene = extensions[EXTENSION] - for error in jsonschema.Draft202012Validator(schemas.load("fspm_scene")).iter_errors(scene): + for error in _jsonschema().Draft202012Validator(schemas.load("fspm_scene")).iter_errors(scene): report.error(f"{EXTENSION}/{'/'.join(str(part) for part in error.absolute_path)}", error.message) metadata = extensions.get(METADATA) if metadata is None: @@ -506,7 +511,7 @@ def validate_sequence(index_path: str) -> Report: except (OSError, ValueError) as error: report.error("index", f"cannot be read: {error}") return report - for error in jsonschema.Draft202012Validator(schemas.load("index")).iter_errors(index): + for error in _jsonschema().Draft202012Validator(schemas.load("index")).iter_errors(index): report.error(f"index/{'/'.join(str(part) for part in error.absolute_path)}", error.message) if not report.ok: return report diff --git a/bindings/python/src/openalea/fspm_tf/views.py b/bindings/python/src/openalea/fspm_tf/views.py index 9307f0a..b44f6b3 100644 --- a/bindings/python/src/openalea/fspm_tf/views.py +++ b/bindings/python/src/openalea/fspm_tf/views.py @@ -19,13 +19,17 @@ ParaView's macros (paraview/apply_fspm_tf_views.py, save_fspm_tf_views.py) go through display_settings(layer) (a layer to its display options) and layer_from_display(layer, display) (back, to save them), plain dicts both ways. + +Selection by scale (paraview/select_at_scale.py, clear_selection.py): the reader gives every cell the ids of its +entity and of the entities containing it (".id", ancestor_ids), and each block its data structure and +its graph's scales as field data; picked_entities(picked, scale) grows picked cells to their entities at *scale*, +selection_mask(...) gives the cells of any block those entities contain, at any finer scale. """ import copy import json import os import re -import jsonschema import numpy as np from openalea.fspm_tf import schema as schemas @@ -89,6 +93,15 @@ def configuration(views: dict, name: str = None) -> tuple: return name, [filled(layer) for layer in views["configurations"][name]["layers"]] +def other_layer_names(views: dict, name: str) -> list: + """The names of the layers of the other configurations than *name* that it has not: the items "Apply fspm-TF + views" made for them, replaced when *name* is applied.""" + own = {layer["name"] for layer in views["configurations"][name]["layers"]} + others = [layer["name"] for other, content in views["configurations"].items() if other != name + for layer in content["layers"] if layer["name"] not in own] + return list(dict.fromkeys(others)) + + def filled(layer: dict) -> dict: """*layer* with the defaults of what it leaves out (a tube's scale: 1).""" complete = dict(copy.deepcopy(DEFAULTS), **copy.deepcopy(layer)) @@ -104,8 +117,14 @@ def check_views(views: dict, frame=None): *frame* every name a layer gives (data structure, mesh, entity sets, columns) found in it.""" from openalea.fspm_tf.validate import Report report = Report("views") - for error in jsonschema.Draft202012Validator(schemas.load("views")).iter_errors(views): - report.error(f"views/{'/'.join(str(part) for part in error.absolute_path)}", error.message) + try: + import jsonschema + except ImportError: # e.g. in ParaView's Python: the structure the code reads + for where, message in _structure_errors(views): + report.error(where, message) + else: + for error in jsonschema.Draft202012Validator(schemas.load("views")).iter_errors(views): + report.error(f"views/{'/'.join(str(part) for part in error.absolute_path)}", error.message) if not report.ok: return report if "active" in views and views["active"] not in views["configurations"]: @@ -122,6 +141,37 @@ def check_views(views: dict, frame=None): return report +def _structure_errors(views) -> list: + """[(where, message)]: what the schema requires that the reader and the macros rely on, without jsonschema.""" + if not isinstance(views, dict): + return [("views", "not an object")] + errors = [] + if views.get("fspm_tf_views") != FORMAT: + errors.append(("views/fspm_tf_views", f"must be {FORMAT}")) + configurations = views.get("configurations") + if not isinstance(configurations, dict) or not configurations: + return errors + [("views/configurations", "an object of at least one configuration")] + for name, content in configurations.items(): + layers = content.get("layers") if isinstance(content, dict) else None + if not isinstance(layers, list): + errors.append((f"views/configurations/{name}", "'layers' is a required list")) + continue + for number, layer in enumerate(layers): + where = f"views/configurations/{name}/layers/{number}" + source = layer.get("source") if isinstance(layer, dict) else None + if not isinstance(layer, dict) or not isinstance(layer.get("name"), str) or not layer["name"]: + errors.append((where, "'name' is a required text")) + if not isinstance(source, dict) or not isinstance(source.get("data_structure"), str): + errors.append((where, "'source' with its 'data_structure' is required")) + elif source.get("kind") not in ("mesh", "segments", "grid"): + errors.append((f"{where}/source", "'kind' is one of mesh, segments, grid")) + elif source["kind"] in ("mesh", "segments") and not isinstance( + source.get("mesh" if source["kind"] == "mesh" else "entity_set"), str): + errors.append((f"{where}/source", f"a {source['kind']} layer names its " + f"{'mesh' if source['kind'] == 'mesh' else 'entity_set'}")) + return errors + + def segment_columns(entity_set) -> list: """The six segment-end columns of *entity_set* (by standard name, else x1 ... z2), or [] when it has none.""" by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() @@ -220,16 +270,41 @@ def selected(table, where: dict) -> np.ndarray: # ── Display options (ParaView's macros) ────────────────────────────────── +def property_text(value) -> str: + """A ParaView string property's value as text: ParaView gives a str, a property (its GetData(): a str, or a list + of one value), or a list-like of one value (empty: "").""" + if hasattr(value, "GetData"): + value = value.GetData() + if value is None or isinstance(value, str): + return value or "" + values = list(value) + return str(values[0]) if values else "" + + def block_selector(name: str) -> str: """The selector of the reader's block *name* in ParaView's ExtractBlock: /Root/ and the name as ParaView makes it a - valid node name (vtkDataAssembly::MakeValidNodeName: characters other than letters, digits, '_', '-' and '.' - replaced by '_', and '_' before a first character that is not a letter or '_').""" - valid = re.sub(r"[^A-Za-z0-9_.\-]", "_", name) - if not re.match(r"[A-Za-z_]", valid): - valid = "_" + valid + valid node name: VTK's vtkDataAssembly.MakeValidNodeName when VTK is there (its rule changed between versions), + else ParaView 6's (the characters other than letters, digits, '_', '-' and '.' removed, and '_' before a first + character that is not a letter or '_').""" + try: + from vtkmodules.vtkCommonDataModel import vtkDataAssembly + except ImportError: + valid = re.sub(r"[^A-Za-z0-9_.\-]", "", name) + if not re.match(r"[A-Za-z_]", valid): + valid = "_" + valid + else: + valid = vtkDataAssembly.MakeValidNodeName(name) return f"/Root/{valid}" +def preset_names(colormap: str) -> list: + """The ParaView preset names to try for *colormap*: itself, then its name in other ParaView versions (ParaView 6 + dropped the " (matplotlib)" of 5's "Viridis (matplotlib)", "Inferno (matplotlib)" ...).""" + suffix = " (matplotlib)" + other = colormap[:-len(suffix)] if colormap.endswith(suffix) else colormap + suffix + return [colormap, other] + + def colour_array(layer: dict, arrays=()) -> str: """The cell data array a layer is coloured by: ".", or its codes ". _code" when *arrays* (the block's arrays) hold those instead (a text variable); None for a solid colour.""" @@ -290,3 +365,95 @@ def save_configuration(views: dict, name: str, layers: list, active: bool = True if active: saved["active"] = name return saved + + +# ── Selection by scale (ParaView's "Select at scale") ───────────────────── + +DATA_STRUCTURE_FIELD = "fspm_tf_data_structure" # field data of each reader block: its data structure's id +SCALES_FIELD = "fspm_tf_scales" # and its graph's ranked entity sets, coarser first + + +def ranked_sets(data_structure) -> list: + """The names of a graph's ranked entity sets (its scales), coarser first; [] for a grid.""" + ranked = [entity_set for entity_set in data_structure.entity_sets if entity_set.rank is not None] + return [entity_set.name for entity_set in sorted(ranked, key=lambda entity_set: entity_set.rank)] + + +def ancestor_ids(data_structure, entity_set: str, rows: np.ndarray) -> dict: + """{entity set: ids per row (NO_ID: none)} of the entities containing the entities at *rows* of *entity_set*, at + each coarser scale their complex chain reaches.""" + from openalea.fspm_tf.model import NO_ID + rows = np.asarray(rows, dtype=np.int64) + sets = [data_structure.entity_set(name) for name in ranked_sets(data_structure)] + if not sets or data_structure.entity_set(entity_set).rank is None: + return {} + ids = np.concatenate([np.asarray(table.columns["id"], dtype=np.int64) for table in sets]) + complexes = np.concatenate([np.asarray(table.columns["complex"], dtype=np.int64) if "complex" in table.columns + else np.full(table.count, NO_ID, dtype=np.int64) for table in sets]) + owner = np.concatenate([np.full(table.count, number) for number, table in enumerate(sets)]) + order = np.argsort(ids, kind="stable") + ids, complexes, owner = ids[order], complexes[order], owner[order] + start = data_structure.entity_set(entity_set) + valid = rows >= 0 + current = np.where(valid, np.asarray(start.columns.get("complex", np.full(start.count, NO_ID)), + dtype=np.int64)[np.where(valid, rows, 0)], NO_ID) + found = {} + for _ in range(len(sets)): + if not (current != NO_ID).any() or not ids.size: + break + position = np.minimum(np.searchsorted(ids, current), ids.size - 1) + known = (current != NO_ID) & (ids[position] == current) + for number in np.unique(owner[position[known]]).tolist(): + here = known & (owner[position] == number) + name = sets[number].name + found.setdefault(name, np.full(rows.size, NO_ID, dtype=np.int64))[here] = current[here] + current = np.where(known, complexes[position], NO_ID) + return found + + +def id_array(scale: str) -> str: + """The reader's cell array of the ids at *scale*: ".id".""" + return f"{scale}.id" + + +def _ids(values) -> np.ndarray: + """Ids read back from a cell array (float64, NaN where no entity): int64, -1 where none.""" + values = np.asarray(values, dtype=np.float64).reshape(-1) + return np.where(np.isfinite(values), np.nan_to_num(values), -1).astype(np.int64) + + +def picked_entities(picked: list, scale: str) -> dict: + """{data structure id: sorted ids} of the entities at *scale* containing picked cells; *picked*: [(data structure + id, {cell array: values of the picked cells})], e.g. one entry per block of ParaView's extracted selection. A + ValueError when no picked cell has an entity at *scale* (a scale finer than the picked elements, or none + picked).""" + found = {} + for data_structure, arrays in picked: + if id_array(scale) not in arrays: + continue + ids = _ids(arrays[id_array(scale)]) + ids = ids[ids >= 0] + if ids.size: + found[data_structure] = sorted(set(found.get(data_structure, [])) | set(ids.tolist())) + if not found: + scales = sorted({name[:-3] for _, arrays in picked for name in arrays if name.endswith(".id")}) + raise ValueError(f"no picked cell has an entity at scale '{scale}'" + + (f" (they have {scales})" if scales else " (pick cells in the view first)")) + return found + + +def selection_mask(data_structure: str, arrays: dict, chosen: dict, scale: str) -> np.ndarray: + """The cells of a block (of *data_structure*, its cell arrays *arrays*) inside the entities *chosen* + ({data structure id: ids at *scale*}, picked_entities'): those whose ".id" is one of them, whatever + finer scale the block's entities are at; none in a block without that array (a grid, a coarser layer).""" + name = id_array(scale) + count = len(next(iter(arrays.values()))) if arrays else 0 + if name not in arrays or data_structure not in chosen: + return np.zeros(count, dtype=bool) + return np.isin(_ids(arrays[name]), np.asarray(chosen[data_structure], dtype=np.int64)) + + +def isolated(items: dict, selection: list) -> list: + """The items to hide to isolate a selection: of *items* ({name: visible}), the visible ones not in *selection* + (restored by clearing it).""" + return [name for name, visible in items.items() if visible and name not in selection] diff --git a/bindings/python/test/test_paraview_reader.py b/bindings/python/test/test_paraview_reader.py index 93373f1..8eff571 100644 --- a/bindings/python/test/test_paraview_reader.py +++ b/bindings/python/test/test_paraview_reader.py @@ -189,3 +189,27 @@ def test_views_from_a_file_and_wrong_names(tmp_path): with pytest.raises(ValueError, match="fspm_tf_views"): reader.frame_to_vtk(generate.root_system(), views={"fspm_tf_views": 2, "configurations": {}}) assert reader.frame_to_vtk(generate.root_system()).GetNumberOfBlocks() == 1 # no views: as before + + +def test_cells_carry_the_ids_of_the_entities_containing_theirs(): + """What selection by scale reads: each cell the ids of its entity's containing ones, each block its data + structure and scales.""" + from openalea.fspm_tf import views + frame = generate.root_system() + blocks = blocks_of(frame) + lines = blocks[("root_system", "SubOrgan segments")] + segments = frame.data_structures[0].entity_set("SubOrgan") + np.testing.assert_array_equal(cell_array(lines, "Axis.id"), segments.columns["complex"]) + assert set(cell_array(lines, "Plant.id")) == {1000.} + fields = lines.GetFieldData() + assert fields.GetAbstractArray(views.DATA_STRUCTURE_FIELD).GetValue(0) == "root_system" + scales = fields.GetAbstractArray(views.SCALES_FIELD) + assert [scales.GetValue(number) for number in range(scales.GetNumberOfValues())] == ["Plant", "Axis", "SubOrgan"] + mesh = blocks[("root_system", "mesh roots")] + arrays = {name: cell_array(mesh, name) for name in ("Plant.id", "Axis.id", "SubOrgan.id")} + chosen = views.picked_entities([("root_system", {"Axis.id": np.array([101.])})], "Axis") + lateral = views.selection_mask("root_system", arrays, chosen, "Axis") + assert 0 < lateral.sum() < mesh.GetNumberOfCells() + assert set(arrays["SubOrgan.id"][lateral]) == {60., 61.} # the lateral's segments, not its apex + soil = blocks_of(generate.soil_and_roots())[("soil", "grid")] + assert soil.GetFieldData().GetAbstractArray(views.DATA_STRUCTURE_FIELD).GetValue(0) == "soil" diff --git a/bindings/python/test/test_views.py b/bindings/python/test/test_views.py index 6a17103..512f5b4 100644 --- a/bindings/python/test/test_views.py +++ b/bindings/python/test/test_views.py @@ -108,11 +108,11 @@ def test_configurations_and_their_defaults(tmp_path): def test_display_settings_and_back(): _, layers = views.configuration(roots_views()) mesh, segments = (views.display_settings(layer) for layer in layers) - assert mesh["selector"] == "/Root/first_order" and mesh["tube"] is None and mesh["range"] is None + assert mesh["selector"] == "/Root/firstorder" and mesh["tube"] is None and mesh["range"] is None assert mesh["color_array"] == "SubOrgan.water_potential" and mesh["visible"] assert segments["tube"] == {"radius_array": views.TUBE_RADIUS} and not segments["visible"] assert views.display_settings(layers[1], {"SubOrgan.label_code"})["color_array"] == "SubOrgan.label_code" - assert views.block_selector("2 roots/p0") == "/Root/_2_roots_p0" + assert views.block_selector("2 roots/p0") == "/Root/_2rootsp0" # as ParaView names blocks unchanged = views.layer_from_display(layers[0], {}) assert unchanged == layers[0] saved = views.layer_from_display(layers[1], {"visible": True, "opacity": 0.5, "representation": "Wireframe", @@ -144,17 +144,9 @@ def UpdatePipeline(self, *arguments): def UpdatePropertyInformation(self): pass - def GetDataInformation(self): - names = ["SubOrgan.water_potential", "SubOrgan.label_code"] - - class Arrays: - def GetNumberOfArrays(self): - return len(names) - - def GetArrayInformation(self, number): - return types.SimpleNamespace(GetName=lambda: names[number]) - - return types.SimpleNamespace(GetCellDataInformation=lambda: Arrays()) + @property + def CellData(self): + return types.SimpleNamespace(keys=lambda: ["SubOrgan.water_potential", "SubOrgan.label_code"]) class _Display: @@ -249,3 +241,54 @@ def test_the_macros_apply_and_save_views(tmp_path, monkeypatch): assert first["opacity"] == 0.25 and roots["visible"] and roots["range"] == [0., 3.] assert roots["color_by"] == {"entity_set": "SubOrgan", "variable": "label"} and roots["tube"]["scale"] == 2 assert saved["configurations"]["plain"] == roots_views()["configurations"]["plain"] + + +# ── Selection by scale ─────────────────────────────────────────────────── + +def test_the_entities_containing_others_at_every_coarser_scale(): + frame = generate.root_system() + source = frame.data_structures[0] + assert views.ranked_sets(source) == ["Plant", "Axis", "SubOrgan"] + segments = source.entity_set("SubOrgan") + rows = np.array([0, segments.count - 1, -1]) # a tap root segment, the lateral's apex, none + found = views.ancestor_ids(source, "SubOrgan", rows) + np.testing.assert_array_equal(found["Axis"], [100, 101, -1]) + np.testing.assert_array_equal(found["Plant"], [1000, 1000, -1]) + assert views.ancestor_ids(source, "Plant", np.array([0])) == {} + assert views.ancestor_ids(generate.soil_and_roots().data_structures[1], "Voxel", np.array([0])) == {} + + +def test_picked_cells_grow_to_their_entity_and_what_it_contains(): + picked = [("plants", {"Axis.id": np.array([101., 101.]), "SubOrgan.id": np.array([60., 61.])}), + ("soil", {"Voxel.id": np.array([3.])})] + assert views.picked_entities(picked, "Axis") == {"plants": [101]} + assert views.picked_entities(picked, "SubOrgan") == {"plants": [60, 61]} + with pytest.raises(ValueError, match="'Compartment'.*Axis"): + views.picked_entities(picked, "Compartment") + with pytest.raises(ValueError, match="pick cells"): + views.picked_entities([], "Axis") + chosen = {"plants": [101]} + sections = {"Axis.id": np.array([100., 101., np.nan]), "Compartment.id": np.array([1., 2., 3.])} + np.testing.assert_array_equal(views.selection_mask("plants", sections, chosen, "Axis"), [False, True, False]) + np.testing.assert_array_equal(views.selection_mask("other", sections, chosen, "Axis"), [False] * 3) + np.testing.assert_array_equal(views.selection_mask("plants", {"Plant.id": np.ones(2)}, chosen, "Axis"), + [False] * 2) # a block without the scale: none of it + assert views.isolated({"roots": True, "shoot": True, "sections": False, "roots selection": True}, + ["roots selection"]) == ["roots", "shoot"] + + +def test_views_are_checked_without_jsonschema(monkeypatch): + import builtins + original = builtins.__import__ + + def without_jsonschema(name, *arguments, **keywords): + if name == "jsonschema": + raise ImportError(name) + return original(name, *arguments, **keywords) + + monkeypatch.setattr(builtins, "__import__", without_jsonschema) + assert views.check_views(roots_views()).ok + broken = copy.deepcopy(roots_views()) + del broken["configurations"]["plain"]["layers"][0]["source"]["mesh"] + assert "names its mesh" in str(views.check_views(broken)) + assert not views.check_views({"fspm_tf_views": 2, "configurations": {}}).ok diff --git a/doc/using.md b/doc/using.md index 0ef11ea..0dc26cb 100644 --- a/doc/using.md +++ b/doc/using.md @@ -18,7 +18,7 @@ then `pip install -e path/to/fspm-TF`. ## In ParaView fspm-TF comes with a ParaView reader plugin: `paraview/fspm_tf_reader.py`. It works with ParaView 5.10 and later, -and has been checked with 6.0.1. +and has been checked with 6.0.1 and 6.2.0. 1. In ParaView, open **Tools → Manage Plugins → Load New…**, choose `fspm_tf_reader.py`, and tick **Auto Load** to keep it loaded. @@ -36,6 +36,8 @@ What you see: at a scale, the value is empty (NaN). - **Labels and other text become integer codes**, `.label_code`; the names behind the codes are in the block's field data. +- **Every cell knows its plant, axis, organ ...**: besides its own entity's `.id`, a cell has the ids of the + entities containing it (`Axis.id`, `Plant.id` on a root segment), which selection by scale uses (below). - **Show Settings**, a reader option, adds the columns a platform keeps for its own bookkeeping, and the tables' links (`complex`, `parent`, `edge_type`). They are hidden by default. @@ -98,6 +100,33 @@ report = views.check_views(document, fspm_tf.read("outputs/frames/frame_000000.g print(report) ``` +### Selecting a plant, an axis, an organ + +Two more macros in `paraview/` select by scale: pick an element, and see its whole plant, axis, organ or segment, +with everything it contains at finer scales, in every layer, hidden ones included (a segment's section drawings, a +hidden layer by default, show for the selection only). + +1. Load them once like the others: **Macros → Import new macro…**, `paraview/select_at_scale.py`, then + `paraview/clear_fspm_tf_selection.py`. At the top of `select_at_scale` (**Macros → Edit…**), set `SCALE` (an + entity set of the frames: `Plant`, `Axis`, `Organ`, `SubOrgan`, … `Compartment`) and `MODE`. +2. Pick cells in the view (**Select Cells On**, or `s`), on any layer: a leaf, a root segment. +3. Run **select_at_scale**. With `MODE = "isolate"`, the view shows only the selection, the other items hidden; + with `MODE = "panel"`, a new view opens beside it with the selection alone, its camera fit to it, the first view + untouched. Each layer's part is an item named ` selection`, drawn as its layer (the same colour map). + Picking again on the selection and running the macro again selects anew (e.g. an axis of the plant shown). +4. Run **clear_fspm_tf_selection**: the selection's items and panel are removed, the hidden items shown again. + +A scale finer than the picked element (a Compartment of a leaf drawn without sections) is reported with the scales +the picked cells have. The macros need the plugin loaded (it puts their code, `paraview/fspm_tf_selection.py`, on +ParaView's path); which cells are selected is `openalea.fspm_tf.views.picked_entities` and `selection_mask`, tested +without ParaView. + +`paraview/check_in_paraview.py` checks all this inside ParaView. `pvpython paraview/check_in_paraview.py +conformance` runs on the conformance files (the binding tests run it when `FSPM_TF_PVPYTHON` names a `pvpython`); +`pvbatch paraview/check_in_paraview.py --design ` is a check by hand, on a sequence with +its `views.json`: it applies and saves the views, selects at several scales, isolated and in a panel, clears, and +saves a screenshot of each step in the folder, to look at. + ## In other viewers Any glTF viewer (Blender, a web viewer, ParaView without the plugin) shows the geometry, and ignores the tables. diff --git a/paraview/apply_fspm_tf_views.py b/paraview/apply_fspm_tf_views.py index 5a2cc91..a3972eb 100644 --- a/paraview/apply_fspm_tf_views.py +++ b/paraview/apply_fspm_tf_views.py @@ -2,7 +2,8 @@ ParaView macro "Apply fspm-TF views": each layer of the selected fspm-TF reader's configuration (its "Views" file and "Configuration", see fspm_tf_reader.py) becomes its own pipeline item, named by the layer: an ExtractBlock of the layer's block, a Tube on "tube_radius" for a segments layer, shown with its colour variable, colour map, range, log -scale, solid colour, opacity, representation and visibility. Applied again, it replaces the items it made. +scale, solid colour, opacity, representation and visibility. Applied again, it replaces the items it made, those of +the views' other configurations included (the reader's Configuration changed). Macros > Import new macro..., choose this file; select the fspm-TF reader in the Pipeline Browser, then Macros > apply_fspm_tf_views. @@ -28,18 +29,27 @@ def reader_views(source): views = _views() if source is None or not hasattr(source, "Configuration"): raise RuntimeError("select an fspm-TF reader in the Pipeline Browser first") - file_name = source.FileName if isinstance(source.FileName, str) else source.FileName[0] - path = views.views_path(file_name, source.Views) + file_name = views.property_text(source.FileName) + path = views.views_path(file_name, views.property_text(source.Views) or None) if path is None: raise RuntimeError(f"no views file for {file_name} (a views.json next to it, or the reader's Views)") document = views.read_views(path) - name, layers = views.configuration(document, source.Configuration or None) + name, layers = views.configuration(document, views.property_text(source.Configuration) or None) return views, path, document, name, layers def _cell_arrays(item) -> set: - information = item.GetDataInformation().GetCellDataInformation() - return {information.GetArrayInformation(number).GetName() for number in range(information.GetNumberOfArrays())} + return set(item.CellData.keys()) + + +def _apply_preset(colours, colormap: str, views) -> None: + for name in views.preset_names(colormap): + try: + colours.ApplyPreset(name, True) + return + except RuntimeError: # not a preset of this ParaView + continue + print(f"fspm-TF views: no colour map preset '{colormap}' in this ParaView, its default kept") def _remove(name: str) -> None: @@ -67,7 +77,7 @@ def show_layer(source, layer: dict, view, views) -> None: if settings["color_array"]: simple.ColorBy(display, ("CELLS", settings["color_array"])) colours = simple.GetColorTransferFunction(settings["color_array"]) - colours.ApplyPreset(settings["colormap"], True) + _apply_preset(colours, settings["colormap"], views) if settings["range"] is None: colours.AutomaticRescaleRangeMode = "Grow and update on 'Apply'" display.RescaleTransferFunctionToDataRange(True, False) @@ -77,7 +87,7 @@ def show_layer(source, layer: dict, view, views) -> None: colours.UseLogScale = int(settings["log"]) display.SetScalarBarVisibility(view, settings["visible"]) else: - simple.ColorBy(display, None) + display.ColorArrayName = ["POINTS", ""] # a solid colour (ColorBy(display, None) fails in 6.2) display.AmbientColor = display.DiffuseColor = settings["solid_color"] display.Opacity = settings["opacity"] if not settings["visible"]: @@ -86,12 +96,16 @@ def show_layer(source, layer: dict, view, views) -> None: def apply_views(source=None, view=None) -> None: source = source or simple.GetActiveSource() - views, _, _, _, layers = reader_views(source) + views, _, document, name, layers = reader_views(source) view = view or simple.GetActiveViewOrCreate("RenderView") source.UpdatePipeline() simple.Hide(source, view) + for other in views.other_layer_names(document, name): # another configuration's items, applied before + _remove(other) for layer in layers: show_layer(source, layer, view, views) + if hasattr(simple, "HideUnusedScalarBars"): + simple.HideUnusedScalarBars(view) simple.SetActiveSource(source) simple.Render(view) diff --git a/paraview/check_in_paraview.py b/paraview/check_in_paraview.py index e6d09b5..a3f4de6 100644 --- a/paraview/check_in_paraview.py +++ b/paraview/check_in_paraview.py @@ -1,38 +1,55 @@ """ The reader inside ParaView: loads the plugin, opens conformance files and a sequence, checks their blocks, arrays and time steps, and renders a picture of each; then a frame with views (one block per layer, the configurations' -drop-down) and the macros applying and saving them. +drop-down), the macros applying and saving them, and selection by scale (isolated, then cleared). pvpython paraview/check_in_paraview.py conformance [output folder] + +A design check, by hand (no test runs it): a sequence with its views.json (e.g. metafspm's +SceneRecorder(frame_views="default", display_period=...) frames of plants with shoots, roots and sections, and a soil +grid) opened, each configuration applied, saved back unchanged and changed (the round trip), then selections at each +scale, isolated and in a panel, and cleared; a screenshot of each step in the output folder, to look at (colours, +opacities, tubes, hidden layers, isolation). The views file is put back as it was. + + pvbatch paraview/check_in_paraview.py --design outputs/frames/scene.fspm.json screenshots """ import os +import runpy import sys -from paraview import simple +from paraview import servermanager, simple HERE = os.path.dirname(os.path.abspath(__file__)) -conformance = sys.argv[1] -output = sys.argv[2] if len(sys.argv) > 2 else None +DESIGN = len(sys.argv) > 1 and sys.argv[1] == "--design" +conformance = sys.argv[1] if not DESIGN else None +pictures = sys.argv[2 + DESIGN] if len(sys.argv) > 2 + DESIGN else None +# (not "output": a Programmable Filter removes that name from the main module once it has run) simple.LoadPlugin(os.path.join(HERE, "fspm_tf_reader.py"), ns=globals()) def arrays(source) -> set: - information = source.GetDataInformation().DataInformation - names = set() - for association in (0, 1): # points, cells - field = source.GetDataInformation().GetPointDataInformation() if association == 0 \ - else source.GetDataInformation().GetCellDataInformation() - for number in range(field.GetNumberOfArrays()): - names.add(field.GetArrayInformation(number).GetName()) - del information - return names + return set(source.PointData.keys()) | set(source.CellData.keys()) + + +def configuration_names(reader) -> list: + reader.UpdatePropertyInformation() + return list(reader.GetProperty("ConfigurationNames")) + + +def blocks(source) -> int: + return servermanager.Fetch(source).GetNumberOfBlocks() + + +def run_macro(name: str, source) -> None: + simple.SetActiveSource(source) + runpy.run_path(os.path.join(HERE, name)) def picture(source, name, colour_by, side=False): - if output is None: + if pictures is None: return - os.makedirs(output, exist_ok=True) + os.makedirs(pictures, exist_ok=True) view = simple.CreateView("RenderView") view.ViewSize = [800, 600] display = simple.Show(source, view) @@ -41,83 +58,249 @@ def picture(source, name, colour_by, side=False): if side: # seen from the side: z up view.CameraPosition, view.CameraFocalPoint, view.CameraViewUp = [0., -1., 0.], [0., 0., 0.], [0., 0., 1.] view.ResetCamera() - simple.SaveScreenshot(os.path.join(output, f"{name}.png"), view) + simple.SaveScreenshot(os.path.join(pictures, f"{name}.png"), view) simple.Delete(view) -roots = fspmTFreader(FileName=os.path.join(conformance, "soil_and_roots.glb")) # noqa: F821 -roots.UpdatePipeline() -found = arrays(roots) -for name in ("SubOrgan.water_potential", "Axis.branching_order", "Voxel.soil_water_potential"): - assert name in found, (name, sorted(found)) -roots_only = fspmTFreader(FileName=os.path.join(conformance, "root_system.glb")) # noqa: F821 -picture(roots_only, "root_system", "SubOrgan.water_potential", side=True) - -shoot = fspmTFreader(FileName=os.path.join(conformance, "wheat_shoot.glb")) # noqa: F821 -shoot.UpdatePipeline() -assert "SubOrgan.PARa" in arrays(shoot) -picture(shoot, "wheat_shoot", "SubOrgan.PARa") - -sequence = fspmTFreader(FileName=os.path.join(conformance, "growing_sequence", "scene.fspm.json")) # noqa: F821 -sequence.UpdatePipelineInformation() -assert list(sequence.TimestepValues) == [0., 1., 2.], list(sequence.TimestepValues) -cells = [] -for time in sequence.TimestepValues: - sequence.UpdatePipeline(time) - cells.append(sequence.GetDataInformation().GetNumberOfCells()) -assert cells[0] < cells[1] < cells[2], cells # the root system grows -print("cells per time step", cells) - -# Without the plugin: ParaView's own glTF reader shows the geometry and the display copies (float32 point data) -sys.path.insert(0, os.path.join(os.path.dirname(HERE), "bindings", "python", "src")) -import tempfile # noqa: E402 - -from openalea import fspm_tf # noqa: E402 -from openalea.fspm_tf.display import add_display # noqa: E402 - -with tempfile.TemporaryDirectory() as directory: - copied = os.path.join(directory, "roots_display.glb") - fspm_tf.write(copied, add_display(fspm_tf.read(os.path.join(conformance, "root_system.glb")), ["water_potential"])) - plain = simple.glTFReader(FileName=copied) # ParaView's own reader, not the plugin - plain.UpdatePipeline() - names = set() - point_data = plain.GetDataInformation().GetPointDataInformation() - for number in range(point_data.GetNumberOfArrays()): - names.add(point_data.GetArrayInformation(number).GetName()) - print("glTF reader", type(plain).__name__, "point data", sorted(names)) - assert any("WATER_POTENTIAL" in name for name in names), sorted(names) - -# Views: a views.json next to the frame gives one block per layer; the macros make and save the layers' items -import runpy # noqa: E402 -import shutil # noqa: E402 - -from openalea.fspm_tf import views # noqa: E402 - -with tempfile.TemporaryDirectory() as directory: - shutil.copy(os.path.join(conformance, "root_system.glb"), directory) - layers = [{"name": "roots", - "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, - "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "tube": {"radius": "radius"}}, - {"name": "mesh", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, - "color_by": None, "opacity": 0.3, "visible": False}] - path = os.path.join(directory, views.VIEWS_FILE) - views.write_views(path, {"fspm_tf_views": 1, "active": "default", - "configurations": {"default": {"layers": layers}, "mesh only": {"layers": layers[1:]}}}) - viewed = fspmTFreader(FileName=os.path.join(directory, "root_system.glb")) # noqa: F821 - viewed.UpdatePropertyInformation() - assert list(viewed.ConfigurationNames) == ["default", "mesh only"], list(viewed.ConfigurationNames) - viewed.UpdatePipeline() - assert viewed.GetDataInformation().DataInformation.GetNumberOfLeaves() == 2 - simple.SetActiveSource(viewed) - runpy.run_path(os.path.join(HERE, "apply_fspm_tf_views.py")) - tube = simple.FindSource("roots") - assert tube is not None and simple.FindSource("mesh") is not None - simple.GetDisplayProperties(tube).Opacity = 0.5 - simple.SetActiveSource(viewed) - runpy.run_path(os.path.join(HERE, "save_fspm_tf_views.py")) - assert views.read_views(path)["configurations"]["default"]["layers"][0]["opacity"] == 0.5 - picture(tube, "root_system_views", "SubOrgan.water_potential", side=True) - viewed.Configuration = "mesh only" - viewed.UpdatePipeline() - assert viewed.GetDataInformation().DataInformation.GetNumberOfLeaves() == 1 +def check_conformance() -> None: + roots = fspmTFreader(FileName=os.path.join(conformance, "soil_and_roots.glb")) # noqa: F821 + roots.UpdatePipeline() + found = arrays(roots) + for name in ("SubOrgan.water_potential", "Axis.branching_order", "Voxel.soil_water_potential"): + assert name in found, (name, sorted(found)) + roots_only = fspmTFreader(FileName=os.path.join(conformance, "root_system.glb")) # noqa: F821 + picture(roots_only, "root_system", "SubOrgan.water_potential", side=True) + + shoot = fspmTFreader(FileName=os.path.join(conformance, "wheat_shoot.glb")) # noqa: F821 + shoot.UpdatePipeline() + assert "SubOrgan.PARa" in arrays(shoot) + picture(shoot, "wheat_shoot", "SubOrgan.PARa") + + sequence = fspmTFreader(FileName=os.path.join(conformance, "growing_sequence", "scene.fspm.json")) # noqa: F821 + sequence.UpdatePipelineInformation() + assert list(sequence.TimestepValues) == [0., 1., 2.], list(sequence.TimestepValues) + cells = [] + for time in sequence.TimestepValues: + sequence.UpdatePipeline(time) + cells.append(sequence.GetDataInformation().GetNumberOfCells()) + assert cells[0] < cells[1] < cells[2], cells # the root system grows + print("cells per time step", cells) + + # Without the plugin: ParaView's own glTF reader shows the geometry and the display copies (float32 point data) + import tempfile + + from openalea import fspm_tf + from openalea.fspm_tf.display import add_display + + with tempfile.TemporaryDirectory() as directory: + copied = os.path.join(directory, "roots_display.glb") + fspm_tf.write(copied, add_display(fspm_tf.read(os.path.join(conformance, "root_system.glb")), + ["water_potential"])) + plain = simple.glTFReader(FileName=copied) # ParaView's own reader, not the plugin + plain.UpdatePipeline() + names = set(plain.PointData.keys()) + print("glTF reader", type(plain).__name__, "point data", sorted(names)) + assert any("WATER_POTENTIAL" in name for name in names), sorted(names) + + # Views: a views.json next to the frame gives one block per layer; the macros make and save the layers' items; + # a picked segment grows to its axis, isolated, then cleared + import shutil + + from openalea.fspm_tf import views + + with tempfile.TemporaryDirectory() as directory: + shutil.copy(os.path.join(conformance, "root_system.glb"), directory) + layers = [{"name": "roots", + "source": {"data_structure": "root_system", "kind": "segments", "entity_set": "SubOrgan"}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_potential"}, "tube": {"radius": "radius"}}, + {"name": "mesh", "source": {"data_structure": "root_system", "kind": "mesh", "mesh": "roots"}, + "color_by": None, "opacity": 0.3, "visible": False}] + path = os.path.join(directory, views.VIEWS_FILE) + views.write_views(path, {"fspm_tf_views": 1, "active": "default", + "configurations": {"default": {"layers": layers}, + "mesh only": {"layers": layers[1:]}}}) + viewed = fspmTFreader(FileName=os.path.join(directory, "root_system.glb")) # noqa: F821 + assert configuration_names(viewed) == ["default", "mesh only"], configuration_names(viewed) + viewed.UpdatePipeline() + assert blocks(viewed) == 2 + simple.GetActiveViewOrCreate("RenderView") + run_macro("apply_fspm_tf_views.py", viewed) + tube = simple.FindSource("roots") + assert tube is not None and simple.FindSource("mesh") is not None + simple.GetDisplayProperties(tube).Opacity = 0.5 + run_macro("save_fspm_tf_views.py", viewed) + assert views.read_views(path)["configurations"]["default"]["layers"][0]["opacity"] == 0.5 + picture(tube, "root_system_views", "SubOrgan.water_potential", side=True) + + import fspm_tf_selection + lateral = pick(tube, "Axis.id", 101.) + made = fspm_tf_selection.select_at_scale("Axis", "isolate") + assert sorted(made) == ["mesh selection", "roots selection"], made + assert set(servermanager.Fetch(simple.FindSource("roots selection")).GetBlock(0).GetCellData() + .GetArray("Axis.id").GetRange()) == {101.}, lateral + assert not simple.GetDisplayProperties(tube).Visibility + fspm_tf_selection.clear_selection() + assert simple.GetDisplayProperties(tube).Visibility and simple.FindSource("roots selection") is None + viewed.Configuration = "mesh only" + viewed.UpdatePipeline() + assert blocks(viewed) == 1 + + +def pick(item, array: str, value: float) -> int: + """Pick (select) the first cell of *item* whose *array* is *value*, as a click would; its index.""" + import numpy as np + from vtkmodules.util.numpy_support import vtk_to_numpy + data = servermanager.Fetch(item) + leaf = data.GetBlock(0) if hasattr(data, "GetNumberOfBlocks") else data + cell = int(np.flatnonzero(vtk_to_numpy(leaf.GetCellData().GetArray(array)) == value)[0]) + simple.SelectIDs(IDs=[0, cell], FieldType="CELL", Source=item) + return cell + + +# ── The design check, by hand ──────────────────────────────────────────────── + +def screenshot(view, name: str) -> None: + os.makedirs(pictures, exist_ok=True) + path = os.path.join(pictures, f"{name}.png") + simple.SaveScreenshot(path, view, ImageResolution=[900, 700]) + print("screenshot", path) + + +def side_view(view) -> None: + """Seen from the side and a little above: z up.""" + view.CameraPosition, view.CameraFocalPoint, view.CameraViewUp = [0.3, -1., 0.35], [0., 0., 0.], [0., 0., 1.] + view.ResetCamera() + + +def displayed(view, name: str) -> dict: + display = simple.GetDisplayProperties(simple.FindSource(name), view) + return {"visible": bool(display.Visibility), "opacity": float(display.Opacity), + "color_array": display.ColorArrayName[1] or None, + "representation": display.GetProperty("Representation").GetData()} + + +def check_design(index_path: str) -> None: + """Views and selection by scale on a sequence with its views.json, screenshots in *pictures*.""" + import json + import shutil + + from openalea.fspm_tf import views + import fspm_tf_selection + + views_file = os.path.join(os.path.dirname(os.path.abspath(index_path)), views.VIEWS_FILE) + shutil.copy(views_file, views_file + ".before_check") + try: + document = views.read_views(views_file) + name, layers = views.configuration(document) + roots_only = [dict(layer, visible=True) for layer in layers if "roots" in layer["name"] + or "Section" in layer["name"]] # a second configuration: the sections shown, the + for layer in roots_only: # roots by radius, a range, a log scale, a colour map + if layer.get("tube"): + layer.update(color_by={"entity_set": "SubOrgan", "variable": "radius"}, range=[0.002, 0.005], + log=True, colormap="Inferno (matplotlib)") + views.write_views(views_file, views.save_configuration(document, "roots and sections", roots_only, + active=False)) + reader = fspmTFreader(FileName=os.path.abspath(index_path)) # noqa: F821 + names = configuration_names(reader) + print("configurations", names) + assert names == [name, "roots and sections"], names + domain = reader.GetProperty("Configuration").SMProperty.GetDomain("list") + assert [domain.GetString(number) for number in range(domain.GetNumberOfStrings())] == names + reader.UpdatePipelineInformation() + last = list(reader.TimestepValues)[-1] if reader.TimestepValues else None + view = simple.GetActiveViewOrCreate("RenderView") + view.ViewSize = [900, 700] + view.ViewTime = last if last is not None else 0. + reader.UpdatePipeline(last) + assert blocks(reader) == len(layers) + + run_macro("apply_fspm_tf_views.py", reader) + side_view(view) + screenshot(view, "1_views_default") + before = {layer["name"]: displayed(view, layer["name"]) for layer in layers} + print("displays", json.dumps(before)) + for layer in layers: # the views as the layers say + settings = views.display_settings(layer) + assert before[layer["name"]]["visible"] == settings["visible"], layer["name"] + assert abs(before[layer["name"]]["opacity"] - settings["opacity"]) < 1e-9, layer["name"] + + run_macro("save_fspm_tf_views.py", reader) # saved unchanged: the same configuration + saved = views.read_views(views_file) + assert [views.filled(layer) for layer in saved["configurations"][name]["layers"]] == layers, \ + (saved["configurations"][name]["layers"], layers) + shoot = next(layer["name"] for layer in layers if layer["name"].endswith("shoot")) + simple.GetDisplayProperties(simple.FindSource(shoot), view).Opacity = 0.3 + run_macro("save_fspm_tf_views.py", reader) # changed, saved, applied again: as changed + run_macro("apply_fspm_tf_views.py", reader) + assert abs(displayed(view, shoot)["opacity"] - 0.3) < 1e-9 + screenshot(view, "2_views_shoot_opacity_0.3") + + roots = next(layer["name"] for layer in layers if layer["name"].endswith("roots")) + roots_item = simple.FindSource(roots) + data = servermanager.Fetch(roots_item) + leaf = data.GetBlock(0) if hasattr(data, "GetNumberOfBlocks") else data + import numpy as np + from vtkmodules.util.numpy_support import vtk_to_numpy + axes = vtk_to_numpy(leaf.GetCellData().GetArray("Axis.id")) + lateral = float(axes.max()) # the last axis: a lateral root + pick(roots_item, "Axis.id", lateral) + fspm_tf_selection.select_at_scale("Axis", "isolate") + screenshot(view, "3_isolate_axis") + hidden = [layer["name"] for layer in layers if displayed(view, layer["name"])["visible"]] + assert hidden == [], hidden + fspm_tf_selection.clear_selection() + after = {layer["name"]: displayed(view, layer["name"]) for layer in layers} + before[shoot]["opacity"] = 0.3 + assert after == before, (after, before) # as they were + screenshot(view, "4_cleared") + + segment = float(vtk_to_numpy(leaf.GetCellData().GetArray("SubOrgan.id"))[ + np.flatnonzero(axes == lateral)[0]]) + pick(roots_item, "SubOrgan.id", segment) + made = fspm_tf_selection.select_at_scale("SubOrgan", "isolate") + sections = [item for item in made if "Section" in item] + assert sections and simple.FindSource(sections[0]).GetDataInformation().GetNumberOfCells() > 0, made + view.ResetCamera() + screenshot(view, "5_isolate_segment_with_its_section") + fspm_tf_selection.clear_selection() + side_view(view) + + shoot_item = simple.FindSource(shoot) + shoot_data = servermanager.Fetch(shoot_item) + shoot_leaf = shoot_data.GetBlock(0) if hasattr(shoot_data, "GetNumberOfBlocks") else shoot_data + plant = float(vtk_to_numpy(shoot_leaf.GetCellData().GetArray("Plant.id"))[0]) + pick(shoot_item, "Plant.id", plant) + fspm_tf_selection.select_at_scale("Plant", "panel") + layout = simple.GetLayout(view) + layout.SetSize(1600, 700) + os.makedirs(pictures, exist_ok=True) + simple.SaveScreenshot(os.path.join(pictures, "6_panel_plant.png"), layout, ImageResolution=[1600, 700]) + print("screenshot", os.path.join(pictures, "6_panel_plant.png")) + assert {layer["name"]: displayed(view, layer["name"]) for layer in layers} == before # the view untouched + fspm_tf_selection.clear_selection() + view.ViewSize = [900, 700] + screenshot(view, "7_panel_cleared") + + reader.Configuration = "roots and sections" # the drop-down + reader.UpdatePipeline(last) + assert blocks(reader) == len(roots_only) + run_macro("apply_fspm_tf_views.py", reader) + assert simple.FindSource(shoot) is None # the other configuration's items replaced + colours = simple.GetColorTransferFunction("SubOrgan.radius") + points = list(colours.RGBPoints) + assert colours.UseLogScale and abs(points[0] - 0.002) < 1e-12 and abs(points[-4] - 0.005) < 1e-12, points + side_view(view) + screenshot(view, "8_configuration_roots_and_sections") + run_macro("save_fspm_tf_views.py", reader) # its range and log scale read back + saved = views.configuration(views.read_views(views_file), "roots and sections")[1] + assert saved == [views.filled(layer) for layer in roots_only], (saved, roots_only) + finally: + shutil.move(views_file + ".before_check", views_file) + print("fspm-TF design check: done, look at the screenshots in", pictures) + + +if DESIGN: + check_design(sys.argv[2]) +else: + check_conformance() print("fspm-TF reader: ok") diff --git a/paraview/clear_fspm_tf_selection.py b/paraview/clear_fspm_tf_selection.py new file mode 100644 index 0000000..129a455 --- /dev/null +++ b/paraview/clear_fspm_tf_selection.py @@ -0,0 +1,12 @@ +""" +ParaView macro "Clear fspm-TF selection": removes what select_at_scale made (the " selection" items, its +panel) and shows again the items it hid. + + Macros > Import new macro..., choose this file; then Macros > clear_fspm_tf_selection. +""" +try: + import fspm_tf_selection +except ImportError: + raise RuntimeError("fspm_tf_selection is not on ParaView's path: load the fspm-TF reader plugin first") from None + +fspm_tf_selection.clear_selection() diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py index 2e9b327..d619767 100644 --- a/paraview/fspm_tf_reader.py +++ b/paraview/fspm_tf_reader.py @@ -32,6 +32,11 @@ apply_fspm_tf_views.py and save_fspm_tf_views.py (next to this file; Macros > Import new macro...) make each layer its own pipeline item, and write the items' display options back into the views file. +Every cell also has the ids of the entities containing its own (".id" at each coarser scale, through the +complex links: a root segment's Organ, Axis and Plant), and every block its data structure (field data +fspm_tf_data_structure) and its graph's scales (fspm_tf_scales, coarser first): the macros select_at_scale.py and +clear_selection.py grow picked cells to their plant, axis, organ ... with everything it holds, in every layer. + frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests. """ import os @@ -152,11 +157,39 @@ def _mesh(mesh, data_structure, settings: bool = False, kept: np.ndarray = None, for set_name, rows in mesh.feature_rows.items(): _add_columns(polydata.GetCellData(), data_structure.entity_set(set_name), rows[triangles[:, 0]], polydata.GetFieldData(), settings, (forced or {}).get(set_name, ())) + ranks = {name: data_structure.entity_set(name).rank for name in mesh.feature_rows} + ranked = [name for name in ranks if ranks[name] is not None] + if ranked: # the finest scale's containing entities, where the mesh lacks them + finest = max(ranked, key=ranks.get) + _add_ancestors(polydata, data_structure, finest, mesh.feature_rows[finest][triangles[:, 0]]) + _identify(polydata, data_structure) for name, values in mesh.display.items(): polydata.GetPointData().AddArray(_array(f"display.{name}", np.asarray(values, dtype=np.float32))) return polydata +def _add_ancestors(data_set, data_structure, set_name: str, rows: np.ndarray) -> None: + """Cell data ".id" of the entities containing each cell's entity (at *rows* of *set_name*), for the + coarser scales the cells have no array of yet: what selection by scale grows a picked cell to.""" + cell_data = data_set.GetCellData() + for name, ids in _views().ancestor_ids(data_structure, set_name, rows).items(): + label = _views().id_array(name) + if not cell_data.HasArray(label): + cell_data.AddArray(_array(label, np.where(ids >= 0, ids, np.nan).astype(np.float64))) + + +def _identify(data_set, data_structure) -> None: + """Field data: the block's data structure, and its graph's scales (coarser first).""" + views = _views() + for name, values in ((views.DATA_STRUCTURE_FIELD, [data_structure.id]), + (views.SCALES_FIELD, views.ranked_sets(data_structure))): + listed = vtkStringArray() + listed.SetName(name) + for value in values: + listed.InsertNextValue(str(value)) + data_set.GetFieldData().AddArray(listed) + + def _segment_columns(entity_set) -> list: by_standard = {variable.standard_name: name for name, variable in entity_set.variables.items() if variable.standard_name} @@ -186,6 +219,8 @@ def _segments(entity_set, columns: list, data_structure, settings: bool = False, numpy_to_vtkIdTypeArray(connectivity, deep=True)) polydata.SetLines(cells) _add_columns(polydata.GetCellData(), entity_set, rows, polydata.GetFieldData(), settings, forced) + _add_ancestors(polydata, data_structure, entity_set.name, rows) + _identify(polydata, data_structure) if tube_radius is not None: radius = np.asarray(tube_radius, dtype=np.float64)[rows] polydata.GetPointData().AddArray(_array(_views().TUBE_RADIUS, np.concatenate([radius, radius]))) @@ -211,6 +246,7 @@ def _regular_grid(data_structure, settings: bool = False, kept: np.ndarray = Non if kept is not None: ghosts = np.where(kept[order], 0, vtkDataSetAttributes.HIDDENCELL).astype(np.uint8) image.GetCellData().AddArray(_array(vtkDataSetAttributes.GhostArrayName(), ghosts)) + _identify(image, data_structure) return image @@ -242,6 +278,7 @@ def _adaptive_grid(data_structure, settings: bool = False, kept: np.ndarray = No numpy_to_vtkIdTypeArray(np.arange(8 * count, dtype=np.int64), deep=True)) unstructured.SetCells(VTK_HEXAHEDRON, cell_array) _add_columns(unstructured.GetCellData(), cells, rows, unstructured.GetFieldData(), settings, forced) + _identify(unstructured, data_structure) return unstructured @@ -367,6 +404,8 @@ def frame_at(self, time: float = None): _fspm_tf() # on ParaView's path now: the views macros import it too except ImportError: pass + if os.path.dirname(os.path.abspath(__file__)) not in sys.path: # and fspm_tf_selection, the selection macros' + sys.path.append(os.path.dirname(os.path.abspath(__file__))) @smproxy.reader(name="fspmTFReader", label="fspm-TF reader", extensions="glb json", file_description="fspm-TF frames (.glb) and sequences (scene.fspm.json)") diff --git a/paraview/fspm_tf_selection.py b/paraview/fspm_tf_selection.py new file mode 100644 index 0000000..9019190 --- /dev/null +++ b/paraview/fspm_tf_selection.py @@ -0,0 +1,296 @@ +""" +Selection by scale in ParaView, the glue of the macros select_at_scale.py and clear_fspm_tf_selection.py: the cells +picked in a view (on any fspm-TF item: a layer that "Apply fspm-TF views" made, or the reader) grown to their entity +at a scale (a plant, an axis, an organ, a segment ...) with everything it contains at finer scales, in every layer of +the same reader (hidden ones too: a segment's section), shown + + "isolate" in the view, the other items hidden until the selection is cleared; + "panel" in a new render view beside it, its camera fit to the selection, the view itself untouched. + +Each layer's part is an item " selection" (a Programmable Filter calling extract below), drawn as its layer. +The view keeps what the selection made and hid (its annotation fspm_tf_selection), for clear_selection. + +The plugin (fspm_tf_reader.py) puts this folder and openalea.fspm_tf on ParaView's path. What is selected is +openalea.fspm_tf.views' (picked_entities, selection_mask, isolated), tested without ParaView. +""" +import json + +import numpy as np +from vtkmodules.util.numpy_support import numpy_to_vtk, vtk_to_numpy +from vtkmodules.vtkCommonDataModel import vtkDataObject, vtkDataObjectTree, vtkDataSetAttributes +from vtkmodules.vtkFiltersCore import vtkThreshold + +STATE = "fspm_tf_selection" # the view's annotation: {"items", "hidden", "scalar_bars", "panel", ...} +SUFFIX = " selection" +MARK = "fspm_tf_selected" # the cell array extract thresholds + + +def _views(): + from openalea.fspm_tf import views + return views + + +def _leaves(data): + """The data sets of *data*, a data set or a composite one.""" + if data is None: + return [] + if not isinstance(data, vtkDataObjectTree): + return [data] + iterator = data.NewTreeIterator() + iterator.InitTraversal() + found = [] + while not iterator.IsDoneWithTraversal(): + found.append(iterator.GetCurrentDataObject()) + iterator.GoToNextItem() + return found + + +def _texts(data_set, name: str) -> list: + array = data_set.GetFieldData().GetAbstractArray(name) + return [array.GetValue(number) for number in range(array.GetNumberOfValues())] if array is not None else [] + + +def _id_arrays(data_set) -> dict: + cell_data = data_set.GetCellData() + names = [cell_data.GetArrayName(number) for number in range(cell_data.GetNumberOfArrays())] + return {name: vtk_to_numpy(cell_data.GetArray(name)) for name in names if name and name.endswith(".id")} + + +# ── On the server: a layer's selected cells ────────────────────────────────── + +def _kept(data_set, chosen: dict, scale: str): + views = _views() + data_structure = (_texts(data_set, views.DATA_STRUCTURE_FIELD) or [""])[0] + mask = views.selection_mask(data_structure, _id_arrays(data_set), chosen, scale) + if mask.size != data_set.GetNumberOfCells(): # a block without id arrays + mask = np.zeros(data_set.GetNumberOfCells(), dtype=bool) + marked = data_set.NewInstance() + marked.ShallowCopy(data_set) + marked.GetCellData().RemoveArray(vtkDataSetAttributes.GhostArrayName()) # a grid layer's hidden cells + marked.GetCellData().AddArray(_named(numpy_to_vtk(mask.astype(np.uint8), deep=True), MARK)) + threshold = vtkThreshold() + threshold.SetInputData(marked) + threshold.SetInputArrayToProcess(0, 0, 0, vtkDataObject.FIELD_ASSOCIATION_CELLS, MARK) + threshold.SetThresholdFunction(vtkThreshold.THRESHOLD_UPPER) + threshold.SetUpperThreshold(0.5) + threshold.SetLowerThreshold(0.5) + threshold.Update() + output = threshold.GetOutput() + output.GetCellData().RemoveArray(MARK) + output.GetFieldData().ShallowCopy(data_set.GetFieldData()) + return output + + +def _named(array, name: str): + array.SetName(name) + return array + + +def extract(algorithm, chosen: dict, scale: str) -> None: + """A Programmable Filter's RequestData: its input's cells inside the entities *chosen* at *scale*, in each + block.""" + source, output = algorithm.GetInputDataObject(0, 0), algorithm.GetOutputDataObject(0) + if isinstance(source, vtkDataObjectTree): + output.CopyStructure(source) + iterator = source.NewTreeIterator() + iterator.InitTraversal() + while not iterator.IsDoneWithTraversal(): + output.SetDataSet(iterator, _kept(iterator.GetCurrentDataObject(), chosen, scale)) + iterator.GoToNextItem() + else: + output.ShallowCopy(_kept(source, chosen, scale)) + + +# ── On the client: the macros ──────────────────────────────────────────────── + +def _simple(): + from paraview import simple + return simple + + +def fspm_tf_items() -> dict: + """{reader: [(name, item)]} of what shows each fspm-TF reader: its layer items when "Apply fspm-TF views" made + them, else the reader.""" + simple, views = _simple(), _views() + found = {} + for (name, _), item in simple.GetSources().items(): + if not (item.ListProperties() and "ConfigurationNames" in item.ListProperties()): + continue # not an fspm-TF reader + path = views.views_path(views.property_text(item.FileName), views.property_text(item.Views) or None) + layers = [] + if path is not None: + _, configuration = views.configuration(views.read_views(path), + views.property_text(item.Configuration) or None) + layers = [(layer["name"], simple.FindSource(layer["name"])) for layer in configuration] + layers = [(layer, proxy) for layer, proxy in layers if proxy is not None] + found[name] = layers or [(name, item)] + return found + + +def picked(items: list) -> tuple: + """[(data structure, {".id": values})] of the cells picked on *items* ([(name, item)]), one entry per + block; and the scales they are in.""" + simple = _simple() + from paraview import servermanager + found, scales = [], [] + for _, item in items: + selection = item.SMProxy.GetSelectionInput(0) + if selection is None: + continue + extraction = simple.ExtractSelection(Input=item, Selection=servermanager._getPyProxy(selection)) + data = servermanager.Fetch(extraction) + simple.Delete(extraction) + simple.ClearSelection(item) # used: the next pick starts afresh + for leaf in _leaves(data): + if leaf.GetNumberOfCells(): + views = _views() + found.append(((_texts(leaf, views.DATA_STRUCTURE_FIELD) or [""])[0], _id_arrays(leaf))) + scales += [name for name in _texts(leaf, views.SCALES_FIELD) if name not in scales] + return found, scales + + +def _shown(view) -> dict: + """{item name: visible} of the items drawn in *view* (asking a display of the others would make one).""" + names = {item.SMProxy: name for (name, _), item in _simple().GetSources().items()} + found = {} + for display in view.Representations: + if "Input" not in display.ListProperties(): # a scalar bar, a widget + continue + source = display.Input.SMProxy if display.Input is not None else None + if source in names: + found[names[source]] = found.get(names[source], False) or bool(display.Visibility) + return found + + +def _state(view) -> dict: + proxy = view.SMProxy + return json.loads(proxy.GetAnnotation(STATE)) if proxy.HasAnnotation(STATE) else None + + +def _draw_as(display, layer_display) -> bool: + """*display* drawn as *layer_display*: its colouring (the same colour map, its range kept: not ColorBy, which + rescales it to the selection's values), solid colour, opacity, representation; whether it is coloured.""" + array = list(layer_display.ColorArrayName) if layer_display.ColorArrayName else [] + coloured = len(array) == 2 and bool(array[1]) + if coloured: + display.ColorArrayName = array + display.LookupTable = layer_display.LookupTable + else: + display.ColorArrayName = ["POINTS", ""] + display.DiffuseColor = list(layer_display.DiffuseColor) + display.AmbientColor = list(layer_display.AmbientColor) + display.Opacity = float(layer_display.Opacity) + display.SetRepresentationType(_views().property_text(layer_display.Representation)) + return coloured + + +def select_at_scale(scale: str, mode: str = "isolate", view=None) -> list: + """Grow the picked cells to their entities at *scale* and show them (mode "isolate" or "panel"); the names of + the selection's items. A previous selection is cleared first.""" + simple, views = _simple(), _views() + if mode not in ("isolate", "panel"): + raise ValueError(f"MODE: 'isolate' or 'panel', not {mode!r}") + view = view or simple.GetActiveViewOrCreate("RenderView") + readers = fspm_tf_items() + previous = _state(view) or {} + cells, scales, items = [], [], [] + for reader, shown in readers.items(): # the readers picked on (a selection's items: its readers) + made = [(name, simple.FindSource(f"{name}{SUFFIX}")) for name, _ in shown + if f"{name}{SUFFIX}" in previous.get("items", [])] + found, its_scales = picked(shown + [(name, item) for name, item in made if item is not None]) + if found: + cells += found + scales += [name for name in its_scales if name not in scales] + items += shown + if scales and scale not in scales: + raise ValueError(f"SCALE '{scale}' is not a scale of the picked elements: {scales}") + chosen = views.picked_entities(cells, scale) + clear_selection() + target = view + if mode == "panel": + target = simple.CreateView("RenderView") + layout = simple.GetLayout(view) + if layout is None: + layout = simple.CreateLayout("fspm-TF") + layout.AssignView(0, view) + location = layout.GetViewLocation(view) + layout.SplitHorizontal(location, 0.5) + layout.AssignView(2 * location + 2, target) + target.ViewSize = list(view.ViewSize) + target.CameraPosition, target.CameraFocalPoint = list(view.CameraPosition), list(view.CameraFocalPoint) + target.CameraViewUp = list(view.CameraViewUp) + target.ViewTime = view.ViewTime + made = [] + for name, item in items: + layer_display = simple.GetDisplayProperties(item, view) + part = simple.ProgrammableFilter(registrationName=f"{name}{SUFFIX}", Input=item) + part.OutputDataSetType = "Same as Input" + part.Script = f"import fspm_tf_selection\nfspm_tf_selection.extract(self, {chosen!r}, {scale!r})" + part.UpdatePipeline() + display = simple.Show(part, target) + coloured = _draw_as(display, layer_display) + if not part.GetDataInformation().GetNumberOfCells(): + simple.Hide(part, target) # nothing of the selection in this layer + elif coloured and mode == "panel": + display.SetScalarBarVisibility(target, True) + made.append(f"{name}{SUFFIX}") + hidden, bars = [], [] + if mode == "isolate": + hidden = views.isolated(_shown(view), made) + for name in hidden: + display = simple.GetDisplayProperties(simple.FindSource(name), view) + if display.LookupTable is not None and simple.GetScalarBar(display.LookupTable, view).Visibility: + bars.append(name) + simple.Hide(simple.FindSource(name), view) + simple.HideUnusedScalarBars(view) + else: + target.ResetCamera() + simple.Render(target) + state = {"items": made, "hidden": hidden, "scalar_bars": bars, "scale": scale, "chosen": chosen, + "panel": None if mode == "isolate" else _view_name(target)} + view.SMProxy.SetAnnotation(STATE, json.dumps(state)) + simple.SetActiveView(view) + simple.Render(view) + print(f"fspm-TF selection at {scale}: {chosen} ({mode})") + return made + + +def _named_views() -> dict: + """{registration name: view} of the session's views.""" + from paraview import servermanager + return {name: view for (name, _), view in servermanager.ProxyManager().GetProxiesInGroup("views").items()} + + +def _view_name(view) -> str: + return next(name for name, proxy in _named_views().items() if proxy.SMProxy is view.SMProxy) + + +def clear_selection() -> None: + """Remove the selection's items and panel, and show again what it hid, in every view holding one.""" + simple = _simple() + for view in _named_views().values(): + state = _state(view) + if state is None: + continue + for name in state["items"]: + item = simple.FindSource(name) + if item is not None: + simple.Delete(item) + for name in state["hidden"]: + item = simple.FindSource(name) + if item is not None: + display = simple.Show(item, view) + if name in state.get("scalar_bars", []): + display.SetScalarBarVisibility(view, True) + if state["panel"]: + panel = _named_views().get(state["panel"]) + if panel is not None: + layout = simple.GetLayout(panel) + if layout is not None: + location = layout.GetViewLocation(panel) + layout.RemoveView(panel) + if location > 0: + layout.Collapse(location) + simple.Delete(panel) + view.SMProxy.RemoveAnnotation(STATE) + simple.Render(view) diff --git a/paraview/save_fspm_tf_views.py b/paraview/save_fspm_tf_views.py index 07d9967..90ce565 100644 --- a/paraview/save_fspm_tf_views.py +++ b/paraview/save_fspm_tf_views.py @@ -29,7 +29,7 @@ def display_of(item, view) -> dict: display = simple.GetDisplayProperties(item, view) array = display.ColorArrayName[1] if display.ColorArrayName and display.ColorArrayName[1] else None found = {"visible": bool(display.Visibility), "opacity": float(display.Opacity), - "representation": str(display.Representation), "color_array": array, + "representation": _views().property_text(display.Representation), "color_array": array, "solid_color": [float(value) for value in display.DiffuseColor]} if array: colours = simple.GetColorTransferFunction(array) @@ -45,12 +45,12 @@ def save_views(source=None, view=None, configuration: str = None) -> str: source = source or simple.GetActiveSource() if source is None or not hasattr(source, "Configuration"): raise RuntimeError("select an fspm-TF reader in the Pipeline Browser first") - file_name = source.FileName if isinstance(source.FileName, str) else source.FileName[0] - path = views.views_path(file_name, source.Views) + file_name = views.property_text(source.FileName) + path = views.views_path(file_name, views.property_text(source.Views) or None) if path is None: raise RuntimeError(f"no views file for {file_name} (a views.json next to it, or the reader's Views)") document = views.read_views(path) - name, _ = views.configuration(document, source.Configuration or None) + name, _ = views.configuration(document, views.property_text(source.Configuration) or None) view = view or simple.GetActiveViewOrCreate("RenderView") layers = [] for layer in document["configurations"][name]["layers"]: diff --git a/paraview/select_at_scale.py b/paraview/select_at_scale.py new file mode 100644 index 0000000..572635a --- /dev/null +++ b/paraview/select_at_scale.py @@ -0,0 +1,25 @@ +""" +ParaView macro "Select at scale": the cells picked in the view (Select Cells On, on any fspm-TF layer or reader) grown +to their entity at SCALE (a plant, an axis, an organ, a segment ...) with everything it contains at finer scales, in +every layer, hidden ones included (a segment's section); shown as MODE says: + + "isolate" in the view, the other items hidden (Macros > clear_fspm_tf_selection shows them again); + "panel" in a new render view beside it, its own camera fit to the selection; the view itself untouched + (clear_fspm_tf_selection closes the panel). + + Macros > Import new macro..., choose this file (and clear_fspm_tf_selection.py); set SCALE and MODE below + (Macros > Edit...); load the fspm-TF reader plugin, pick cells, then Macros > select_at_scale. + +The scales are the frame's entity sets (Plant, Axis, Organ, SubOrgan, ... Compartment); an unknown one is reported +with the picked elements' scales. The glue is fspm_tf_selection.py (next to the plugin, on ParaView's path once it +is loaded); what is selected, openalea.fspm_tf.views' picked_entities and selection_mask, tested without ParaView. +""" +SCALE = "Plant" +MODE = "isolate" # or "panel" + +try: + import fspm_tf_selection +except ImportError: + raise RuntimeError("fspm_tf_selection is not on ParaView's path: load the fspm-TF reader plugin first") from None + +fspm_tf_selection.select_at_scale(SCALE, MODE) From d0638319c5e8e9f36e062130b98fc84a2484965a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Tue, 3 Nov 2026 17:01:09 +0100 Subject: [PATCH 25/39] Selection by scale as a filter of the plugin (fspm-TF Select at scale): ParaView's current selection taken at creation (a Selection input, Copy Active Selection), its Scale a drop-down of the frames' scales, the picked cells grown in every layer, following time steps; show_fspm_tf_selection the one macro left (isolate or a split panel, run again to go back); select_at_scale and clear_fspm_tf_selection removed; checked in ParaView 6.2 (pvbatch); tests --- CHANGELOG.md | 6 + bindings/python/src/openalea/fspm_tf/views.py | 16 +- bindings/python/test/test_paraview_reader.py | 51 +++ bindings/python/test/test_views.py | 2 + doc/using.md | 56 ++- paraview/check_in_paraview.py | 120 +++-- paraview/clear_fspm_tf_selection.py | 12 - paraview/fspm_tf_reader.py | 264 ++++++++++- paraview/fspm_tf_selection.py | 423 +++++++++--------- paraview/select_at_scale.py | 25 -- paraview/show_fspm_tf_selection.py | 27 ++ 11 files changed, 695 insertions(+), 307 deletions(-) delete mode 100644 paraview/clear_fspm_tf_selection.py delete mode 100644 paraview/select_at_scale.py create mode 100644 paraview/show_fspm_tf_selection.py diff --git a/CHANGELOG.md b/CHANGELOG.md index abb7502..00a38d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ - The ParaView reader: "Views" and "Configuration" (a drop-down), one block per layer holding only its entities; `frame_to_vtk(frame, views=, configuration=)`, unchanged without views. The macros `apply_fspm_tf_views.py` (each layer its own pipeline item) and `save_fspm_tf_views.py` (the items' display options back into `views.json`). +- Selection by scale in ParaView: the plugin's filter "fspm-TF Select at scale" (Filters → fspm-TF) takes ParaView's + selection and grows the picked cells to their entity at its **Scale** (a drop-down of the frames' scales) with + everything it contains, in every layer of the reader, hidden ones included, following the picked entities through + the time steps; the macro `show_fspm_tf_selection.py` shows it as its layers, isolated in the view or in a panel of + its own, one selection over another, run again to go back. Each cell carries the ids of the entities containing it + (`.id`), each block its data structure and scales. ## Unreleased: specification 0.3 (draft), coupling platforms diff --git a/bindings/python/src/openalea/fspm_tf/views.py b/bindings/python/src/openalea/fspm_tf/views.py index b44f6b3..793002f 100644 --- a/bindings/python/src/openalea/fspm_tf/views.py +++ b/bindings/python/src/openalea/fspm_tf/views.py @@ -20,10 +20,11 @@ ParaView's macros (paraview/apply_fspm_tf_views.py, save_fspm_tf_views.py) go through display_settings(layer) (a layer to its display options) and layer_from_display(layer, display) (back, to save them), plain dicts both ways. -Selection by scale (paraview/select_at_scale.py, clear_selection.py): the reader gives every cell the ids of its -entity and of the entities containing it (".id", ancestor_ids), and each block its data structure and -its graph's scales as field data; picked_entities(picked, scale) grows picked cells to their entities at *scale*, -selection_mask(...) gives the cells of any block those entities contain, at any finer scale. +Selection by scale (the plugin's filter "fspm-TF Select at scale", the macro paraview/show_fspm_tf_selection.py): the +reader gives every cell the ids of its entity and of the entities containing it (".id", ancestor_ids), +and each block its data structure and its graph's scales as field data (scale_names: the filter's drop-down); +picked_entities(picked, scale) grows picked cells to their entities at *scale*, selection_mask(...) gives the cells of +any block those entities contain, at any finer scale. """ import copy import json @@ -379,6 +380,13 @@ def ranked_sets(data_structure) -> list: return [entity_set.name for entity_set in sorted(ranked, key=lambda entity_set: entity_set.rank)] +def scale_names(scales: list) -> list: + """The scales of several blocks (*scales*: each block's, coarser first) in one list, coarser first: the "Select at + scale" filter's drop-down.""" + longest_first = sorted(scales, key=len, reverse=True) + return list(dict.fromkeys(name for names in longest_first for name in names)) + + def ancestor_ids(data_structure, entity_set: str, rows: np.ndarray) -> dict: """{entity set: ids per row (NO_ID: none)} of the entities containing the entities at *rows* of *entity_set*, at each coarser scale their complex chain reaches.""" diff --git a/bindings/python/test/test_paraview_reader.py b/bindings/python/test/test_paraview_reader.py index 8eff571..72b0f9a 100644 --- a/bindings/python/test/test_paraview_reader.py +++ b/bindings/python/test/test_paraview_reader.py @@ -213,3 +213,54 @@ def test_cells_carry_the_ids_of_the_entities_containing_theirs(): assert set(arrays["SubOrgan.id"][lateral]) == {60., 61.} # the lateral's segments, not its apex soil = blocks_of(generate.soil_and_roots())[("soil", "grid")] assert soil.GetFieldData().GetAbstractArray(views.DATA_STRUCTURE_FIELD).GetValue(0) == "soil" + + +def test_select_at_scale_grows_picked_cells_in_every_layer(): + """The "Select at scale" filter's work: the cells picked on a layer (a vtkSelection, as ParaView's) grown to their + axis or plant in every layer, the hidden one too, the same blocks, polygonal data kept polygonal; the frame found + upstream of items drawn from the reader.""" + from vtkmodules.vtkCommonCore import vtkIdTypeArray + from vtkmodules.vtkCommonDataModel import vtkPolyData, vtkSelection, vtkSelectionNode + from vtkmodules.vtkCommonExecutionModel import vtkTrivialProducer + from vtkmodules.vtkFiltersCore import vtkPassThrough + from test_views import roots_views + frame = generate.root_system() + data = reader.frame_to_vtk(frame, views=roots_views()) # layers "first order", "roots" (hidden) + lines = data.GetBlock(1) + cell = int(np.flatnonzero(cell_array(lines, "Axis.id") == 101.)[0]) + node = vtkSelectionNode() + node.SetFieldType(vtkSelectionNode.CELL) + node.SetContentType(vtkSelectionNode.INDICES) + ids = vtkIdTypeArray() + ids.InsertNextValue(cell) + node.SetSelectionList(ids) + node.GetProperties().Set(vtkSelectionNode.COMPOSITE_INDEX(), 2) # the roots block (flat index 2) + selection = vtkSelection() + selection.AddNode(node) + picked = reader.picked_cells(data, selection) + assert [data_structure for data_structure, _ in picked] == ["root_system"] + assert set(picked[0][1]["Axis.id"]) == {101.} + assert reader.scales_of(data) == ["Plant", "Axis", "SubOrgan"] + grown = reader.grow_selection(data, picked, "Axis") + assert [grown.GetMetaData(number).Get(grown.NAME()) for number in range(2)] == ["first order", "roots"] + axis_lines = grown.GetBlock(1) + assert isinstance(axis_lines, vtkPolyData) and set(cell_array(axis_lines, "Axis.id")) == {101.} + assert axis_lines.GetNumberOfCells() == (cell_array(lines, "Axis.id") == 101.).sum() + assert axis_lines.GetCellData().GetScalars().GetName() == "SubOrgan.label_code" # the layer's colouring + assert axis_lines.GetPointData().GetArray("tube_radius") is not None # a Tube can follow + first_order = grown.GetBlock(0) + assert first_order.GetNumberOfCells() == 0 # the lateral is not of the first order + plant = reader.grow_selection(data, picked, "Plant") + assert plant.GetBlock(0).GetNumberOfCells() == data.GetBlock(0).GetNumberOfCells() + assert plant.GetBlock(1).GetNumberOfCells() == lines.GetNumberOfCells() + with pytest.raises(ValueError, match="Compartment"): + reader.grow_selection(data, picked, "Compartment") + assert reader.picked_cells(data, vtkSelection()) == [] + producer = vtkTrivialProducer() # the reader, an item drawn from it, the filter + producer.SetOutput(data) + item, filter_algorithm = vtkPassThrough(), vtkPassThrough() + item.SetInputConnection(producer.GetOutputPort()) + filter_algorithm.SetInputConnection(item.GetOutputPort()) + filter_algorithm.Update() + assert reader.frame_upstream(filter_algorithm) is data + assert reader.frame_upstream(producer) is None diff --git a/bindings/python/test/test_views.py b/bindings/python/test/test_views.py index 512f5b4..f290b7d 100644 --- a/bindings/python/test/test_views.py +++ b/bindings/python/test/test_views.py @@ -275,6 +275,8 @@ def test_picked_cells_grow_to_their_entity_and_what_it_contains(): [False] * 2) # a block without the scale: none of it assert views.isolated({"roots": True, "shoot": True, "sections": False, "roots selection": True}, ["roots selection"]) == ["roots", "shoot"] + blocks = [["Plant", "Axis", "SubOrgan"], [], ["Plant", "Axis", "Organ", "SubOrgan", "Compartment"]] + assert views.scale_names(blocks) == ["Plant", "Axis", "Organ", "SubOrgan", "Compartment"] # every block's def test_views_are_checked_without_jsonschema(monkeypatch): diff --git a/doc/using.md b/doc/using.md index 0dc26cb..14fb2b7 100644 --- a/doc/using.md +++ b/doc/using.md @@ -102,30 +102,48 @@ print(report) ### Selecting a plant, an axis, an organ -Two more macros in `paraview/` select by scale: pick an element, and see its whole plant, axis, organ or segment, +The plugin has a filter that selects by scale: pick an element, and get its whole plant, axis, organ or segment, with everything it contains at finer scales, in every layer, hidden ones included (a segment's section drawings, a -hidden layer by default, show for the selection only). - -1. Load them once like the others: **Macros → Import new macro…**, `paraview/select_at_scale.py`, then - `paraview/clear_fspm_tf_selection.py`. At the top of `select_at_scale` (**Macros → Edit…**), set `SCALE` (an - entity set of the frames: `Plant`, `Axis`, `Organ`, `SubOrgan`, … `Compartment`) and `MODE`. -2. Pick cells in the view (**Select Cells On**, or `s`), on any layer: a leaf, a root segment. -3. Run **select_at_scale**. With `MODE = "isolate"`, the view shows only the selection, the other items hidden; - with `MODE = "panel"`, a new view opens beside it with the selection alone, its camera fit to it, the first view - untouched. Each layer's part is an item named ` selection`, drawn as its layer (the same colour map). - Picking again on the selection and running the macro again selects anew (e.g. an axis of the plant shown). -4. Run **clear_fspm_tf_selection**: the selection's items and panel are removed, the hidden items shown again. - -A scale finer than the picked element (a Compartment of a leaf drawn without sections) is reported with the scales -the picked cells have. The macros need the plugin loaded (it puts their code, `paraview/fspm_tf_selection.py`, on -ParaView's path); which cells are selected is `openalea.fspm_tf.views.picked_entities` and `selection_mask`, tested -without ParaView. +hidden layer by default, come with the selection). + +1. Pick cells in the view (**Select Cells On**, or `s`), on any fspm-TF item: a layer, the reader, or a selection + shown before. The item picked on becomes the active one in the Pipeline Browser. +2. **Filters → fspm-TF → fspm-TF Select at scale**, then **Apply**. The filter takes ParaView's current selection + when it is made, as *Extract Selection* does (its **Selection** property: **Copy Active Selection** takes the + selection made since, on the same item). +3. Choose the **Scale** in its Properties: a drop-down of the frames' scales, coarser first (`Plant`, `Axis`, + `Organ`, `SubOrgan`, … `Compartment`), `Plant` at first. **Apply** grows the picked cells again. +4. Play the time steps: the picked entities are kept, so the plant or organ is followed as it grows. + +The filter's output is one block per layer, named and filled like the reader's (their variables kept, so the +layers' colourings apply), each holding the selection's part of the layer (empty when it has none). A scale finer +than the picked element (a Compartment of a leaf drawn without sections) is reported in the Output Messages, with +the scales the picked cells have. To drop the selection, delete the filter. + +To look at the selection alone, a macro shows it as its layers, each part drawn as its layer (the same colour map, +a tube for roots): + +1. Load it once like the others: **Macros → Import new macro…**, `paraview/show_fspm_tf_selection.py`. At its top + (**Macros → Edit…**), set `MODE`. +2. With the filter selected in the Pipeline Browser, run **show_fspm_tf_selection**. With `MODE = "isolate"`, the + view shows only the selection, the other items hidden; with `MODE = "panel"`, a new view opens beside it with + the selection alone, its camera fit to it, the first view untouched. Each layer's part is an item named + ` selection`; changing the filter's Scale or the time step updates them. +3. To narrow it (an axis of the plant shown), pick on these items, apply another fspm-TF Select at scale, and run + the macro again: it is shown over the first. +4. Run the macro again (with anything else selected, or the same filter) to go back one step: the items it made are + removed, the panel closed, the hidden items shown again. Then delete the filter. + +The macro is only for what a filter cannot do: hiding the other items, opening a panel. Both need the plugin +loaded (it puts the macro's code, `paraview/fspm_tf_selection.py`, on ParaView's path); which cells are selected is +`openalea.fspm_tf.views.picked_entities` and `selection_mask`, tested without ParaView. `paraview/check_in_paraview.py` checks all this inside ParaView. `pvpython paraview/check_in_paraview.py conformance` runs on the conformance files (the binding tests run it when `FSPM_TF_PVPYTHON` names a `pvpython`); `pvbatch paraview/check_in_paraview.py --design ` is a check by hand, on a sequence with -its `views.json`: it applies and saves the views, selects at several scales, isolated and in a panel, clears, and -saves a screenshot of each step in the folder, to look at. +its `views.json`: it applies and saves the views, selects at several scales with the filter (switching its Scale, +stepping in time), shows the selections isolated, in a panel and one over another, goes back, and saves a screenshot +of each step in the folder, to look at. ## In other viewers diff --git a/paraview/check_in_paraview.py b/paraview/check_in_paraview.py index a3f4de6..a2f8ff5 100644 --- a/paraview/check_in_paraview.py +++ b/paraview/check_in_paraview.py @@ -1,15 +1,18 @@ """ The reader inside ParaView: loads the plugin, opens conformance files and a sequence, checks their blocks, arrays and time steps, and renders a picture of each; then a frame with views (one block per layer, the configurations' -drop-down), the macros applying and saving them, and selection by scale (isolated, then cleared). +drop-down), the macros applying and saving them, and selection by scale (the filter "fspm-TF Select at scale", its +Scale drop-down, the macro showing it isolated, then going back). pvpython paraview/check_in_paraview.py conformance [output folder] A design check, by hand (no test runs it): a sequence with its views.json (e.g. metafspm's SceneRecorder(frame_views="default", display_period=...) frames of plants with shoots, roots and sections, and a soil -grid) opened, each configuration applied, saved back unchanged and changed (the round trip), then selections at each -scale, isolated and in a panel, and cleared; a screenshot of each step in the output folder, to look at (colours, -opacities, tubes, hidden layers, isolation). The views file is put back as it was. +grid) opened, each configuration applied, saved back unchanged and changed (the round trip), then selections by the +filter "fspm-TF Select at scale" (cells picked as a click would, the filter made with ParaView's selection as the GUI +makes it, its Scale switched, the time stepped), shown by the macro isolated, in a panel, one over another, and put +back; a screenshot of each step in the output folder, to look at (colours, opacities, tubes, hidden layers, +isolation). The views file is put back as it was. pvbatch paraview/check_in_paraview.py --design outputs/frames/scene.fspm.json screenshots """ @@ -134,18 +137,36 @@ def check_conformance() -> None: import fspm_tf_selection lateral = pick(tube, "Axis.id", 101.) - made = fspm_tf_selection.select_at_scale("Axis", "isolate") - assert sorted(made) == ["mesh selection", "roots selection"], made - assert set(servermanager.Fetch(simple.FindSource("roots selection")).GetBlock(0).GetCellData() - .GetArray("Axis.id").GetRange()) == {101.}, lateral + selection = fspmTFSelectatscale(Input=tube) # noqa: F821 (ParaView's selection, as the GUI takes it) + assert selection.Selection is not None + assert scale_names(selection) == ["Plant", "Axis", "SubOrgan"], scale_names(selection) + assert selection.Scale == "Plant" # the drop-down's first, the coarsest + selection.Scale = "Axis" + selection.UpdatePipeline() + grown = servermanager.Fetch(selection) + assert grown.GetNumberOfBlocks() == 2 # the layers, the hidden one too + assert set(grown.GetBlock(0).GetCellData().GetArray("Axis.id").GetRange()) == {101.}, lateral + assert 0 < grown.GetBlock(1).GetNumberOfCells() < servermanager.Fetch(viewed).GetBlock(1).GetNumberOfCells() + made = fspm_tf_selection.show_selection(selection, "isolate") + assert sorted(made) == ["mesh selection", "roots selection", "roots selection block"], made assert not simple.GetDisplayProperties(tube).Visibility - fspm_tf_selection.clear_selection() + assert fspm_tf_selection.back() assert simple.GetDisplayProperties(tube).Visibility and simple.FindSource("roots selection") is None + simple.Delete(selection) # the selection dropped viewed.Configuration = "mesh only" viewed.UpdatePipeline() assert blocks(viewed) == 1 +def cells(name: str) -> int: + return simple.FindSource(name).GetDataInformation().GetNumberOfCells() + + +def scale_names(selection) -> list: + selection.UpdatePropertyInformation() + return list(selection.GetProperty("ScaleNames")) + + def pick(item, array: str, value: float) -> int: """Pick (select) the first cell of *item* whose *array* is *value*, as a click would; its index.""" import numpy as np @@ -166,6 +187,13 @@ def screenshot(view, name: str) -> None: print("screenshot", path) +def layout_screenshot(layout, name: str) -> None: + os.makedirs(pictures, exist_ok=True) + path = os.path.join(pictures, f"{name}.png") + simple.SaveScreenshot(path, layout, ImageResolution=[1600, 700]) + print("screenshot", path) + + def side_view(view) -> None: """Seen from the side and a little above: z up.""" view.CameraPosition, view.CameraFocalPoint, view.CameraViewUp = [0.3, -1., 0.35], [0., 0., 0.], [0., 0., 1.] @@ -236,6 +264,7 @@ def check_design(index_path: str) -> None: screenshot(view, "2_views_shoot_opacity_0.3") roots = next(layer["name"] for layer in layers if layer["name"].endswith("roots")) + sections = next(layer["name"] for layer in layers if "Section" in layer["name"]) roots_item = simple.FindSource(roots) data = servermanager.Fetch(roots_item) leaf = data.GetBlock(0) if hasattr(data, "GetNumberOfBlocks") else data @@ -244,25 +273,49 @@ def check_design(index_path: str) -> None: axes = vtk_to_numpy(leaf.GetCellData().GetArray("Axis.id")) lateral = float(axes.max()) # the last axis: a lateral root pick(roots_item, "Axis.id", lateral) - fspm_tf_selection.select_at_scale("Axis", "isolate") + selection = fspmTFSelectatscale(Input=roots_item) # noqa: F821 (the GUI's Filters > fspm-TF, Apply) + scales = scale_names(selection) + print("scales", scales) + assert scales[0] == "Plant" and "Axis" in scales and "Compartment" in scales, scales + selection.Scale = "Axis" + run_macro("show_fspm_tf_selection.py", selection) # the macro, MODE "isolate" screenshot(view, "3_isolate_axis") - hidden = [layer["name"] for layer in layers if displayed(view, layer["name"])["visible"]] - assert hidden == [], hidden - fspm_tf_selection.clear_selection() + assert [layer["name"] for layer in layers if displayed(view, layer["name"])["visible"]] == [] + part = simple.FindSource(f"{roots} selection") + assert set(servermanager.Fetch(part).GetBlock(0).GetCellData().GetArray("Axis.id").GetRange()) == {lateral} + axis_cells = cells(f"{roots} selection block") + selection.Scale = "Plant" # the drop-down changed, Apply: re-grown + simple.Render(view) + assert cells(f"{roots} selection block") > axis_cells, (cells(f"{roots} selection block"), axis_cells) + screenshot(view, "4_isolate_scale_switched_to_plant") + if reader.TimestepValues: # stepping in time: the same plant followed + first = list(reader.TimestepValues)[0] + view.ViewTime = first + simple.Render(view) + plant_ids = servermanager.Fetch(simple.FindSource(f"{roots} selection block")).GetBlock(0) + print("first time step", first, "plant", plant_ids.GetCellData().GetArray("Plant.id").GetRange()) + screenshot(view, "5_isolate_plant_first_time_step") + view.ViewTime = last + simple.Render(view) + run_macro("show_fspm_tf_selection.py", selection) # run again: put back after = {layer["name"]: displayed(view, layer["name"]) for layer in layers} before[shoot]["opacity"] = 0.3 assert after == before, (after, before) # as they were - screenshot(view, "4_cleared") + assert simple.FindSource(f"{roots} selection") is None + simple.Delete(selection) # the selection dropped + screenshot(view, "6_put_back") segment = float(vtk_to_numpy(leaf.GetCellData().GetArray("SubOrgan.id"))[ np.flatnonzero(axes == lateral)[0]]) pick(roots_item, "SubOrgan.id", segment) - made = fspm_tf_selection.select_at_scale("SubOrgan", "isolate") - sections = [item for item in made if "Section" in item] - assert sections and simple.FindSource(sections[0]).GetDataInformation().GetNumberOfCells() > 0, made + selection = fspmTFSelectatscale(Input=roots_item, Scale="SubOrgan") # noqa: F821 + fspm_tf_selection.show_selection(selection, "isolate") + assert cells(f"{sections} selection") > 0 # the hidden layer's part: the segment's section + assert displayed(view, f"{sections} selection")["visible"] view.ResetCamera() - screenshot(view, "5_isolate_segment_with_its_section") - fspm_tf_selection.clear_selection() + screenshot(view, "7_isolate_segment_with_its_section") + fspm_tf_selection.back() + simple.Delete(selection) side_view(view) shoot_item = simple.FindSource(shoot) @@ -270,16 +323,31 @@ def check_design(index_path: str) -> None: shoot_leaf = shoot_data.GetBlock(0) if hasattr(shoot_data, "GetNumberOfBlocks") else shoot_data plant = float(vtk_to_numpy(shoot_leaf.GetCellData().GetArray("Plant.id"))[0]) pick(shoot_item, "Plant.id", plant) - fspm_tf_selection.select_at_scale("Plant", "panel") + plant_selection = fspmTFSelectatscale(Input=shoot_item) # noqa: F821 (Scale: Plant, the first) + fspm_tf_selection.show_selection(plant_selection, "panel") layout = simple.GetLayout(view) layout.SetSize(1600, 700) - os.makedirs(pictures, exist_ok=True) - simple.SaveScreenshot(os.path.join(pictures, "6_panel_plant.png"), layout, ImageResolution=[1600, 700]) - print("screenshot", os.path.join(pictures, "6_panel_plant.png")) + layout_screenshot(layout, "8_panel_plant") assert {layer["name"]: displayed(view, layer["name"]) for layer in layers} == before # the view untouched - fspm_tf_selection.clear_selection() + + panel = [proxy for proxy in fspm_tf_selection._named_views().values() if proxy.SMProxy is not view.SMProxy][0] + simple.SetActiveView(panel) # an axis of the plant, picked in the panel + plant_roots = simple.FindSource(f"{roots} selection") + plant_axes = vtk_to_numpy(servermanager.Fetch(plant_roots).GetBlock(0).GetCellData().GetArray("Axis.id")) + pick(plant_roots, "Axis.id", float(plant_axes.max())) + axis_selection = fspmTFSelectatscale(Input=plant_roots, Scale="Axis") # noqa: F821 + made = fspm_tf_selection.show_selection(axis_selection, "isolate", panel) + assert f"{roots} selection 2" in made, made + assert not displayed(panel, f"{roots} selection")["visible"] + layout_screenshot(layout, "9_panel_an_axis_of_the_plant") + assert fspm_tf_selection.back() # back to the plant in the panel + assert displayed(panel, f"{roots} selection")["visible"] + simple.Delete(axis_selection) + assert fspm_tf_selection.back() # the panel closed + simple.Delete(plant_selection) + simple.SetActiveView(view) view.ViewSize = [900, 700] - screenshot(view, "7_panel_cleared") + screenshot(view, "10_panel_closed") reader.Configuration = "roots and sections" # the drop-down reader.UpdatePipeline(last) @@ -290,7 +358,7 @@ def check_design(index_path: str) -> None: points = list(colours.RGBPoints) assert colours.UseLogScale and abs(points[0] - 0.002) < 1e-12 and abs(points[-4] - 0.005) < 1e-12, points side_view(view) - screenshot(view, "8_configuration_roots_and_sections") + screenshot(view, "11_configuration_roots_and_sections") run_macro("save_fspm_tf_views.py", reader) # its range and log scale read back saved = views.configuration(views.read_views(views_file), "roots and sections")[1] assert saved == [views.filled(layer) for layer in roots_only], (saved, roots_only) diff --git a/paraview/clear_fspm_tf_selection.py b/paraview/clear_fspm_tf_selection.py deleted file mode 100644 index 129a455..0000000 --- a/paraview/clear_fspm_tf_selection.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -ParaView macro "Clear fspm-TF selection": removes what select_at_scale made (the " selection" items, its -panel) and shows again the items it hid. - - Macros > Import new macro..., choose this file; then Macros > clear_fspm_tf_selection. -""" -try: - import fspm_tf_selection -except ImportError: - raise RuntimeError("fspm_tf_selection is not on ParaView's path: load the fspm-TF reader plugin first") from None - -fspm_tf_selection.clear_selection() diff --git a/paraview/fspm_tf_reader.py b/paraview/fspm_tf_reader.py index d619767..5e1f3fd 100644 --- a/paraview/fspm_tf_reader.py +++ b/paraview/fspm_tf_reader.py @@ -34,20 +34,29 @@ Every cell also has the ids of the entities containing its own (".id" at each coarser scale, through the complex links: a root segment's Organ, Axis and Plant), and every block its data structure (field data -fspm_tf_data_structure) and its graph's scales (fspm_tf_scales, coarser first): the macros select_at_scale.py and -clear_selection.py grow picked cells to their plant, axis, organ ... with everything it holds, in every layer. - -frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests. +fspm_tf_data_structure) and its graph's scales (fspm_tf_scales, coarser first). The plugin's filter "fspm-TF Select +at scale" (Filters > fspm-TF), applied with cells picked on the reader or any item drawn from it, grows them to their +plant, axis, organ ... ("Scale", a drop-down of the scales) with everything it holds at finer scales, in every block +of the reader (every layer, the hidden ones included), the same blocks with their arrays; the picked entities are +followed through the time steps. The macro show_fspm_tf_selection.py draws it as its layers, isolated in the view or +in a panel of its own. + +frame_to_vtk(frame) builds the output without ParaView (VTK only), e.g. for scripts and tests; picked_cells and +grow_selection, the filter's. """ import os import sys import numpy as np -from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray -from vtkmodules.vtkCommonCore import vtkPoints, vtkStringArray -from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkDataSetAttributes, vtkImageData, - vtkMultiBlockDataSet, vtkPolyData, vtkUnstructuredGrid) -from vtkmodules.vtkCommonExecutionModel import vtkStreamingDemandDrivenPipeline +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray, vtk_to_numpy +from vtkmodules.vtkCommonCore import vtkOutputWindow, vtkPoints, vtkStringArray +from vtkmodules.vtkCommonDataModel import (VTK_HEXAHEDRON, vtkCellArray, vtkDataObject, vtkDataObjectTree, + vtkDataSetAttributes, vtkImageData, vtkMultiBlockDataSet, vtkPolyData, + vtkSelection, vtkUnstructuredGrid) +from vtkmodules.vtkCommonExecutionModel import vtkAlgorithm, vtkStreamingDemandDrivenPipeline +from vtkmodules.vtkFiltersCore import vtkThreshold +from vtkmodules.vtkFiltersExtraction import vtkExtractSelection +from vtkmodules.vtkFiltersGeometry import vtkGeometryFilter def _fspm_tf(): @@ -392,6 +401,153 @@ def frame_at(self, time: float = None): return self._cache[number] +# ── Selection by scale ─────────────────────────────────────────────────────── + +SELECT_AT_SCALE = "fspmTFSelectAtScale" # the filter's proxy name +SELECTED = "fspm_tf_selected" # the cell array _kept thresholds + + +def leaves(data) -> list: + """The data sets of *data*, a data set or a composite one, in its tree's order.""" + if data is None: + return [] + if not isinstance(data, vtkDataObjectTree): + return [data] + iterator = data.NewTreeIterator() + iterator.VisitOnlyLeavesOn() + iterator.InitTraversal() + found = [] + while not iterator.IsDoneWithTraversal(): + found.append(iterator.GetCurrentDataObject()) + iterator.GoToNextItem() + return found + + +def _texts(data_set, name: str) -> list: + array = data_set.GetFieldData().GetAbstractArray(name) + return [array.GetValue(number) for number in range(array.GetNumberOfValues())] if array is not None else [] + + +def _id_arrays(data_set) -> dict: + """{".id": values} of a data set's cells.""" + cell_data = data_set.GetCellData() + names = [cell_data.GetArrayName(number) for number in range(cell_data.GetNumberOfArrays())] + return {name: vtk_to_numpy(cell_data.GetArray(name)) for name in names if name and name.endswith(".id")} + + +def fspm_tf_data(data) -> bool: + """Whether *data* holds blocks of the fspm-TF reader (their data structure in field data).""" + return any(_texts(leaf, _views().DATA_STRUCTURE_FIELD) for leaf in leaves(data)) + + +def scales_of(data) -> list: + """The scales of *data*'s blocks (their field data fspm_tf_scales), coarser first.""" + return _views().scale_names([_texts(leaf, _views().SCALES_FIELD) for leaf in leaves(data)]) + + +def picked_cells(data, selection) -> list: + """[(data structure, {".id": values})] of the cells of *data* (the reader's output or an item drawn from + it: an ExtractBlock, a Tube ...) in *selection* (a vtkSelection: ids, a frustum, a query ...), one entry per block + holding some.""" + if data is None or selection is None or not selection.GetNumberOfNodes(): + return [] + copied, chosen = data.NewInstance(), vtkSelection() + copied.ShallowCopy(data) + chosen.ShallowCopy(selection) + extraction = vtkExtractSelection() + extraction.PreserveTopologyOn() # the same blocks, each cell marked inside or not + extraction.SetInputData(0, copied) + extraction.SetInputData(1, chosen) + extraction.Update() + marked_data, found = extraction.GetOutputDataObject(0), [] + if isinstance(data, vtkDataObjectTree): # each block and its marks (a block without any: none) + iterator = data.NewTreeIterator() + iterator.InitTraversal() + pairs = [] + while not iterator.IsDoneWithTraversal(): + pairs.append((iterator.GetCurrentDataObject(), marked_data.GetDataSet(iterator))) + iterator.GoToNextItem() + else: + pairs = [(data, marked_data)] + for leaf, marked in pairs: + inside = marked.GetCellData().GetArray("vtkInsidedness") if marked is not None else None + data_structure = _texts(leaf, _views().DATA_STRUCTURE_FIELD) + if inside is None or not data_structure: + continue + cells = vtk_to_numpy(inside).astype(bool) + if cells.any(): + found.append((data_structure[0], {name: values[cells] for name, values in _id_arrays(leaf).items()})) + return found + + +def _kept(data_set, chosen: dict, scale: str): + """The cells of *data_set* inside the entities *chosen* at *scale* (a grid layer's hidden cells included), its + field data and active scalars kept; polygonal data stays polygonal (a Tube can follow).""" + views = _views() + data_structure = (_texts(data_set, views.DATA_STRUCTURE_FIELD) or [""])[0] + mask = views.selection_mask(data_structure, _id_arrays(data_set), chosen, scale) + if mask.size != data_set.GetNumberOfCells() or not mask.any(): # a block without id arrays, or none in it + empty = data_set.NewInstance() + empty.GetFieldData().ShallowCopy(data_set.GetFieldData()) + return empty + marked = data_set.NewInstance() + marked.ShallowCopy(data_set) + marked.GetCellData().RemoveArray(vtkDataSetAttributes.GhostArrayName()) + marked.GetCellData().AddArray(_array(SELECTED, mask.astype(np.uint8))) + threshold = vtkThreshold() + threshold.SetInputData(marked) + threshold.SetInputArrayToProcess(0, 0, 0, vtkDataObject.FIELD_ASSOCIATION_CELLS, SELECTED) + threshold.SetThresholdFunction(vtkThreshold.THRESHOLD_UPPER) + threshold.SetUpperThreshold(0.5) + threshold.SetLowerThreshold(0.5) + last = threshold + if isinstance(data_set, vtkPolyData): + last = vtkGeometryFilter() + last.SetInputConnection(threshold.GetOutputPort()) + last.Update() + output = last.GetOutput() + output.GetCellData().RemoveArray(SELECTED) + scalars = data_set.GetCellData().GetScalars() + if scalars is not None and scalars.GetName(): + output.GetCellData().SetActiveScalars(scalars.GetName()) + output.GetFieldData().ShallowCopy(data_set.GetFieldData()) + return output + + +def grow_selection(frame_data, picked: list, scale: str) -> vtkMultiBlockDataSet: + """The cells of *frame_data* (the reader's output: its blocks, a layer's hidden ones included) inside the entities + at *scale* containing the *picked* cells (picked_cells'), with everything they contain at finer scales: the same + blocks, named alike, each holding its part (empty when none). A ValueError when no picked cell has an entity at + *scale*.""" + chosen = _views().picked_entities(picked, scale) + output = vtkMultiBlockDataSet() + if isinstance(frame_data, vtkDataObjectTree): + output.CopyStructure(frame_data) + iterator = frame_data.NewTreeIterator() + iterator.InitTraversal() + while not iterator.IsDoneWithTraversal(): + output.SetDataSet(iterator, _kept(iterator.GetCurrentDataObject(), chosen, scale)) + iterator.GoToNextItem() + elif frame_data is not None: + output.SetBlock(0, _kept(frame_data, chosen, scale)) + return output + + +def frame_upstream(algorithm, port: int = 0): + """The output of the fspm-TF reader upstream of *algorithm*'s input *port*, through the items drawn from it (the + farthest output holding fspm-TF blocks): where selection by scale grows, in every layer, the hidden ones included; + None when there is none.""" + found = None + while algorithm.GetNumberOfInputPorts() > port and algorithm.GetNumberOfInputConnections(port): + connection = algorithm.GetInputConnection(port, 0) + producer = connection.GetProducer() + data = producer.GetOutputDataObject(connection.GetIndex()) + if fspm_tf_data(data): + found = data + algorithm, port = producer, 0 + return found + + # ── The ParaView reader ────────────────────────────────────────────────────── try: @@ -404,7 +560,7 @@ def frame_at(self, time: float = None): _fspm_tf() # on ParaView's path now: the views macros import it too except ImportError: pass - if os.path.dirname(os.path.abspath(__file__)) not in sys.path: # and fspm_tf_selection, the selection macros' + if os.path.dirname(os.path.abspath(__file__)) not in sys.path: # and fspm_tf_selection, the selection macro's sys.path.append(os.path.dirname(os.path.abspath(__file__))) @smproxy.reader(name="fspmTFReader", label="fspm-TF reader", extensions="glb json", @@ -499,3 +655,91 @@ def RequestData(self, request, in_info, out_info): if time is not None: output.GetInformation().Set(output.DATA_TIME_STEP(), frame.time) return 1 + + @smproxy.filter(name=SELECT_AT_SCALE, label="fspm-TF Select at scale") + @smhint.xml("""""") + @smhint.xml("""""") + @smhint.xml("""""") + @smproperty.xml(""" + + + + + The cells picked in the view: ParaView's selection when the filter is made + ("Copy Active Selection" to take another). + + """) + @smproperty.xml(""" + + + + + The item the cells were picked on: the fspm-TF reader, a layer, a selection + ... + """) + class FspmTFSelectAtScale(VTKPythonAlgorithmBase): + """The cells picked on its input grown to their entity at Scale (a plant, an axis, an organ, a segment ...) + with everything it contains at finer scales, in every block of the fspm-TF reader upstream (every layer, the + hidden ones included: a segment's section). The picked entities are kept across time steps (the plant + followed as it grows) until the selection changes; Scale re-grows them.""" + + def __init__(self): + VTKPythonAlgorithmBase.__init__(self, nInputPorts=2, nOutputPorts=1, outputType="vtkMultiBlockDataSet") + self._scale, self._scales = "", [] + self._picked, self._picked_from = [], None + + def FillInputPortInformation(self, port, info): + if port == 1: + info.Set(vtkAlgorithm.INPUT_REQUIRED_DATA_TYPE(), "vtkSelection") + info.Set(vtkAlgorithm.INPUT_IS_OPTIONAL(), 1) + else: + info.Set(vtkAlgorithm.INPUT_REQUIRED_DATA_TYPE(), "vtkDataObject") + return 1 + + @smproperty.stringvector(name="ScaleNames", information_only="1") + def GetScaleNames(self): + return self._scales + + @smproperty.stringvector(name="Scale", number_of_elements="1", default_values="") + @smdomain.xml(""" + + + + """) + def SetScale(self, name): + """The scale the picked cells grow to (empty: the coarsest, e.g. Plant).""" + if (name or "") != self._scale: + self._scale = name or "" + self.Modified() + + def RequestInformation(self, request, in_info, out_info): + frame_data = frame_upstream(self) + if frame_data is None: + frame_data = in_info[0].GetInformationObject(0).Get(vtkDataObject.DATA_OBJECT()) + self._scales = scales_of(frame_data) + return 1 + + def RequestData(self, request, in_info, out_info): + output = vtkMultiBlockDataSet.GetData(out_info, 0) + source = in_info[0].GetInformationObject(0).Get(vtkDataObject.DATA_OBJECT()) + selection = vtkSelection.GetData(in_info[1], 0) if in_info[1].GetNumberOfInformationObjects() else None + frame_data = frame_upstream(self) or source + self._scales = scales_of(frame_data) + picked_from = (selection.GetMTime() if selection is not None else None, + self.GetInputConnection(0, 0).GetProducer().GetAddressAsString("vtkObject")) + if picked_from != self._picked_from: # a new selection: its entities picked at this time, + self._picked = picked_cells(source, selection) # then followed through the time steps + self._picked_from = picked_from + scale = self._scale or (self._scales[0] if self._scales else "") + try: + output.ShallowCopy(grow_selection(frame_data, self._picked, scale)) + except ValueError as error: + vtkOutputWindow.GetInstance().DisplayWarningText(f"fspm-TF Select at scale: {error}\n") + empty = vtkMultiBlockDataSet() + if isinstance(frame_data, vtkDataObjectTree): + empty.CopyStructure(frame_data) + output.ShallowCopy(empty) + if source is not None and source.GetInformation().Has(vtkDataObject.DATA_TIME_STEP()): + output.GetInformation().Set(vtkDataObject.DATA_TIME_STEP(), + source.GetInformation().Get(vtkDataObject.DATA_TIME_STEP())) + return 1 diff --git a/paraview/fspm_tf_selection.py b/paraview/fspm_tf_selection.py index 9019190..f92fda8 100644 --- a/paraview/fspm_tf_selection.py +++ b/paraview/fspm_tf_selection.py @@ -1,28 +1,25 @@ """ -Selection by scale in ParaView, the glue of the macros select_at_scale.py and clear_fspm_tf_selection.py: the cells -picked in a view (on any fspm-TF item: a layer that "Apply fspm-TF views" made, or the reader) grown to their entity -at a scale (a plant, an axis, an organ, a segment ...) with everything it contains at finer scales, in every layer of -the same reader (hidden ones too: a segment's section), shown +The glue of the macro show_fspm_tf_selection.py: an "fspm-TF Select at scale" item (the plugin's filter: the cells +picked in a view grown to their entity at a scale, in every layer of the reader upstream, hidden ones included) shown +as its layers, each part an item drawn as its layer (the same colour map, solid colour, opacity, representation, a +Tube for segments), and - "isolate" in the view, the other items hidden until the selection is cleared; + "isolate" in the view, the other items hidden until it is put back; "panel" in a new render view beside it, its camera fit to the selection, the view itself untouched. -Each layer's part is an item " selection" (a Programmable Filter calling extract below), drawn as its layer. -The view keeps what the selection made and hid (its annotation fspm_tf_selection), for clear_selection. +Run again, the macro goes back one step: the items it made removed, its panel closed, what it hid shown again. A +selection made on a shown selection's items (picking an axis of the plant shown) is shown over it, and going back +returns to the plant. The view keeps what each step made and hid (its annotation fspm_tf_selection). -The plugin (fspm_tf_reader.py) puts this folder and openalea.fspm_tf on ParaView's path. What is selected is -openalea.fspm_tf.views' (picked_entities, selection_mask, isolated), tested without ParaView. +The plugin (fspm_tf_reader.py) puts this folder and openalea.fspm_tf on ParaView's path. Which items to hide is +openalea.fspm_tf.views.isolated, tested without ParaView; which cells are selected, the filter's. """ import json +import time -import numpy as np -from vtkmodules.util.numpy_support import numpy_to_vtk, vtk_to_numpy -from vtkmodules.vtkCommonDataModel import vtkDataObject, vtkDataObjectTree, vtkDataSetAttributes -from vtkmodules.vtkFiltersCore import vtkThreshold - -STATE = "fspm_tf_selection" # the view's annotation: {"items", "hidden", "scalar_bars", "panel", ...} +STATE = "fspm_tf_selection" # the view's annotation: [{"selection", "items", "hidden", "panel", ...}] SUFFIX = " selection" -MARK = "fspm_tf_selected" # the cell array extract thresholds +SELECT_AT_SCALE = "fspmTFSelectAtScale" # the filter's proxy name (fspm_tf_reader.SELECT_AT_SCALE) def _views(): @@ -30,123 +27,43 @@ def _views(): return views -def _leaves(data): - """The data sets of *data*, a data set or a composite one.""" - if data is None: - return [] - if not isinstance(data, vtkDataObjectTree): - return [data] - iterator = data.NewTreeIterator() - iterator.InitTraversal() - found = [] - while not iterator.IsDoneWithTraversal(): - found.append(iterator.GetCurrentDataObject()) - iterator.GoToNextItem() - return found - - -def _texts(data_set, name: str) -> list: - array = data_set.GetFieldData().GetAbstractArray(name) - return [array.GetValue(number) for number in range(array.GetNumberOfValues())] if array is not None else [] - - -def _id_arrays(data_set) -> dict: - cell_data = data_set.GetCellData() - names = [cell_data.GetArrayName(number) for number in range(cell_data.GetNumberOfArrays())] - return {name: vtk_to_numpy(cell_data.GetArray(name)) for name in names if name and name.endswith(".id")} +def _simple(): + from paraview import simple + return simple -# ── On the server: a layer's selected cells ────────────────────────────────── +def _name_of(item) -> str: + from paraview import servermanager + return servermanager.ProxyManager().GetProxyName("sources", item) -def _kept(data_set, chosen: dict, scale: str): - views = _views() - data_structure = (_texts(data_set, views.DATA_STRUCTURE_FIELD) or [""])[0] - mask = views.selection_mask(data_structure, _id_arrays(data_set), chosen, scale) - if mask.size != data_set.GetNumberOfCells(): # a block without id arrays - mask = np.zeros(data_set.GetNumberOfCells(), dtype=bool) - marked = data_set.NewInstance() - marked.ShallowCopy(data_set) - marked.GetCellData().RemoveArray(vtkDataSetAttributes.GhostArrayName()) # a grid layer's hidden cells - marked.GetCellData().AddArray(_named(numpy_to_vtk(mask.astype(np.uint8), deep=True), MARK)) - threshold = vtkThreshold() - threshold.SetInputData(marked) - threshold.SetInputArrayToProcess(0, 0, 0, vtkDataObject.FIELD_ASSOCIATION_CELLS, MARK) - threshold.SetThresholdFunction(vtkThreshold.THRESHOLD_UPPER) - threshold.SetUpperThreshold(0.5) - threshold.SetLowerThreshold(0.5) - threshold.Update() - output = threshold.GetOutput() - output.GetCellData().RemoveArray(MARK) - output.GetFieldData().ShallowCopy(data_set.GetFieldData()) - return output - - -def _named(array, name: str): - array.SetName(name) - return array - - -def extract(algorithm, chosen: dict, scale: str) -> None: - """A Programmable Filter's RequestData: its input's cells inside the entities *chosen* at *scale*, in each - block.""" - source, output = algorithm.GetInputDataObject(0, 0), algorithm.GetOutputDataObject(0) - if isinstance(source, vtkDataObjectTree): - output.CopyStructure(source) - iterator = source.NewTreeIterator() - iterator.InitTraversal() - while not iterator.IsDoneWithTraversal(): - output.SetDataSet(iterator, _kept(iterator.GetCurrentDataObject(), chosen, scale)) - iterator.GoToNextItem() - else: - output.ShallowCopy(_kept(source, chosen, scale)) +def is_selection(item) -> bool: + """Whether *item* is an "fspm-TF Select at scale" item.""" + return item is not None and item.SMProxy.GetXMLName() == SELECT_AT_SCALE -# ── On the client: the macros ──────────────────────────────────────────────── -def _simple(): - from paraview import simple - return simple +def upstream(item) -> list: + """The items *item* is drawn from, nearest first (through each one's Input).""" + found = [] + while item is not None and "Input" in item.ListProperties(): + item = item.Input[0] if isinstance(item.Input, (list, tuple)) else item.Input + if item is not None: + found.append(item) + return found -def fspm_tf_items() -> dict: - """{reader: [(name, item)]} of what shows each fspm-TF reader: its layer items when "Apply fspm-TF views" made - them, else the reader.""" - simple, views = _simple(), _views() - found = {} - for (name, _), item in simple.GetSources().items(): - if not (item.ListProperties() and "ConfigurationNames" in item.ListProperties()): - continue # not an fspm-TF reader - path = views.views_path(views.property_text(item.FileName), views.property_text(item.Views) or None) - layers = [] - if path is not None: - _, configuration = views.configuration(views.read_views(path), - views.property_text(item.Configuration) or None) - layers = [(layer["name"], simple.FindSource(layer["name"])) for layer in configuration] - layers = [(layer, proxy) for layer, proxy in layers if proxy is not None] - found[name] = layers or [(name, item)] - return found +def reader_of(item): + """The fspm-TF reader *item* is drawn from (None: none).""" + return next((proxy for proxy in [item] + upstream(item) if "ConfigurationNames" in proxy.ListProperties()), None) -def picked(items: list) -> tuple: - """[(data structure, {".id": values})] of the cells picked on *items* ([(name, item)]), one entry per - block; and the scales they are in.""" - simple = _simple() - from paraview import servermanager - found, scales = [], [] - for _, item in items: - selection = item.SMProxy.GetSelectionInput(0) - if selection is None: - continue - extraction = simple.ExtractSelection(Input=item, Selection=servermanager._getPyProxy(selection)) - data = servermanager.Fetch(extraction) - simple.Delete(extraction) - simple.ClearSelection(item) # used: the next pick starts afresh - for leaf in _leaves(data): - if leaf.GetNumberOfCells(): - views = _views() - found.append(((_texts(leaf, views.DATA_STRUCTURE_FIELD) or [""])[0], _id_arrays(leaf))) - scales += [name for name in _texts(leaf, views.SCALES_FIELD) if name not in scales] - return found, scales +def layers_of(reader) -> list: + """The layers of *reader*'s configuration ([]: a reader without views).""" + views = _views() + path = views.views_path(views.property_text(reader.FileName), views.property_text(reader.Views) or None) + if path is None: + return [] + return views.configuration(views.read_views(path), views.property_text(reader.Configuration) or None)[1] def _shown(view) -> dict: @@ -162,9 +79,32 @@ def _shown(view) -> dict: return found -def _state(view) -> dict: +def _named_views() -> dict: + """{registration name: view} of the session's views.""" + from paraview import servermanager + return {name: view for (name, _), view in servermanager.ProxyManager().GetProxiesInGroup("views").items()} + + +def _view_name(view) -> str: + return next(name for name, proxy in _named_views().items() if proxy.SMProxy is view.SMProxy) + + +def _steps(view) -> list: proxy = view.SMProxy - return json.loads(proxy.GetAnnotation(STATE)) if proxy.HasAnnotation(STATE) else None + return json.loads(proxy.GetAnnotation(STATE)) if proxy.HasAnnotation(STATE) else [] + + +def _keep_steps(view, steps: list) -> None: + if steps: + view.SMProxy.SetAnnotation(STATE, json.dumps(steps)) + elif view.SMProxy.HasAnnotation(STATE): + view.SMProxy.RemoveAnnotation(STATE) + + +def shown_steps() -> list: + """[(view, step)] of every selection shown, the last shown last.""" + found = [(view, step) for view in _named_views().values() for step in _steps(view)] + return sorted(found, key=lambda pair: pair[1]["order"]) def _draw_as(display, layer_display) -> bool: @@ -184,59 +124,82 @@ def _draw_as(display, layer_display) -> bool: return coloured -def select_at_scale(scale: str, mode: str = "isolate", view=None) -> list: - """Grow the picked cells to their entities at *scale* and show them (mode "isolate" or "panel"); the names of - the selection's items. A previous selection is cleared first.""" +def _draw_as_layer(display, layer: dict, arrays: set) -> bool: + """*display* drawn as *layer* says, for a layer without its item (the views not applied); whether coloured.""" + simple, settings = _simple(), _views().display_settings(layer, arrays) + display.SetRepresentationType(settings["representation"]) + display.Opacity = settings["opacity"] + if settings["color_array"]: + display.ColorArrayName = ["CELLS", settings["color_array"]] + display.LookupTable = simple.GetColorTransferFunction(settings["color_array"]) + return True + display.ColorArrayName = ["POINTS", ""] + display.AmbientColor = display.DiffuseColor = settings["solid_color"] + return False + + +def _part(selection, layer: dict, name: str, made: list): + """The item of *layer*'s part of *selection*: an ExtractBlock of its block, and a Tube for a segments layer.""" + simple, settings = _simple(), _views().display_settings(layer) + tube = settings["tube"] + block = simple.ExtractBlock(registrationName=f"{name} block" if tube else name, Input=selection, + Selectors=[settings["selector"]]) + made.append(f"{name} block" if tube else name) + if not tube: + return block + item = simple.Tube(registrationName=name, Input=block) + item.Scalars = ["POINTS", tube["radius_array"]] + item.VaryRadius = "By Absolute Scalar" + made.append(name) + return item + + +def show_selection(selection, mode: str = "isolate", view=None) -> list: + """Show the "fspm-TF Select at scale" item *selection* as its layers (mode "isolate" or "panel"); the names of the + items made. Shown selections it is not drawn from are put back first.""" simple, views = _simple(), _views() if mode not in ("isolate", "panel"): raise ValueError(f"MODE: 'isolate' or 'panel', not {mode!r}") + if not is_selection(selection): + raise RuntimeError("select an 'fspm-TF Select at scale' item in the Pipeline Browser first (pick cells, then " + "Filters > fspm-TF > fspm-TF Select at scale, Apply)") view = view or simple.GetActiveViewOrCreate("RenderView") - readers = fspm_tf_items() - previous = _state(view) or {} - cells, scales, items = [], [], [] - for reader, shown in readers.items(): # the readers picked on (a selection's items: its readers) - made = [(name, simple.FindSource(f"{name}{SUFFIX}")) for name, _ in shown - if f"{name}{SUFFIX}" in previous.get("items", [])] - found, its_scales = picked(shown + [(name, item) for name, item in made if item is not None]) - if found: - cells += found - scales += [name for name in its_scales if name not in scales] - items += shown - if scales and scale not in scales: - raise ValueError(f"SCALE '{scale}' is not a scale of the picked elements: {scales}") - chosen = views.picked_entities(cells, scale) - clear_selection() - target = view - if mode == "panel": - target = simple.CreateView("RenderView") - layout = simple.GetLayout(view) - if layout is None: - layout = simple.CreateLayout("fspm-TF") - layout.AssignView(0, view) - location = layout.GetViewLocation(view) - layout.SplitHorizontal(location, 0.5) - layout.AssignView(2 * location + 2, target) - target.ViewSize = list(view.ViewSize) - target.CameraPosition, target.CameraFocalPoint = list(view.CameraPosition), list(view.CameraFocalPoint) - target.CameraViewUp = list(view.CameraViewUp) - target.ViewTime = view.ViewTime - made = [] - for name, item in items: - layer_display = simple.GetDisplayProperties(item, view) - part = simple.ProgrammableFilter(registrationName=f"{name}{SUFFIX}", Input=item) - part.OutputDataSetType = "Same as Input" - part.Script = f"import fspm_tf_selection\nfspm_tf_selection.extract(self, {chosen!r}, {scale!r})" - part.UpdatePipeline() - display = simple.Show(part, target) - coloured = _draw_as(display, layer_display) - if not part.GetDataInformation().GetNumberOfCells(): - simple.Hide(part, target) # nothing of the selection in this layer - elif coloured and mode == "panel": + above = {_name_of(item) for item in upstream(selection)} + while shown_steps() and not above & set(shown_steps()[-1][1]["items"]): + back() # not drawn from the last selection shown: put it back + reader = reader_of(selection) + if reader is None: + raise RuntimeError("this selection is not drawn from an fspm-TF reader") + depth = 1 + len(shown_steps()) + suffix = SUFFIX if depth == 1 else f"{SUFFIX} {depth}" + selection.UpdatePipeline(view.ViewTime) + target = _panel(view) if mode == "panel" else view + simple.Hide(selection, view) # its own display (Apply shows it): its layers instead + simple.ClearSelection(selection.Input) # the picked cells' highlight (the filter keeps a copy) + made, parts = [], [] + layers = layers_of(reader) + for layer in layers: + item = _part(selection, layer, f"{layer['name']}{suffix}", made) + layer_item = simple.FindSource(layer["name"]) + parts.append((item, simple.GetDisplayProperties(layer_item, view) if layer_item is not None else layer)) + if not layers: # the reader without views: drawn as the reader + item = simple.PassArrays(registrationName=f"{_name_of(reader)}{suffix}", Input=selection) + made.append(f"{_name_of(reader)}{suffix}") + parts.append((item, simple.GetDisplayProperties(reader, view))) + for item, drawn in parts: + item.UpdatePipeline(view.ViewTime) + display = simple.Show(item, target) + if isinstance(drawn, dict): + coloured = _draw_as_layer(display, drawn, set(item.CellData.keys())) + else: + coloured = _draw_as(display, drawn) + if coloured and mode == "panel" and item.GetDataInformation().GetNumberOfCells(): # (a layer without any + # of it shown too, empty: its part may come with another + # Scale or time step) display.SetScalarBarVisibility(target, True) - made.append(f"{name}{SUFFIX}") hidden, bars = [], [] if mode == "isolate": - hidden = views.isolated(_shown(view), made) + hidden = views.isolated(_shown(view), made + [_name_of(selection)]) for name in hidden: display = simple.GetDisplayProperties(simple.FindSource(name), view) if display.LookupTable is not None and simple.GetScalarBar(display.LookupTable, view).Visibility: @@ -246,51 +209,89 @@ def select_at_scale(scale: str, mode: str = "isolate", view=None) -> list: else: target.ResetCamera() simple.Render(target) - state = {"items": made, "hidden": hidden, "scalar_bars": bars, "scale": scale, "chosen": chosen, - "panel": None if mode == "isolate" else _view_name(target)} - view.SMProxy.SetAnnotation(STATE, json.dumps(state)) + step = {"selection": _name_of(selection), "items": made, "hidden": hidden, "scalar_bars": bars, + "panel": None if mode == "isolate" else _view_name(target), "order": time.time()} + _keep_steps(view, _steps(view) + [step]) simple.SetActiveView(view) simple.Render(view) - print(f"fspm-TF selection at {scale}: {chosen} ({mode})") + print(f"fspm-TF selection shown ({mode}): {_name_of(selection)}, scale {views.property_text(selection.Scale)}") return made -def _named_views() -> dict: - """{registration name: view} of the session's views.""" - from paraview import servermanager - return {name: view for (name, _), view in servermanager.ProxyManager().GetProxiesInGroup("views").items()} - - -def _view_name(view) -> str: - return next(name for name, proxy in _named_views().items() if proxy.SMProxy is view.SMProxy) +def _panel(view): + """A new render view beside *view*, at its time, its camera to start with.""" + simple = _simple() + panel = simple.CreateView("RenderView") + layout = simple.GetLayout(view) + if layout is None: + layout = simple.CreateLayout("fspm-TF") + layout.AssignView(0, view) + location = layout.GetViewLocation(view) + layout.SplitHorizontal(location, 0.5) + layout.AssignView(2 * location + 2, panel) + panel.ViewSize = list(view.ViewSize) + panel.CameraPosition, panel.CameraFocalPoint = list(view.CameraPosition), list(view.CameraFocalPoint) + panel.CameraViewUp = list(view.CameraViewUp) + panel.ViewTime = view.ViewTime + return panel + + +def _consumed(item) -> bool: + """Whether another pipeline item reads *item* (a selection made on it): then it is kept, hidden.""" + proxy = item.SMProxy + for number in range(proxy.GetNumberOfConsumers()): + consumer = proxy.GetConsumerProxy(number) + if consumer is not None and consumer.GetXMLGroup() in ("sources", "filters"): + return True + return False + + +def back() -> bool: + """Put back the last selection shown: its items removed (kept hidden when a selection was made on them), its + panel closed, what it hid shown again; whether there was one.""" + simple = _simple() + steps = shown_steps() + if not steps: + return False + view, step = steps[-1] + for name in reversed(step["items"]): # a tube before the block it reads + item = simple.FindSource(name) + if item is None: + continue + if _consumed(item): + for shown in _named_views().values(): + simple.Hide(item, shown) + else: + simple.Delete(item) + for name in step["hidden"]: + item = simple.FindSource(name) + if item is not None: + display = simple.Show(item, view) + if name in step.get("scalar_bars", []): + display.SetScalarBarVisibility(view, True) + if step["panel"]: + panel = _named_views().get(step["panel"]) + if panel is not None: + layout = simple.GetLayout(panel) + if layout is not None: + location = layout.GetViewLocation(panel) + layout.RemoveView(panel) + if location > 0: + layout.Collapse(location) + simple.Delete(panel) + _keep_steps(view, [kept for kept in _steps(view) if kept["order"] != step["order"]]) + simple.Render(view) + print(f"fspm-TF selection put back: {step['selection']}") + return True -def clear_selection() -> None: - """Remove the selection's items and panel, and show again what it hid, in every view holding one.""" +def show_or_back(mode: str = "isolate") -> None: + """The macro: the active item, an "fspm-TF Select at scale" not shown yet, shown; else the last selection shown + put back.""" simple = _simple() - for view in _named_views().values(): - state = _state(view) - if state is None: - continue - for name in state["items"]: - item = simple.FindSource(name) - if item is not None: - simple.Delete(item) - for name in state["hidden"]: - item = simple.FindSource(name) - if item is not None: - display = simple.Show(item, view) - if name in state.get("scalar_bars", []): - display.SetScalarBarVisibility(view, True) - if state["panel"]: - panel = _named_views().get(state["panel"]) - if panel is not None: - layout = simple.GetLayout(panel) - if layout is not None: - location = layout.GetViewLocation(panel) - layout.RemoveView(panel) - if location > 0: - layout.Collapse(location) - simple.Delete(panel) - view.SMProxy.RemoveAnnotation(STATE) - simple.Render(view) + active = simple.GetActiveSource() + shown = {step["selection"] for _, step in shown_steps()} + if is_selection(active) and _name_of(active) not in shown: + show_selection(active, mode) + elif not back(): + show_selection(active, mode) # nothing to put back: says what to select diff --git a/paraview/select_at_scale.py b/paraview/select_at_scale.py deleted file mode 100644 index 572635a..0000000 --- a/paraview/select_at_scale.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -ParaView macro "Select at scale": the cells picked in the view (Select Cells On, on any fspm-TF layer or reader) grown -to their entity at SCALE (a plant, an axis, an organ, a segment ...) with everything it contains at finer scales, in -every layer, hidden ones included (a segment's section); shown as MODE says: - - "isolate" in the view, the other items hidden (Macros > clear_fspm_tf_selection shows them again); - "panel" in a new render view beside it, its own camera fit to the selection; the view itself untouched - (clear_fspm_tf_selection closes the panel). - - Macros > Import new macro..., choose this file (and clear_fspm_tf_selection.py); set SCALE and MODE below - (Macros > Edit...); load the fspm-TF reader plugin, pick cells, then Macros > select_at_scale. - -The scales are the frame's entity sets (Plant, Axis, Organ, SubOrgan, ... Compartment); an unknown one is reported -with the picked elements' scales. The glue is fspm_tf_selection.py (next to the plugin, on ParaView's path once it -is loaded); what is selected, openalea.fspm_tf.views' picked_entities and selection_mask, tested without ParaView. -""" -SCALE = "Plant" -MODE = "isolate" # or "panel" - -try: - import fspm_tf_selection -except ImportError: - raise RuntimeError("fspm_tf_selection is not on ParaView's path: load the fspm-TF reader plugin first") from None - -fspm_tf_selection.select_at_scale(SCALE, MODE) diff --git a/paraview/show_fspm_tf_selection.py b/paraview/show_fspm_tf_selection.py new file mode 100644 index 0000000..127e55d --- /dev/null +++ b/paraview/show_fspm_tf_selection.py @@ -0,0 +1,27 @@ +""" +ParaView macro "Show fspm-TF selection": the selected "fspm-TF Select at scale" item (Filters > fspm-TF, applied with +cells picked) shown as its layers, each part drawn as its layer (the same colour map, a Tube for segments), as MODE +says: + + "isolate" in the view, the other items hidden; + "panel" in a new render view beside it, its own camera fit to the selection; the view itself untouched. + +Run again, it goes back: the items it made removed, the panel closed, the hidden items shown again (then delete the +filter to drop the selection). A selection made on the items shown (an axis of the plant shown) is shown over it, and +going back returns to the plant. + + Macros > Import new macro..., choose this file; set MODE below (Macros > Edit...). Load the fspm-TF reader + plugin, pick cells, Filters > fspm-TF > fspm-TF Select at scale, Apply, then Macros > show_fspm_tf_selection. + +The filter alone needs no macro: its output is the selection, every layer's part as a block. This macro does what a +filter cannot: hide the other items, open a panel. The glue is fspm_tf_selection.py (next to the plugin, on +ParaView's path once it is loaded). +""" +MODE = "isolate" # or "panel" + +try: + import fspm_tf_selection +except ImportError: + raise RuntimeError("fspm_tf_selection is not on ParaView's path: load the fspm-TF reader plugin first") from None + +fspm_tf_selection.show_or_back(MODE) From 203ecade14030db24b0215ce28c1fa5e3d50ff6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Wed, 4 Nov 2026 07:50:27 +0100 Subject: [PATCH 26/39] check_in_paraview --design finds the sections layer by its source (a display mesh), not its name --- paraview/check_in_paraview.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/paraview/check_in_paraview.py b/paraview/check_in_paraview.py index a2f8ff5..ed95f05 100644 --- a/paraview/check_in_paraview.py +++ b/paraview/check_in_paraview.py @@ -207,6 +207,13 @@ def displayed(view, name: str) -> dict: "representation": display.GetProperty("Representation").GetData()} +def display_mesh(layer: dict) -> bool: + """Whether *layer* shows a component's display mesh (e.g. sections: a mesh other than the shoot's triangles), + whatever its name (" ").""" + source = layer["source"] + return source["kind"] == "mesh" and source.get("mesh") != "triangles" + + def check_design(index_path: str) -> None: """Views and selection by scale on a sequence with its views.json, screenshots in *pictures*.""" import json @@ -221,7 +228,7 @@ def check_design(index_path: str) -> None: document = views.read_views(views_file) name, layers = views.configuration(document) roots_only = [dict(layer, visible=True) for layer in layers if "roots" in layer["name"] - or "Section" in layer["name"]] # a second configuration: the sections shown, the + or display_mesh(layer)] # a second configuration: the sections shown, the for layer in roots_only: # roots by radius, a range, a log scale, a colour map if layer.get("tube"): layer.update(color_by={"entity_set": "SubOrgan", "variable": "radius"}, range=[0.002, 0.005], @@ -264,7 +271,7 @@ def check_design(index_path: str) -> None: screenshot(view, "2_views_shoot_opacity_0.3") roots = next(layer["name"] for layer in layers if layer["name"].endswith("roots")) - sections = next(layer["name"] for layer in layers if "Section" in layer["name"]) + sections = next(layer["name"] for layer in layers if display_mesh(layer)) roots_item = simple.FindSource(roots) data = servermanager.Fetch(roots_item) leaf = data.GetBlock(0) if hasattr(data, "GetNumberOfBlocks") else data From a4f94f45ce17702ae39606d53253fa18a7d47e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Wed, 4 Nov 2026 14:01:51 +0100 Subject: [PATCH 27/39] Renamed fspm-TF to encoding3d everywhere: the Python package openalea.encoding3d and the encoding3d command, the Julia package encoding3d (and its extensions), the C++ library encoding3d (namespace, headers, targets, tools), the Java packages org.openalea.encoding3d (Encoding3dException, the GroIMP plugin encoding3d and Encoding3dExport), ENCODING3D_ environment variables and CMake options, the ParaView reader and macros, the docs and the URLs; the conformance files written again (their generator) --- .github/actions/conformance/action.yml | 22 ++--- .github/workflows/conformance.yml | 10 +- .github/workflows/cpp-codec.yml | 18 ++-- .github/workflows/java-codec.yml | 16 ++-- .github/workflows/julia-codec.yml | 26 +++--- CHANGELOG.md | 36 ++++---- CODE_OF_CONDUCT.md | 2 +- CONTRIBUTING.md | 4 +- README.md | 42 ++++----- bindings/cpp/CMakeLists.txt | 82 ++++++++--------- bindings/cpp/README.md | 16 ++-- bindings/cpp/cmake/encoding3dConfig.cmake.in | 6 ++ bindings/cpp/cmake/fspm_tfConfig.cmake.in | 6 -- bindings/cpp/cplantbox/CMakeLists.txt | 30 +++--- .../{fspm_tf => encoding3d}/cplantbox.hpp | 12 +-- bindings/cpp/cplantbox/src/cplantbox.cpp | 12 +-- .../cpp/cplantbox/test/test_cplantbox.cpp | 10 +- .../{fspm_tf => encoding3d}/coupling.hpp | 18 ++-- .../fspm_tf.hpp => encoding3d/encoding3d.hpp} | 12 +-- .../include/{fspm_tf => encoding3d}/model.hpp | 8 +- bindings/cpp/src/bytes.hpp | 6 +- bindings/cpp/src/codec.cpp | 24 ++--- bindings/cpp/src/coupling.cpp | 20 ++-- bindings/cpp/src/dump.cpp | 8 +- bindings/cpp/src/glb.cpp | 6 +- bindings/cpp/src/model.cpp | 6 +- bindings/cpp/src/sequence.cpp | 6 +- .../{test_fspm_tf.cpp => test_encoding3d.cpp} | 10 +- bindings/cpp/tools/conformance_run.cpp | 16 ++-- bindings/cpp/tools/coupling_conformance.cpp | 18 ++-- bindings/java/README.md | 12 +-- bindings/java/groimp-plugin/README.md | 10 +- bindings/java/groimp-plugin/pom.xml | 18 ++-- .../groimp/plugin/Encoding3dExport.java} | 24 ++--- .../src/main/resources/plugin.properties | 8 +- .../src/main/resources/plugin.xml | 16 ++-- .../java/groimp-plugin/test/run_in_groimp.sh | 28 +++--- bindings/java/groimp/pom.xml | 12 +-- .../{fspmtf => encoding3d}/groimp/GroImp.java | 26 +++--- .../groimp/GroImpTest.java | 16 ++-- .../{fspmtf => encoding3d}/groimp/Organs.java | 2 +- bindings/java/pom.xml | 20 ++-- .../{fspmtf => encoding3d}/Codec.java | 70 +++++++------- .../{fspmtf => encoding3d}/Column.java | 12 +-- .../{fspmtf => encoding3d}/ComponentType.java | 2 +- .../ConformanceRun.java | 6 +- .../{fspmtf => encoding3d}/DataStructure.java | 4 +- .../openalea/{fspmtf => encoding3d}/Dump.java | 6 +- .../encoding3d/Encoding3dException.java | 14 +++ .../{fspmtf => encoding3d}/EntitySet.java | 2 +- .../{fspmtf => encoding3d}/Frame.java | 4 +- .../openalea/{fspmtf => encoding3d}/Glb.java | 20 ++-- .../openalea/{fspmtf => encoding3d}/Grid.java | 2 +- .../{fspmtf => encoding3d}/Mapping.java | 2 +- .../openalea/{fspmtf => encoding3d}/Mesh.java | 2 +- .../{fspmtf => encoding3d}/Relation.java | 2 +- .../{fspmtf => encoding3d}/Sequences.java | 4 +- .../{fspmtf => encoding3d}/Table.java | 10 +- .../{fspmtf => encoding3d}/Update.java | 2 +- .../{fspmtf => encoding3d}/Variable.java | 2 +- .../coupling/Channel.java | 4 +- .../coupling/ConformanceParticipant.java | 6 +- .../coupling/FolderChannel.java | 8 +- .../{fspmtf => encoding3d}/coupling/Json.java | 2 +- .../coupling/Participant.java | 26 +++--- .../{fspmtf => encoding3d}/coupling/Turn.java | 4 +- .../coupling/ZmqChannel.java | 8 +- .../org/openalea/fspmtf/FspmTfException.java | 14 --- .../{fspmtf => encoding3d}/CodecTest.java | 18 ++-- bindings/julia/README.md | 16 ++-- bindings/julia/conformance_run.jl | 4 +- bindings/julia/coupling_conformance.jl | 8 +- bindings/julia/{FspmTF => encoding3d}/LICENSE | 0 .../julia/{FspmTF => encoding3d}/Project.toml | 10 +- .../ext/encoding3dMultiScaleTreeGraphExt.jl} | 34 +++---- .../ext/encoding3dPlantGeomTurtleExt.jl} | 12 +-- .../ext/encoding3dPlantGraphsExt.jl} | 22 ++--- .../ext/encoding3dZMQExt.jl} | 14 +-- .../julia/{FspmTF => encoding3d}/src/codec.jl | 8 +- .../{FspmTF => encoding3d}/src/coupling.jl | 16 ++-- .../julia/{FspmTF => encoding3d}/src/dump.jl | 2 +- .../src/encoding3d.jl} | 18 ++-- .../julia/{FspmTF => encoding3d}/src/glb.jl | 2 +- .../julia/{FspmTF => encoding3d}/src/model.jl | 2 +- .../{FspmTF => encoding3d}/src/sequence.jl | 0 .../{FspmTF => encoding3d}/src/values.jl | 2 +- .../{FspmTF => encoding3d}/test/Project.toml | 0 .../test/coupling/function.jl | 4 +- .../test/plantgraphs/Project.toml | 2 +- .../test/plantgraphs/runtests.jl | 10 +- .../{FspmTF => encoding3d}/test/runtests.jl | 46 +++++----- bindings/python/conformance_run.py | 8 +- .../src/openalea/encoding3d/__init__.py | 15 +++ .../openalea/{fspm_tf => encoding3d}/_spec.py | 8 +- .../openalea/{fspm_tf => encoding3d}/cli.py | 38 ++++---- .../openalea/{fspm_tf => encoding3d}/codec.py | 16 ++-- .../{fspm_tf => encoding3d}/compare.py | 6 +- .../coupling/__init__.py | 12 +-- .../coupling/channels.py | 4 +- .../coupling/conformance.py | 18 ++-- .../coupling/coordinator.py | 16 ++-- .../coupling/messages.py | 2 +- .../coupling/participant.py | 14 +-- .../coupling/replay.py | 8 +- .../coupling/resolution.py | 4 +- .../{fspm_tf => encoding3d}/coupling/store.py | 6 +- .../{fspm_tf => encoding3d}/display.py | 4 +- .../openalea/{fspm_tf => encoding3d}/dump.py | 2 +- .../openalea/{fspm_tf => encoding3d}/glb.py | 2 +- .../openalea/{fspm_tf => encoding3d}/model.py | 2 +- .../{fspm_tf => encoding3d}/schema.py | 4 +- .../openalea/{fspm_tf => encoding3d}/ucum.py | 4 +- .../{fspm_tf => encoding3d}/validate.py | 16 ++-- .../openalea/{fspm_tf => encoding3d}/views.py | 28 +++--- .../{fspm_tf => encoding3d}/vocabulary.py | 4 +- .../python/src/openalea/fspm_tf/__init__.py | 15 --- bindings/python/test/test_codec.py | 6 +- bindings/python/test/test_compare.py | 6 +- bindings/python/test/test_conformance.py | 6 +- bindings/python/test/test_coupling.py | 76 ++++++++-------- bindings/python/test/test_lazy.py | 18 ++-- bindings/python/test/test_paraview_reader.py | 26 +++--- bindings/python/test/test_tools.py | 6 +- bindings/python/test/test_validate.py | 6 +- bindings/python/test/test_views.py | 18 ++-- conda/environment.yml | 2 +- conda/meta.yaml | 4 +- conformance/README.md | 8 +- conformance/adaptive_soil.expected.json | 2 +- conformance/adaptive_soil.glb | Bin 2428 -> 2436 bytes conformance/coupling/generate.py | 6 +- .../two_data_structures/peer/frame_000000.glb | Bin 1420 -> 1428 bytes .../two_data_structures/peer/frame_000001.glb | Bin 1420 -> 1428 bytes .../two_data_structures/peer/frame_000002.glb | Bin 1436 -> 1444 bytes .../two_data_structures/peer/frame_000003.glb | Bin 1436 -> 1444 bytes .../two_data_structures/peer/frame_000004.glb | Bin 1436 -> 1444 bytes .../subject/update_0000.glb | Bin 1988 -> 1988 bytes .../subject/update_0001.glb | Bin 2092 -> 2092 bytes .../subject/update_0002.glb | Bin 2092 -> 2092 bytes .../subject/update_0003.glb | Bin 2140 -> 2140 bytes .../subject/update_0004.glb | Bin 2140 -> 2140 bytes .../coupling/two_scales/peer/frame_000000.glb | Bin 1908 -> 1908 bytes .../coupling/two_scales/peer/frame_000001.glb | Bin 1932 -> 1940 bytes .../coupling/two_scales/peer/frame_000002.glb | Bin 1956 -> 1964 bytes .../coupling/two_scales/peer/frame_000003.glb | Bin 1956 -> 1964 bytes .../coupling/two_scales/peer/frame_000004.glb | Bin 1988 -> 1988 bytes .../two_scales/subject/update_0000.glb | Bin 1956 -> 1964 bytes .../two_scales/subject/update_0001.glb | Bin 2028 -> 2028 bytes .../two_scales/subject/update_0002.glb | Bin 2092 -> 2092 bytes .../two_scales/subject/update_0003.glb | Bin 2092 -> 2092 bytes .../two_scales/subject/update_0004.glb | Bin 2156 -> 2156 bytes conformance/generate.py | 26 +++--- .../frame_000000.expected.json | 2 +- conformance/growing_sequence/frame_000000.glb | Bin 12892 -> 12900 bytes .../frame_000001.expected.json | 2 +- conformance/growing_sequence/frame_000001.glb | Bin 14852 -> 14860 bytes .../frame_000002.expected.json | 2 +- conformance/growing_sequence/frame_000002.glb | Bin 17764 -> 17772 bytes conformance/missing_values.expected.json | 2 +- conformance/missing_values.glb | Bin 6988 -> 6996 bytes conformance/parameter_set.expected.json | 2 +- conformance/parameter_set.glb | Bin 3484 -> 3500 bytes conformance/root_system.expected.json | 2 +- conformance/root_system.glb | Bin 15820 -> 15828 bytes conformance/section_cells.expected.json | 4 +- conformance/section_cells.glb | Bin 5284 -> 5300 bytes conformance/soil_and_roots.expected.json | 4 +- conformance/soil_and_roots.glb | Bin 18356 -> 18364 bytes conformance/wheat_shoot.expected.json | 2 +- conformance/wheat_shoot.glb | Bin 6956 -> 6964 bytes doc/api.md | 34 +++---- doc/conf.py | 16 ++-- doc/coupling.md | 24 ++--- doc/cpp.md | 60 ++++++------ doc/design_note.md | 84 ++++++++--------- doc/index.md | 14 +-- doc/java.md | 14 +-- doc/julia.md | 30 +++--- ...ython.ipynb => encoding3d_in_python.ipynb} | 38 ++++---- doc/overview.md | 6 +- doc/platforms.md | 24 ++--- doc/python.md | 28 +++--- doc/reading_the_specification.md | 14 +-- doc/using.md | 68 +++++++------- implementations.md | 18 ++-- ..._tf_views.py => apply_encoding3d_views.py} | 22 ++--- paraview/check_in_paraview.py | 86 +++++++++--------- ...fspm_tf_reader.py => encoding3d_reader.py} | 80 ++++++++-------- ...f_selection.py => encoding3d_selection.py} | 34 +++---- ...m_tf_views.py => save_encoding3d_views.py} | 18 ++-- ...ection.py => show_encoding3d_selection.py} | 14 +-- pyproject.toml | 22 ++--- spec/schema/coupling.schema.json | 4 +- spec/schema/fspm_scene.schema.json | 4 +- spec/schema/index.schema.json | 6 +- spec/schema/views.schema.json | 10 +- spec/specification.md | 16 ++-- 197 files changed, 1235 insertions(+), 1235 deletions(-) create mode 100644 bindings/cpp/cmake/encoding3dConfig.cmake.in delete mode 100644 bindings/cpp/cmake/fspm_tfConfig.cmake.in rename bindings/cpp/cplantbox/include/{fspm_tf => encoding3d}/cplantbox.hpp (94%) rename bindings/cpp/include/{fspm_tf => encoding3d}/coupling.hpp (88%) rename bindings/cpp/include/{fspm_tf/fspm_tf.hpp => encoding3d/encoding3d.hpp} (90%) rename bindings/cpp/include/{fspm_tf => encoding3d}/model.hpp (98%) rename bindings/cpp/test/{test_fspm_tf.cpp => test_encoding3d.cpp} (97%) rename bindings/java/groimp-plugin/src/main/java/org/openalea/{fspmtf/groimp/plugin/FspmTfExport.java => encoding3d/groimp/plugin/Encoding3dExport.java} (92%) rename bindings/java/groimp/src/main/java/org/openalea/{fspmtf => encoding3d}/groimp/GroImp.java (97%) rename bindings/java/groimp/src/test/java/org/openalea/{fspmtf => encoding3d}/groimp/GroImpTest.java (93%) rename bindings/java/groimp/src/test/java/org/openalea/{fspmtf => encoding3d}/groimp/Organs.java (99%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Codec.java (93%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Column.java (88%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/ComponentType.java (97%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/ConformanceRun.java (92%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/DataStructure.java (91%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Dump.java (98%) create mode 100644 bindings/java/src/main/java/org/openalea/encoding3d/Encoding3dException.java rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/EntitySet.java (97%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Frame.java (90%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Glb.java (81%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Grid.java (95%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Mapping.java (95%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Mesh.java (96%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Relation.java (94%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Sequences.java (95%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Table.java (83%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Update.java (95%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/Variable.java (98%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/Channel.java (87%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/ConformanceParticipant.java (93%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/FolderChannel.java (93%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/Json.java (83%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/Participant.java (79%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/Turn.java (79%) rename bindings/java/src/main/java/org/openalea/{fspmtf => encoding3d}/coupling/ZmqChannel.java (86%) delete mode 100644 bindings/java/src/main/java/org/openalea/fspmtf/FspmTfException.java rename bindings/java/src/test/java/org/openalea/{fspmtf => encoding3d}/CodecTest.java (93%) rename bindings/julia/{FspmTF => encoding3d}/LICENSE (100%) rename bindings/julia/{FspmTF => encoding3d}/Project.toml (76%) rename bindings/julia/{FspmTF/ext/FspmTFMultiScaleTreeGraphExt.jl => encoding3d/ext/encoding3dMultiScaleTreeGraphExt.jl} (95%) rename bindings/julia/{FspmTF/ext/FspmTFPlantGeomTurtleExt.jl => encoding3d/ext/encoding3dPlantGeomTurtleExt.jl} (90%) rename bindings/julia/{FspmTF/ext/FspmTFPlantGraphsExt.jl => encoding3d/ext/encoding3dPlantGraphsExt.jl} (92%) rename bindings/julia/{FspmTF/ext/FspmTFZMQExt.jl => encoding3d/ext/encoding3dZMQExt.jl} (68%) rename bindings/julia/{FspmTF => encoding3d}/src/codec.jl (99%) rename bindings/julia/{FspmTF => encoding3d}/src/coupling.jl (96%) rename bindings/julia/{FspmTF => encoding3d}/src/dump.jl (99%) rename bindings/julia/{FspmTF/src/FspmTF.jl => encoding3d/src/encoding3d.jl} (85%) rename bindings/julia/{FspmTF => encoding3d}/src/glb.jl (98%) rename bindings/julia/{FspmTF => encoding3d}/src/model.jl (99%) rename bindings/julia/{FspmTF => encoding3d}/src/sequence.jl (100%) rename bindings/julia/{FspmTF => encoding3d}/src/values.jl (98%) rename bindings/julia/{FspmTF => encoding3d}/test/Project.toml (100%) rename bindings/julia/{FspmTF => encoding3d}/test/coupling/function.jl (97%) rename bindings/julia/{FspmTF => encoding3d}/test/plantgraphs/Project.toml (85%) rename bindings/julia/{FspmTF => encoding3d}/test/plantgraphs/runtests.jl (89%) rename bindings/julia/{FspmTF => encoding3d}/test/runtests.jl (91%) create mode 100644 bindings/python/src/openalea/encoding3d/__init__.py rename bindings/python/src/openalea/{fspm_tf => encoding3d}/_spec.py (58%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/cli.py (86%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/codec.py (98%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/compare.py (96%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/__init__.py (56%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/channels.py (98%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/conformance.py (89%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/coordinator.py (94%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/messages.py (99%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/participant.py (88%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/replay.py (96%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/resolution.py (98%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/coupling/store.py (98%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/display.py (92%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/dump.py (99%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/glb.py (97%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/model.py (99%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/schema.py (72%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/ucum.py (97%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/validate.py (98%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/views.py (95%) rename bindings/python/src/openalea/{fspm_tf => encoding3d}/vocabulary.py (96%) delete mode 100644 bindings/python/src/openalea/fspm_tf/__init__.py rename doc/notebooks/{fspm_tf_in_python.ipynb => encoding3d_in_python.ipynb} (83%) rename paraview/{apply_fspm_tf_views.py => apply_encoding3d_views.py} (79%) rename paraview/{fspm_tf_reader.py => encoding3d_reader.py} (92%) rename paraview/{fspm_tf_selection.py => encoding3d_selection.py} (89%) rename paraview/{save_fspm_tf_views.py => save_encoding3d_views.py} (77%) rename paraview/{show_fspm_tf_selection.py => show_encoding3d_selection.py} (58%) diff --git a/.github/actions/conformance/action.yml b/.github/actions/conformance/action.yml index 60270d3..395b34a 100644 --- a/.github/actions/conformance/action.yml +++ b/.github/actions/conformance/action.yml @@ -1,13 +1,13 @@ -# Checks a codec against fspm-TF's conformance files, in a codec's own CI, once the codec has run: +# Checks a codec against encoding3d's conformance files, in a codec's own CI, once the codec has run: # -# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/fspm-TF/.github/actions/conformance@main) +# - uses: ./.github/actions/conformance (in this repository; elsewhere openalea/encoding3d/.github/actions/conformance@main) # with: # dumps: out/dumps # what the codec read: one JSON dump per conformance GLB, x.json for x.expected.json # written: out/written # what the codec wrote: the conformance scenes, as .glb files (optional) # -# Reading passes when every dump equals its expected values; writing, when every written file passes fspm-tf validate +# Reading passes when every dump equals its expected values; writing, when every written file passes encoding3d validate # and Khronos' glTF-Validator with no error. -name: fspm-TF conformance +name: encoding3d conformance description: Compare a codec's reading of the conformance files with their expected values, and validate what it wrote inputs: dumps: @@ -17,8 +17,8 @@ inputs: description: the folder of the .glb files the codec wrote for the conformance scenes (empty to skip) required: false default: "" - fspm-tf: - description: the path of an fspm-TF checkout (its conformance folder and Python package) + encoding3d: + description: the path of an encoding3d checkout (its conformance folder and Python package) required: false default: "." gltf-validator: @@ -31,16 +31,16 @@ runs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf + - name: Install openalea.encoding3d shell: bash - run: python -m pip install numpy jsonschema "${{ inputs.fspm-tf }}" + run: python -m pip install numpy jsonschema "${{ inputs.encoding3d }}" - name: Reading shell: bash - run: fspm-tf compare-all "${{ inputs.dumps }}" "${{ inputs.fspm-tf }}/conformance" - - name: Writing, fspm-TF validator + run: encoding3d compare-all "${{ inputs.dumps }}" "${{ inputs.encoding3d }}/conformance" + - name: Writing, encoding3d validator if: inputs.written != '' shell: bash - run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 fspm-tf validate + run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 encoding3d validate - name: Writing, Khronos' glTF-Validator if: inputs.written != '' && runner.os == 'Linux' shell: bash diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 04e6bd0..efe7585 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -1,4 +1,4 @@ -# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid fspm-TF (our validator) +# The conformance files are valid glTF for every viewer (Khronos' glTF-Validator) and valid encoding3d (our validator) name: Conformance on: @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests + - name: Install openalea.encoding3d (its dependencies, conda-only in pyproject.toml, given here), and VTK for the ParaView reader's tests run: python -m pip install numpy jsonschema pytest pyzmq vtk . - name: Tests, the ParaView reader's conversion included run: pytest bindings/python/test -v @@ -25,8 +25,8 @@ jobs: run: | python conformance/generate.py git diff --exit-code -- conformance - - name: fspm-TF validator - run: fspm-tf validate conformance/*.glb conformance/growing_sequence/scene.fspm.json + - name: encoding3d validator + run: encoding3d validate conformance/*.glb conformance/growing_sequence/scene.fspm.json - name: Khronos glTF-Validator run: | curl -sSL -o validator.tar.xz "https://github.com/KhronosGroup/glTF-Validator/releases/download/${GLTF_VALIDATOR}/gltf_validator-${GLTF_VALIDATOR}-linux64.tar.xz" @@ -42,7 +42,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Install openalea.fspm_tf and the documentation's tools + - name: Install openalea.encoding3d and the documentation's tools run: | sudo apt-get install -y pandoc python -m pip install numpy jsonschema pytest sphinx ".[doc]" diff --git a/.github/workflows/cpp-codec.yml b/.github/workflows/cpp-codec.yml index 71d9078..1062d3c 100644 --- a/.github/workflows/cpp-codec.yml +++ b/.github/workflows/cpp-codec.yml @@ -37,8 +37,8 @@ jobs: - name: Dump and write the conformance scenes with the C++ codec run: | cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release - cmake --build build --target fspm_tf_conformance_run - ./build/fspm_tf_conformance_run conformance out + cmake --build build --target encoding3d_conformance_run + ./build/encoding3d_conformance_run conformance out - uses: ./.github/actions/conformance with: dumps: out/dumps @@ -48,9 +48,9 @@ jobs: run: | sudo apt-get install -y -q libzmq3-dev cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release - cmake --build build --target fspm_tf_coupling_conformance + cmake --build build --target encoding3d_coupling_conformance python -m pip install numpy jsonschema pyzmq . - fspm-tf couple-conformance conformance/coupling --participant "$PWD/build/fspm_tf_coupling_conformance" + encoding3d couple-conformance conformance/coupling --participant "$PWD/build/encoding3d_coupling_conformance" cplantbox-adapter: runs-on: ubuntu-latest @@ -65,17 +65,17 @@ jobs: cmake --build cplantbox-build --target CPlantBox -j 4 - name: The codec and the adapter run: | - cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release -DFSPM_TF_CPLANTBOX_SOURCE_DIR="$PWD/cplantbox" \ - -DFSPM_TF_CPLANTBOX_LIBRARY="$PWD/cplantbox-build/src/libCPlantBox.a" + cmake -S bindings/cpp -B build -DCMAKE_BUILD_TYPE=Release -DENCODING3D_CPLANTBOX_SOURCE_DIR="$PWD/cplantbox" \ + -DENCODING3D_CPLANTBOX_LIBRARY="$PWD/cplantbox-build/src/libCPlantBox.a" cmake --build build -j 4 ctest --test-dir build --output-on-failure - - name: What the adapter wrote is valid fspm-TF + - name: What the adapter wrote is valid encoding3d run: | python -m pip install numpy jsonschema . - fspm-tf validate build/cplantbox/cplantbox_out/*.glb + encoding3d validate build/cplantbox/cplantbox_out/*.glb - name: CPlantBox's log replayed in a coupling session env: - FSPM_TF_CPLANTBOX_LOG: build/cplantbox/cplantbox_out/anagallis_log/scene.fspm.json + ENCODING3D_CPLANTBOX_LOG: build/cplantbox/cplantbox_out/anagallis_log/scene.fspm.json run: | python -m pip install pytest pyzmq pytest -v bindings/python/test/test_coupling.py -k platform_log diff --git a/.github/workflows/java-codec.yml b/.github/workflows/java-codec.yml index e3b2ffa..6852aef 100644 --- a/.github/workflows/java-codec.yml +++ b/.github/workflows/java-codec.yml @@ -43,7 +43,7 @@ jobs: - name: Dump and write the conformance scenes with the Java codec run: | mvn -B -q -f bindings/java/pom.xml package -DskipTests dependency:copy-dependencies -DincludeScope=runtime - java -cp "bindings/java/target/fspm-tf-0.1.0.jar:bindings/java/target/dependency/*" org.openalea.fspmtf.ConformanceRun conformance out + java -cp "bindings/java/target/encoding3d-0.1.0.jar:bindings/java/target/dependency/*" org.openalea.encoding3d.ConformanceRun conformance out - uses: ./.github/actions/conformance with: dumps: out/dumps @@ -52,8 +52,8 @@ jobs: - name: The coupling conformance sessions run: | python -m pip install pyzmq - fspm-tf couple-conformance conformance/coupling --participant \ - "java -cp $PWD/bindings/java/target/fspm-tf-0.1.0.jar:$PWD/bindings/java/target/dependency/* org.openalea.fspmtf.coupling.ConformanceParticipant" + encoding3d couple-conformance conformance/coupling --participant \ + "java -cp $PWD/bindings/java/target/encoding3d-0.1.0.jar:$PWD/bindings/java/target/dependency/* org.openalea.encoding3d.coupling.ConformanceParticipant" groimp-adapter: runs-on: ubuntu-latest @@ -68,13 +68,13 @@ jobs: run: | mvn -B -q -f bindings/java/pom.xml install -DskipTests mvn -B -f bindings/java/groimp/pom.xml verify - - name: What the adapter wrote is valid fspm-TF + - name: What the adapter wrote is valid encoding3d run: | python -m pip install numpy jsonschema . - fspm-tf validate bindings/java/groimp/target/groimp_out/*.glb + encoding3d validate bindings/java/groimp/target/groimp_out/*.glb # The plugin in GroIMP itself, headless: GroIMP 3.0.0's release, its glTF plugin (a pinned commit, which meshes the - # shapes), an example grown and its 3D view exported as an fspm-TF frame + # shapes), an example grown and its 3D view exported as an encoding3d frame groimp-plugin: runs-on: ubuntu-latest steps: @@ -109,10 +109,10 @@ jobs: run: | python -m pip install numpy jsonschema . bash bindings/java/groimp-plugin/test/run_in_groimp.sh groimp-install/usr/share/GroIMP gltf-plugin \ - bindings/java/groimp-plugin/target/plugin/fspmTF out + bindings/java/groimp-plugin/target/plugin/encoding3d out - name: GroIMP's log replayed in a coupling session env: - FSPM_TF_GROIMP_LOG: out/log/scene.fspm.json + ENCODING3D_GROIMP_LOG: out/log/scene.fspm.json run: | python -m pip install pytest pyzmq pytest -v bindings/python/test/test_coupling.py -k platform_log diff --git a/.github/workflows/julia-codec.yml b/.github/workflows/julia-codec.yml index ce56b30..0372f57 100644 --- a/.github/workflows/julia-codec.yml +++ b/.github/workflows/julia-codec.yml @@ -1,6 +1,6 @@ -# The Julia codec (bindings/julia/FspmTF): its tests, on the oldest supported Julia and the latest release, and the +# The Julia codec (bindings/julia/encoding3d): its tests, on the oldest supported Julia and the latest release, and the # conformance check other codecs run (its dumps of the conformance files compared with their expected values, the files -# it writes validated by fspm-tf validate and Khronos' glTF-Validator). OpenAlea's conda CI builds the Python package +# it writes validated by encoding3d validate and Khronos' glTF-Validator). OpenAlea's conda CI builds the Python package # only. name: Julia codec @@ -32,10 +32,10 @@ jobs: - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 with: - project: bindings/julia/FspmTF + project: bindings/julia/encoding3d - uses: julia-actions/julia-runtest@v1 with: - project: bindings/julia/FspmTF + project: bindings/julia/encoding3d # VirtualPlantLab's PlantGraphs needs Julia 1.12 or later: its extensions are tested in their own project virtualplantlab: @@ -48,8 +48,8 @@ jobs: - uses: julia-actions/cache@v2 - name: PlantGraphs and PlantGeomTurtle extensions run: | - julia --project=bindings/julia/FspmTF/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/FspmTF"); Pkg.instantiate()' - julia --project=bindings/julia/FspmTF/test/plantgraphs bindings/julia/FspmTF/test/plantgraphs/runtests.jl + julia --project=bindings/julia/encoding3d/test/plantgraphs -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.instantiate()' + julia --project=bindings/julia/encoding3d/test/plantgraphs bindings/julia/encoding3d/test/plantgraphs/runtests.jl conformance: runs-on: ubuntu-latest @@ -61,14 +61,14 @@ jobs: - uses: julia-actions/cache@v2 - name: Dump and write the conformance scenes with the Julia codec run: | - julia --project=bindings/julia/FspmTF -e 'using Pkg; Pkg.instantiate()' - julia --project=bindings/julia/FspmTF bindings/julia/conformance_run.jl conformance out + julia --project=bindings/julia/encoding3d -e 'using Pkg; Pkg.instantiate()' + julia --project=bindings/julia/encoding3d bindings/julia/conformance_run.jl conformance out - uses: ./.github/actions/conformance with: dumps: out/dumps written: out/written - # FspmTF.jl as a participant of coupling sessions run by the Python coordinator (specification §11), over ZeroMQ + # encoding3d.jl as a participant of coupling sessions run by the Python coordinator (specification §11), over ZeroMQ # and over a folder coupling: runs-on: ubuntu-latest @@ -83,16 +83,16 @@ jobs: python-version: "3.12" - name: The Python coordinator run: pip install -e . pytest pyzmq jsonschema - - name: A Julia project with FspmTF and ZMQ + - name: A Julia project with encoding3d and ZMQ run: | mkdir -p coupling-env - julia --project=coupling-env -e 'using Pkg; Pkg.develop(path="bindings/julia/FspmTF"); Pkg.add("ZMQ")' + julia --project=coupling-env -e 'using Pkg; Pkg.develop(path="bindings/julia/encoding3d"); Pkg.add("ZMQ")' - name: The coupling tests, with the Julia participant env: - FSPM_TF_JULIA: julia --project=coupling-env + ENCODING3D_JULIA: julia --project=coupling-env run: pytest -v bindings/python/test/test_coupling.py - name: The coupling conformance sessions, the Julia participant run: | julia --project=coupling-env -e 'using Pkg; Pkg.add("JSON")' - fspm-tf couple-conformance conformance/coupling --participant \ + encoding3d couple-conformance conformance/coupling --participant \ "julia --project=$PWD/coupling-env $PWD/bindings/julia/coupling_conformance.jl" diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a38d0..907d5ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,15 +5,15 @@ - Views, an optional companion of a sequence (`views.json`, `spec/schema/views.schema.json`): named configurations of layers, each a mesh's, segments' or grid's entities selected by `where` conditions, coloured by a variable (or a solid colour), with a colour map, range, log scale, opacity, visibility, representation, and tubes for segments. -- `openalea.fspm_tf.views`: reading, writing and checking views (the schema, and every name against a frame), the +- `openalea.encoding3d.views`: reading, writing and checking views (the schema, and every name against a frame), the selection, and the display options the ParaView macros apply and save back. - The ParaView reader: "Views" and "Configuration" (a drop-down), one block per layer holding only its entities; - `frame_to_vtk(frame, views=, configuration=)`, unchanged without views. The macros `apply_fspm_tf_views.py` (each - layer its own pipeline item) and `save_fspm_tf_views.py` (the items' display options back into `views.json`). -- Selection by scale in ParaView: the plugin's filter "fspm-TF Select at scale" (Filters → fspm-TF) takes ParaView's + `frame_to_vtk(frame, views=, configuration=)`, unchanged without views. The macros `apply_encoding3d_views.py` (each + layer its own pipeline item) and `save_encoding3d_views.py` (the items' display options back into `views.json`). +- Selection by scale in ParaView: the plugin's filter "encoding3d Select at scale" (Filters → encoding3d) takes ParaView's selection and grows the picked cells to their entity at its **Scale** (a drop-down of the frames' scales) with everything it contains, in every layer of the reader, hidden ones included, following the picked entities through - the time steps; the macro `show_fspm_tf_selection.py` shows it as its layers, isolated in the view or in a panel of + the time steps; the macro `show_encoding3d_selection.py` shows it as its layers, isolated in the view or in a panel of its own, one selection over another, run again to go back. Each cell carries the ids of the entities containing it (`.id`), each block its data structure and scales. @@ -31,23 +31,23 @@ - Links between two data structures, through a mapping a participant provides (declared, sent in its updates): `sum` shares each source's amount among its targets by weight, `mean` gives each target its sources' weighted mean; replayed sequences provide their mappings; a replay declares every entity set and variable of its frames. -- Coupling conformance sessions (`conformance/coupling/`, written by its `generate.py`), `fspm-tf couple-conformance`, +- Coupling conformance sessions (`conformance/coupling/`, written by its `generate.py`), `encoding3d couple-conformance`, and a participant's program in each codec; partial frames (`update`) in every codec's dump. -- The C++ participant (`fspm_tf/coupling.hpp`; ZeroMQ with libzmq) and the Java one (`org.openalea.fspmtf.coupling`; +- The C++ participant (`encoding3d/coupling.hpp`; ZeroMQ with libzmq) and the Java one (`org.openalea.encoding3d.coupling`; ZeroMQ with JeroMQ, optional); partial frames in both codecs. - CPlantBox's and GroIMP's logs (the adapter's tests, the plugin's headless run) replayed in coupling sessions. - The coordinator's lookups by id vectorized, and the Python codec's enum and text columns encoded without per-row loops: a step of 100,000 segments takes 99 ms instead of 230 ms. -- Coupling sessions: platforms' models run together, each in its own process, through fspm-TF. Partial frames +- Coupling sessions: platforms' models run together, each in its own process, through encoding3d. Partial frames (`FSPM_scene.update`: the sender, removed entities); the messages (`spec/schema/coupling.schema.json`): a participant's declaration (its entity sets and rights, `owns`, `refines`, `follows`; its outputs and inputs), links (factors, units, a change of scale), turns and updates; stages; one shared structure with id ranges and removals down the scales; two transports, ZeroMQ and a folder of messages; a recorded sequence as a participant (replay, held or interpolated linearly). -- The reference coordinator and participant (`openalea.fspm_tf.coupling`, `fspm-tf couple`), unit conversion factors - (`ucum.conversion_factor`); FspmTF.jl's participant (`Participant`, `turns`, `send_update`; ZeroMQ through the - `FspmTFZMQExt` extension), tested against the coordinator over both transports. +- The reference coordinator and participant (`openalea.encoding3d.coupling`, `encoding3d couple`), unit conversion factors + (`ucum.conversion_factor`); encoding3d.jl's participant (`Participant`, `turns`, `send_update`; ZeroMQ through the + `encoding3dZMQExt` extension), tested against the coordinator over both transports. - The Python codec refuses enum columns (labels) with missing values instead of writing an unnamed enum value. ## Unreleased: specification 0.2 (draft) @@ -68,11 +68,11 @@ - The specification (`spec/specification.md`), its JSON Schemas and the first vocabulary (41 standard names, the labels with their Plant Ontology terms, the enumerations, a column of names per platform). -- `openalea.fspm_tf`, the Python reference codec (`bindings/python/`): reading and writing frames and sequences, the - validator, display copies, the comparison with expected values, the `fspm-tf` command line. +- `openalea.encoding3d`, the Python reference codec (`bindings/python/`): reading and writing frames and sequences, the + validator, display copies, the comparison with expected values, the `encoding3d` command line. - The conformance files (`conformance/`), their expected values, and the GitHub action checking a codec against them. - The ParaView reader plugin (`paraview/`). -- FspmTF.jl, the Julia codec (`bindings/julia/FspmTF/`): conformant reading and writing, sequences, and a +- encoding3d.jl, the Julia codec (`bindings/julia/encoding3d/`): conformant reading and writing, sequences, and a MultiScaleTreeGraph.jl package extension (`from_mtg`, `to_mtg`); its CI, `.github/workflows/julia-codec.yml`. - The C++ codec (`bindings/cpp/`): conformant reading and writing, sequences, a CMake package; CPlantBox's adapter (`bindings/cpp/cplantbox/`: organisms with their geometry, soil grid and mapping; static root systems from any graph @@ -80,11 +80,11 @@ - The Java codec (`bindings/java/`): conformant reading and writing, sequences; GroIMP's adapter (`bindings/java/groimp/`: scales from refinement edges, topology, edge bits as relations, managed fields; graphs back into the model's node classes); its CI, `.github/workflows/java-codec.yml`. -- GroIMP's plugin (`bindings/java/groimp-plugin/`): the 3D view exported as an fspm-TF frame from GroIMP 3.0 (File > +- GroIMP's plugin (`bindings/java/groimp-plugin/`): the 3D view exported as an encoding3d frame from GroIMP 3.0 (File > Export, CLI `export3d`), its geometry meshed by GroIMP's glTF plugin, each triangle with its node; tested headless in GroIMP 3.0.0 in CI. -- FspmTF.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry +- encoding3d.jl: VirtualPlantLab's graphs (PlantGraphs: `from_graph`, `to_graph`) and their turtle geometry (PlantGeomTurtle: `add_geometry!`), as package extensions. -- FspmTF.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `fspm_tf_id`), and parents - in another complex kept (`fspm_tf_parent`). +- encoding3d.jl: MTGs from graphs with ids from 0 or sparse (numbered 1, 2, ..., ids kept in `encoding3d_id`), and parents + in another complex kept (`encoding3d_parent`). - The documentation (`doc/`), with a notebook; the design note (`doc/design_note.md`). diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 3ea8cf2..da16f1f 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,6 +1,6 @@ # Code of conduct -fspm-TF is developed by several plant-modelling teams together. Everyone taking part (in issues, pull requests, +encoding3d is developed by several plant-modelling teams together. Everyone taking part (in issues, pull requests, reviews, meetings) is expected to follow the [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/): be welcoming and respectful, give and accept constructive criticism, and focus on what is best for the community. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f451b1d..12b699c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -fspm-TF is a format shared by several FSPM platforms. Contributions are welcome from each of them: to the +encoding3d is a format shared by several FSPM platforms. Contributions are welcome from each of them: to the specification, to the vocabulary, to a codec, to the tools. ## What goes where @@ -19,7 +19,7 @@ specification, to the vocabulary, to a codec, to the tools. ```bash mamba env create -f conda/environment.yml # the package, editable, with the tests' and the docs' tools -mamba activate fspm_tf +mamba activate encoding3d pytest # the package's tests (bindings/python/test) pytest --nbmake doc/notebooks # the documentation's notebooks run python -m sphinx -b html doc doc/_build -W # the documentation, warnings as errors diff --git a/README.md b/README.md index cf83c7e..d9bff3c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# FSPM-TF - Functional Structural Plant Model Transmission Format - based on GLTF 2.0 +# encoding3d - functional-structural plant model scenes, based on glTF 2.0 An exchange format for whole functional-structural plant model (FSPM) scenes, across platforms and languages (openalea.metafspm, CPlantBox, GroIMP, VirtualPlantLab). It holds: @@ -30,8 +30,8 @@ change there, released as a new version. Everything else implements, checks or d | `spec/specification.md` | the specification: data model, glTF layout, sequences, conformance | **yes** | | `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension, of a sequence's index and views, and of a coupling session's messages | **yes** | | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** | -| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`); `coupling/`: the coupling sessions a participant runs (`fspm-tf couple-conformance`) | follows `spec/` | -| `bindings//` | each language's codec and coupling participant, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec with lazy reading, validator, display copies, views, the coupling's coordinator, command line), which reads `spec/` and ships it; `bindings/julia/`: FspmTF.jl, the Julia codec, with MultiScaleTreeGraph.jl, VirtualPlantLab and ZeroMQ extensions; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter; `bindings/java/`: the Java codec, with GroIMP's adapter and plugin | no | +| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `encoding3d compare-all` and the action `.github/actions/conformance`); `coupling/`: the coupling sessions a participant runs (`encoding3d couple-conformance`) | follows `spec/` | +| `bindings//` | each language's codec and coupling participant, with its own build, tests and release; `bindings/python/`: `openalea.encoding3d`, the reference (codec with lazy reading, validator, display copies, views, the coupling's coordinator, command line), which reads `spec/` and ships it; `bindings/julia/`: encoding3d.jl, the Julia codec, with MultiScaleTreeGraph.jl, VirtualPlantLab and ZeroMQ extensions; `bindings/cpp/`: the C++ codec, with CPlantBox's adapter; `bindings/java/`: the Java codec, with GroIMP's adapter and plugin | no | | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no | | `doc/design_note.md` | how the decisions were reached | no | | `pyproject.toml`, `conda/` | the Python package's build, at the root next to `spec/`, which it ships | no | @@ -39,32 +39,32 @@ change there, released as a new version. Everything else implements, checks or d ## The Python package ```bash -git clone https://github.com/openalea/fspm-TF.git -cd fspm-TF +git clone https://github.com/openalea/encoding3d.git +cd encoding3d mamba env create -f conda/environment.yml # numpy and jsonschema from conda-forge, then pip install -e . -mamba activate fspm_tf +mamba activate encoding3d pytest ``` ```python -from openalea import fspm_tf +from openalea import encoding3d -frame = fspm_tf.read("scene.glb") # data structures, entity sets as numpy columns, meshes -fspm_tf.write("copy.glb", frame) +frame = encoding3d.read("scene.glb") # data structures, entity sets as numpy columns, meshes +encoding3d.write("copy.glb", frame) ``` ```bash -fspm-tf validate scene.glb # or a sequence: fspm-tf validate run/scene.fspm.json -fspm-tf info scene.glb -fspm-tf display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView +encoding3d validate scene.glb # or a sequence: encoding3d validate run/scene.fspm.json +encoding3d info scene.glb +encoding3d display scene.glb water_potential # float32 per-vertex copies, for viewers such as ParaView ``` ## Viewing in ParaView -`paraview/fspm_tf_reader.py` is a ParaView reader plugin (ParaView ≥ 5.10): +`paraview/encoding3d_reader.py` is a ParaView reader plugin (ParaView ≥ 5.10): 1. In ParaView, open Tools → Manage Plugins → Load New…, and choose the file. Tick "Auto Load" to keep it loaded. -2. File → Open a `.glb` frame or a sequence's `scene.fspm.json`, and pick "fspm-TF reader". +2. File → Open a `.glb` frame or a sequence's `scene.fspm.json`, and pick "encoding3d reader". What it shows: @@ -74,27 +74,27 @@ What it shows: - Text columns such as labels become integer codes, `._code`; the names behind the codes are in the block's field data. -The plugin finds `openalea.fspm_tf` in ParaView's Python if it is installed there, otherwise in this clone's `bindings/python/src/`. +The plugin finds `openalea.encoding3d` in ParaView's Python if it is installed there, otherwise in this clone's `bindings/python/src/`. It needs nothing beyond what ParaView ships (numpy). In `pvpython` scripts the reader is -`fspmTFreader(FileName=...)`, after `LoadPlugin("paraview/fspm_tf_reader.py", ns=globals())`. Checked with +`encoding3dreader(FileName=...)`, after `LoadPlugin("paraview/encoding3d_reader.py", ns=globals())`. Checked with ParaView 6.0.1: `pvpython paraview/check_in_paraview.py conformance` (also run by the tests when -`FSPM_TF_PVPYTHON` names a pvpython). +`ENCODING3D_PVPYTHON` names a pvpython). ## For another platform -Two pieces of code make a platform read and write fspm-TF: +Two pieces of code make a platform read and write encoding3d: - **A codec** for its language: GLB files, property tables, sequences, the vocabulary. It depends on `spec/` only, so it lives here, in `bindings//` (`cpp/`, `java/`, `julia/`...), with its build files, its tests and its own release, maintained by the platform's team. A change to `spec/` then updates every codec in the same pull request, and the CI shows which languages follow. -- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.fspm_tf` for +- **An adapter** between the codec's tables and the platform's own model (e.g. `openalea.metafspm.io.encoding3d` for metafspm's DataStructures). It depends on the platform, so it lives in the platform's repository. Each codec runs the same conformance check in its own tests. It reads the conformance GLBs and compares what it -reads with their `*.expected.json`, and it runs `fspm-tf validate` on what it writes. The steps are in the +reads with their `*.expected.json`, and it runs `encoding3d validate` on what it writes. The steps are in the [specification, §9](spec/specification.md#9-conformance). Each platform has a column of `spec/vocabulary/standard_names.csv`, listing its own names for each standard name -(`;`-separated). The writers read their mapping from it (`openalea.fspm_tf.vocabulary.platform_names("metafspm")`), so +(`;`-separated). The writers read their mapping from it (`openalea.encoding3d.vocabulary.platform_names("metafspm")`), so the vocabulary changes without changing the platforms' code. diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 2079cca..fc6ac56 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -1,19 +1,19 @@ -# fspm_tf, the C++ codec of fspm-TF: a library (fspm_tf::fspm_tf), its tests, the conformance tool, and, when -# CPlantBox is given (FSPM_TF_CPLANTBOX_SOURCE_DIR and FSPM_TF_CPLANTBOX_LIBRARY), CPlantBox's adapter. +# encoding3d, the C++ codec of encoding3d: a library (encoding3d::encoding3d), its tests, the conformance tool, and, when +# CPlantBox is given (ENCODING3D_CPLANTBOX_SOURCE_DIR and ENCODING3D_CPLANTBOX_LIBRARY), CPlantBox's adapter. # # cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build # -# Licence: GPL-3.0 (LICENSE), as CPlantBox; the rest of fspm-TF is CeCILL-C. +# Licence: GPL-3.0 (LICENSE), as CPlantBox; the rest of encoding3d is CeCILL-C. cmake_minimum_required(VERSION 3.16) -project(fspm_tf VERSION 0.1.0 LANGUAGES CXX) +project(encoding3d VERSION 0.1.0 LANGUAGES CXX) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -option(FSPM_TF_BUILD_TESTS "Build the tests and the conformance tool" ${PROJECT_IS_TOP_LEVEL}) -set(FSPM_TF_CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder (its src/ holds structural/), for the adapter") -set(FSPM_TF_CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox), for the adapter") +option(ENCODING3D_BUILD_TESTS "Build the tests and the conformance tool" ${PROJECT_IS_TOP_LEVEL}) +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder (its src/ holds structural/), for the adapter") +set(ENCODING3D_CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox), for the adapter") # The one dependency, header-only: nlohmann_json, found installed or fetched find_package(nlohmann_json 3.11 QUIET) @@ -26,71 +26,71 @@ if(NOT nlohmann_json_FOUND) FetchContent_MakeAvailable(nlohmann_json) endif() -add_library(fspm_tf +add_library(encoding3d src/model.cpp src/glb.cpp src/codec.cpp src/sequence.cpp src/dump.cpp src/coupling.cpp) -add_library(fspm_tf::fspm_tf ALIAS fspm_tf) -target_include_directories(fspm_tf PUBLIC +add_library(encoding3d::encoding3d ALIAS encoding3d) +target_include_directories(encoding3d PUBLIC $ $) -target_link_libraries(fspm_tf PUBLIC nlohmann_json::nlohmann_json) +target_link_libraries(encoding3d PUBLIC nlohmann_json::nlohmann_json) # The coupling's ZeroMQ transport, with libzmq when found (its folder transport needs nothing) -option(FSPM_TF_WITH_ZMQ "Build the coupling's ZeroMQ transport (libzmq)" ON) -if(FSPM_TF_WITH_ZMQ) +option(ENCODING3D_WITH_ZMQ "Build the coupling's ZeroMQ transport (libzmq)" ON) +if(ENCODING3D_WITH_ZMQ) find_path(ZMQ_INCLUDE_DIR zmq.h) find_library(ZMQ_LIBRARY NAMES zmq libzmq) if(ZMQ_INCLUDE_DIR AND ZMQ_LIBRARY) - target_include_directories(fspm_tf PRIVATE ${ZMQ_INCLUDE_DIR}) - target_link_libraries(fspm_tf PRIVATE ${ZMQ_LIBRARY}) - target_compile_definitions(fspm_tf PRIVATE FSPM_TF_WITH_ZMQ) - message(STATUS "fspm_tf: the coupling's ZeroMQ transport, with ${ZMQ_LIBRARY}") + target_include_directories(encoding3d PRIVATE ${ZMQ_INCLUDE_DIR}) + target_link_libraries(encoding3d PRIVATE ${ZMQ_LIBRARY}) + target_compile_definitions(encoding3d PRIVATE ENCODING3D_WITH_ZMQ) + message(STATUS "encoding3d: the coupling's ZeroMQ transport, with ${ZMQ_LIBRARY}") else() - message(STATUS "fspm_tf: libzmq not found, the coupling's folder transport only") + message(STATUS "encoding3d: libzmq not found, the coupling's folder transport only") endif() endif() -target_compile_features(fspm_tf PUBLIC cxx_std_17) -set_target_properties(fspm_tf PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_compile_features(encoding3d PUBLIC cxx_std_17) +set_target_properties(encoding3d PROPERTIES POSITION_INDEPENDENT_CODE ON) if(MSVC) - target_compile_options(fspm_tf PRIVATE /W4 /bigobj) + target_compile_options(encoding3d PRIVATE /W4 /bigobj) else() - target_compile_options(fspm_tf PRIVATE -Wall -Wextra -Wpedantic) + target_compile_options(encoding3d PRIVATE -Wall -Wextra -Wpedantic) endif() -# Installed as a CMake package: find_package(fspm_tf) then target_link_libraries(... fspm_tf::fspm_tf) +# Installed as a CMake package: find_package(encoding3d) then target_link_libraries(... encoding3d::encoding3d) include(GNUInstallDirs) include(CMakePackageConfigHelpers) -install(TARGETS fspm_tf EXPORT fspm_tfTargets +install(TARGETS encoding3d EXPORT encoding3dTargets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/fspm_tf) -install(EXPORT fspm_tfTargets NAMESPACE fspm_tf:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) -configure_package_config_file(cmake/fspm_tfConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfig.cmake - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) -write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfigVersion.cmake +install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_DATADIR}/encoding3d) +install(EXPORT encoding3dTargets NAMESPACE encoding3d:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +configure_package_config_file(cmake/encoding3dConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake COMPATIBILITY SameMinorVersion) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/fspm_tfConfigVersion.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/fspm_tf) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/encoding3dConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/encoding3d) -if(FSPM_TF_BUILD_TESTS) +if(ENCODING3D_BUILD_TESTS) enable_testing() - set(FSPM_TF_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") - add_executable(fspm_tf_conformance_run tools/conformance_run.cpp) - target_link_libraries(fspm_tf_conformance_run PRIVATE fspm_tf::fspm_tf) - add_executable(fspm_tf_coupling_conformance tools/coupling_conformance.cpp) - target_link_libraries(fspm_tf_coupling_conformance PRIVATE fspm_tf::fspm_tf) - add_executable(fspm_tf_tests test/test_fspm_tf.cpp) - target_link_libraries(fspm_tf_tests PRIVATE fspm_tf::fspm_tf) - add_test(NAME fspm_tf_tests COMMAND fspm_tf_tests "${FSPM_TF_CONFORMANCE}") + set(ENCODING3D_CONFORMANCE "${CMAKE_CURRENT_SOURCE_DIR}/../../conformance") + add_executable(encoding3d_conformance_run tools/conformance_run.cpp) + target_link_libraries(encoding3d_conformance_run PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_coupling_conformance tools/coupling_conformance.cpp) + target_link_libraries(encoding3d_coupling_conformance PRIVATE encoding3d::encoding3d) + add_executable(encoding3d_tests test/test_encoding3d.cpp) + target_link_libraries(encoding3d_tests PRIVATE encoding3d::encoding3d) + add_test(NAME encoding3d_tests COMMAND encoding3d_tests "${ENCODING3D_CONFORMANCE}") endif() # CPlantBox's adapter: built when CPlantBox's sources and library are given -if(FSPM_TF_CPLANTBOX_SOURCE_DIR AND FSPM_TF_CPLANTBOX_LIBRARY) +if(ENCODING3D_CPLANTBOX_SOURCE_DIR AND ENCODING3D_CPLANTBOX_LIBRARY) add_subdirectory(cplantbox) endif() diff --git a/bindings/cpp/README.md b/bindings/cpp/README.md index 3582915..d1ada2c 100644 --- a/bindings/cpp/README.md +++ b/bindings/cpp/README.md @@ -1,24 +1,24 @@ -# fspm_tf (C++) +# encoding3d (C++) -The C++ codec of fspm-TF (C++17, CMake): reading and writing frames and sequences (spec 0.3). In `cplantbox/` is +The C++ codec of encoding3d (C++17, CMake): reading and writing frames and sequences (spec 0.3). In `cplantbox/` is CPlantBox's adapter, which turns an organism into a graph with its geometry, soil grid and segment-to-cell mapping, and turns a graph into a static CPlantBox root system. -- `include/fspm_tf/`: the data model (`model.hpp`) and the codec (`fspm_tf.hpp`). +- `include/encoding3d/`: the data model (`model.hpp`) and the codec (`encoding3d.hpp`). - `src/`: the GLB container, reading and writing, sequences, the conformance dump. - `tools/conformance_run.cpp`: the codec's dumps of the conformance files and its written scenes, for `.github/actions/conformance`. - `test/`: the tests, the conformance files included. -- `cplantbox/`: the adapter (`fspm_tf::cplantbox`), built when `FSPM_TF_CPLANTBOX_SOURCE_DIR` and - `FSPM_TF_CPLANTBOX_LIBRARY` are given. +- `cplantbox/`: the adapter (`encoding3d::cplantbox`), built when `ENCODING3D_CPLANTBOX_SOURCE_DIR` and + `ENCODING3D_CPLANTBOX_LIBRARY` are given. ```bash cmake -S bindings/cpp -B build && cmake --build build && ctest --test-dir build ``` -`fspm_tf/coupling.hpp` is a participant of coupling sessions (its ZeroMQ transport built when libzmq is found). +`encoding3d/coupling.hpp` is a participant of coupling sessions (its ZeroMQ transport built when libzmq is found). Its one dependency is nlohmann_json (header-only, MIT), found installed or fetched. It installs as a CMake package: -`find_package(fspm_tf)`, then `target_link_libraries(... fspm_tf::fspm_tf)`. See [the documentation](../../doc/cpp.md). +`find_package(encoding3d)`, then `target_link_libraries(... encoding3d::encoding3d)`. See [the documentation](../../doc/cpp.md). -Licence: GPL-3.0 (`LICENSE`), as CPlantBox; the rest of fspm-TF is CeCILL-C. +Licence: GPL-3.0 (`LICENSE`), as CPlantBox; the rest of encoding3d is CeCILL-C. diff --git a/bindings/cpp/cmake/encoding3dConfig.cmake.in b/bindings/cpp/cmake/encoding3dConfig.cmake.in new file mode 100644 index 0000000..fd62ca3 --- /dev/null +++ b/bindings/cpp/cmake/encoding3dConfig.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(nlohmann_json 3.11) +include("${CMAKE_CURRENT_LIST_DIR}/encoding3dTargets.cmake") +check_required_components(encoding3d) diff --git a/bindings/cpp/cmake/fspm_tfConfig.cmake.in b/bindings/cpp/cmake/fspm_tfConfig.cmake.in deleted file mode 100644 index 6051350..0000000 --- a/bindings/cpp/cmake/fspm_tfConfig.cmake.in +++ /dev/null @@ -1,6 +0,0 @@ -@PACKAGE_INIT@ - -include(CMakeFindDependencyMacro) -find_dependency(nlohmann_json 3.11) -include("${CMAKE_CURRENT_LIST_DIR}/fspm_tfTargets.cmake") -check_required_components(fspm_tf) diff --git a/bindings/cpp/cplantbox/CMakeLists.txt b/bindings/cpp/cplantbox/CMakeLists.txt index c5d730e..2fbf915 100644 --- a/bindings/cpp/cplantbox/CMakeLists.txt +++ b/bindings/cpp/cplantbox/CMakeLists.txt @@ -1,27 +1,27 @@ -# CPlantBox's adapter (fspm_tf::cplantbox), on CPlantBox's built library and its sources' headers: +# CPlantBox's adapter (encoding3d::cplantbox), on CPlantBox's built library and its sources' headers: # -# cmake -S bindings/cpp -B build -DFSPM_TF_CPLANTBOX_SOURCE_DIR=/path/to/CPlantBox \ -# -DFSPM_TF_CPLANTBOX_LIBRARY=/path/to/build/src/libCPlantBox.a +# cmake -S bindings/cpp -B build -DENCODING3D_CPLANTBOX_SOURCE_DIR=/path/to/CPlantBox \ +# -DENCODING3D_CPLANTBOX_LIBRARY=/path/to/build/src/libCPlantBox.a # # It is meant to move into CPlantBox once its team agrees (devplan_bindings.md, P4). -set(CPLANTBOX_SOURCE "${FSPM_TF_CPLANTBOX_SOURCE_DIR}/src") +set(CPLANTBOX_SOURCE "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/src") add_library(CPlantBox::CPlantBox UNKNOWN IMPORTED) set_target_properties(CPlantBox::CPlantBox PROPERTIES - IMPORTED_LOCATION "${FSPM_TF_CPLANTBOX_LIBRARY}" + IMPORTED_LOCATION "${ENCODING3D_CPLANTBOX_LIBRARY}" INTERFACE_INCLUDE_DIRECTORIES "${CPLANTBOX_SOURCE}/structural;${CPLANTBOX_SOURCE}/functional;${CPLANTBOX_SOURCE}/visualisation;${CPLANTBOX_SOURCE}/external/tinyxml2;${CPLANTBOX_SOURCE}/external/eigen") -add_library(fspm_tf_cplantbox src/cplantbox.cpp) -add_library(fspm_tf::cplantbox ALIAS fspm_tf_cplantbox) -target_include_directories(fspm_tf_cplantbox PUBLIC $) -target_link_libraries(fspm_tf_cplantbox PUBLIC fspm_tf::fspm_tf CPlantBox::CPlantBox) +add_library(encoding3d_cplantbox src/cplantbox.cpp) +add_library(encoding3d::cplantbox ALIAS encoding3d_cplantbox) +target_include_directories(encoding3d_cplantbox PUBLIC $) +target_link_libraries(encoding3d_cplantbox PUBLIC encoding3d::encoding3d CPlantBox::CPlantBox) if(MSVC) - target_compile_definitions(fspm_tf_cplantbox PRIVATE _USE_MATH_DEFINES NOMINMAX) + target_compile_definitions(encoding3d_cplantbox PRIVATE _USE_MATH_DEFINES NOMINMAX) endif() -if(FSPM_TF_BUILD_TESTS) - add_executable(fspm_tf_cplantbox_tests test/test_cplantbox.cpp) - target_link_libraries(fspm_tf_cplantbox_tests PRIVATE fspm_tf::cplantbox) - add_test(NAME fspm_tf_cplantbox_tests - COMMAND fspm_tf_cplantbox_tests "${FSPM_TF_CONFORMANCE}" "${FSPM_TF_CPLANTBOX_SOURCE_DIR}/modelparameter/structural" +if(ENCODING3D_BUILD_TESTS) + add_executable(encoding3d_cplantbox_tests test/test_cplantbox.cpp) + target_link_libraries(encoding3d_cplantbox_tests PRIVATE encoding3d::cplantbox) + add_test(NAME encoding3d_cplantbox_tests + COMMAND encoding3d_cplantbox_tests "${ENCODING3D_CONFORMANCE}" "${ENCODING3D_CPLANTBOX_SOURCE_DIR}/modelparameter/structural" "${CMAKE_CURRENT_BINARY_DIR}/cplantbox_out") endif() diff --git a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp similarity index 94% rename from bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp rename to bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp index 556f83c..f78cb4f 100644 --- a/bindings/cpp/cplantbox/include/fspm_tf/cplantbox.hpp +++ b/bindings/cpp/cplantbox/include/encoding3d/cplantbox.hpp @@ -1,8 +1,8 @@ -// CPlantBox's adapter: an organism (a Plant, a RootSystem, a MappedPlant) as an fspm-TF graph, its leaves' and +// CPlantBox's adapter: an organism (a Plant, a RootSystem, a MappedPlant) as an encoding3d graph, its leaves' and // stems' geometry as a mesh, its soil grid and segment-to-cell mapping; and back, a graph (CPlantBox's, or another // platform's) as a static CPlantBox root system. // -// | fspm-TF | CPlantBox | +// | encoding3d | CPlantBox | // |----------------------------------|---------------------------------------------------------------------------| // | Plant (rank 1), one row, id 0 | the organism | // | Organ (rank 5, scale "mixed"), | its organs: a seed, a root or a stem (a whole axis), a leaf; complex the | @@ -24,7 +24,7 @@ #include #include -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" namespace CPlantBox { class Organism; @@ -33,7 +33,7 @@ class MappedSegments; class Plant; } // namespace CPlantBox -namespace fspm_tf::cplantbox { +namespace encoding3d::cplantbox { /// The name of CPlantBox's entry in a data structure's extras, and of its column in the vocabulary. inline constexpr const char *PLATFORM = "cplantbox"; @@ -79,7 +79,7 @@ struct Values { }; /// Adds values per soil cell (in CPlantBox's cell order: x fastest, then y, then z) to the soil grid written by -/// soil_grid, as columns of its cells (reordered to fspm-TF's C order of x, y, z). +/// soil_grid, as columns of its cells (reordered to encoding3d's C order of x, y, z). void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segments, const std::vector &values); /// Adds values per segment of the MappedSegments (e.g. PlantHydraulicModel's radial fluxes) to the plant's SubOrgan @@ -100,4 +100,4 @@ void add_node_values(DataStructure &plant, const CPlantBox::MappedSegments &segm /// CPlantBox has static roots only. std::shared_ptr to_root_system(const DataStructure &data_structure); -} // namespace fspm_tf::cplantbox +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/src/cplantbox.cpp b/bindings/cpp/cplantbox/src/cplantbox.cpp index f4489f6..53f8296 100644 --- a/bindings/cpp/cplantbox/src/cplantbox.cpp +++ b/bindings/cpp/cplantbox/src/cplantbox.cpp @@ -1,4 +1,4 @@ -#include "fspm_tf/cplantbox.hpp" +#include "encoding3d/cplantbox.hpp" #include #include @@ -18,7 +18,7 @@ #include "rootparameter.h" #include "tinyxml2.h" -namespace fspm_tf::cplantbox { +namespace encoding3d::cplantbox { namespace { @@ -210,7 +210,7 @@ Frame to_frame(const CPlantBox::Organism &organism, const Options &options) { frame.data_structures.push_back(to_data_structure(organism, options)); frame.time = organism.getSimTime(); frame.time_unit = "d"; - frame.writer = {{"platform", "CPlantBox"}, {"adapter", "fspm_tf::cplantbox"}}; + frame.writer = {{"platform", "CPlantBox"}, {"adapter", "encoding3d::cplantbox"}}; frame.vocabulary_version = "0.1"; return frame; } @@ -293,7 +293,7 @@ Mapping segments_in_soil(const CPlantBox::MappedSegments &segments, const DataSt if (!known.count(id)) throw Error("segment " + std::to_string(segment) + " of the mapped segments is not in '" + plant.id + "': set the grid without cutting segments"); - // CPlantBox's cell index has x fastest, then y, then z from the bottom: fspm-TF's is C order of (x, y, z) + // CPlantBox's cell index has x fastest, then y, then z from the bottom: encoding3d's is C order of (x, y, z) const std::int64_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); sources.push_back(id); targets.push_back((i * ny + j) * nz + k); @@ -313,7 +313,7 @@ void add_cell_values(DataStructure &soil, const CPlantBox::MappedSegments &segme throw Error("'" + item.name + "': " + std::to_string(item.values.size()) + " values, the grid has " + std::to_string(nx * ny * nz) + " cells"); std::vector column(item.values.size()); - for (std::size_t cell = 0; cell < item.values.size(); ++cell) { // CPlantBox: x fastest; fspm-TF: C order + for (std::size_t cell = 0; cell < item.values.size(); ++cell) { // CPlantBox: x fastest; encoding3d: C order const std::size_t i = cell % nx, j = (cell / nx) % ny, k = cell / (nx * ny); column[(i * ny + j) * nz + k] = item.values[cell]; } @@ -538,4 +538,4 @@ std::shared_ptr to_root_system(const DataStructure &data_struc return plant; } -} // namespace fspm_tf::cplantbox +} // namespace encoding3d::cplantbox diff --git a/bindings/cpp/cplantbox/test/test_cplantbox.cpp b/bindings/cpp/cplantbox/test/test_cplantbox.cpp index fb90529..a999492 100644 --- a/bindings/cpp/cplantbox/test/test_cplantbox.cpp +++ b/bindings/cpp/cplantbox/test/test_cplantbox.cpp @@ -2,7 +2,7 @@ // and read back; a static root system rebuilt from its own frame and from another platform's (the conformance root // system, three scales). // -// fspm_tf_cplantbox_tests CONFORMANCE MODELPARAMETER OUT +// encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT #include #include #include @@ -11,13 +11,13 @@ #include #include -#include "fspm_tf/cplantbox.hpp" +#include "encoding3d/cplantbox.hpp" #include "MappedOrganism.h" #include "Plant.h" namespace filesystem = std::filesystem; -using namespace fspm_tf; +using namespace encoding3d; namespace { @@ -236,7 +236,7 @@ void test_another_platform(const filesystem::path &conformance) { } // namespace /// CPlantBox's log of a growing root system: a frame a day for six days (out/anagallis_log/), the sequence a coupling -/// session replays (bindings/python/test/test_coupling.py, FSPM_TF_CPLANTBOX_LOG). +/// session replays (bindings/python/test/test_coupling.py, ENCODING3D_CPLANTBOX_LOG). void test_growing_log(const filesystem::path ¶meters, const filesystem::path &out) { auto plant = std::make_shared(1); plant->readParameters((parameters / "plant" / "Anagallis_femina_Leitner_2010.xml").string(), "plant", true, false); @@ -263,7 +263,7 @@ void test_growing_log(const filesystem::path ¶meters, const filesystem::path int main(int argument_count, char **arguments) { if (argument_count != 4) { - std::cerr << "usage: fspm_tf_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; + std::cerr << "usage: encoding3d_cplantbox_tests CONFORMANCE MODELPARAMETER OUT\n"; return 2; } const filesystem::path conformance = arguments[1], parameters = arguments[2], out = arguments[3]; diff --git a/bindings/cpp/include/fspm_tf/coupling.hpp b/bindings/cpp/include/encoding3d/coupling.hpp similarity index 88% rename from bindings/cpp/include/fspm_tf/coupling.hpp rename to bindings/cpp/include/encoding3d/coupling.hpp index f5e306c..8962546 100644 --- a/bindings/cpp/include/fspm_tf/coupling.hpp +++ b/bindings/cpp/include/encoding3d/coupling.hpp @@ -1,15 +1,15 @@ // A participant's side of a coupling session (specification §11): a platform declares what it works on, provides and -// requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is fspm-TF's -// (`fspm-tf couple`). +// requires, then answers each turn's inputs with its outputs, as partial frames. The coordinator is encoding3d's +// (`encoding3d couple`). // -// fspm_tf::coupling::Participant participant(declaration); // a JSON declaration; endpoint from the environment +// encoding3d::coupling::Participant participant(declaration); // a JSON declaration; endpoint from the environment // while (auto turn = participant.next_turn()) { // until the coordinator stops the session // ... turn->frame: its entity sets, their entities, its inputs; the model from turn->time to turn->until // participant.send_update(frame); // its outputs and topology changes // } // // Transports: a folder ("files:path") always; ZeroMQ ("tcp://…", "ipc://…") when the library is built with libzmq -// (FSPM_TF_WITH_ZMQ). +// (ENCODING3D_WITH_ZMQ). #pragma once #include @@ -18,15 +18,15 @@ #include #include -#include "fspm_tf/model.hpp" +#include "encoding3d/model.hpp" -namespace fspm_tf::coupling { +namespace encoding3d::coupling { /// The version of the coupling protocol this library speaks. inline constexpr const char *PROTOCOL_VERSION = "0.3"; /// The environment variables a participant launched by the coordinator reads: the session's endpoint, its name. -inline constexpr const char *ENDPOINT_VARIABLE = "FSPM_TF_ENDPOINT"; -inline constexpr const char *PARTICIPANT_VARIABLE = "FSPM_TF_PARTICIPANT"; +inline constexpr const char *ENDPOINT_VARIABLE = "ENCODING3D_ENDPOINT"; +inline constexpr const char *PARTICIPANT_VARIABLE = "ENCODING3D_PARTICIPANT"; /// A message: its JSON header and, for turns and updates, a partial frame's GLB bytes. struct Message { @@ -90,4 +90,4 @@ class Participant { std::int64_t step_ = -1; }; -} // namespace fspm_tf::coupling +} // namespace encoding3d::coupling diff --git a/bindings/cpp/include/fspm_tf/fspm_tf.hpp b/bindings/cpp/include/encoding3d/encoding3d.hpp similarity index 90% rename from bindings/cpp/include/fspm_tf/fspm_tf.hpp rename to bindings/cpp/include/encoding3d/encoding3d.hpp index 93eb769..cddef80 100644 --- a/bindings/cpp/include/fspm_tf/fspm_tf.hpp +++ b/bindings/cpp/include/encoding3d/encoding3d.hpp @@ -1,6 +1,6 @@ -// Reading and writing fspm-TF files in C++: whole functional-structural plant model scenes (multiscale plant graphs, +// Reading and writing encoding3d files in C++: whole functional-structural plant model scenes (multiscale plant graphs, // their variables and parameters, their geometry, the environment's grids) at successive time steps, one glTF 2.0 -// binary (GLB) file per time step. The specification is spec/specification.md in the fspm-TF repository. +// binary (GLB) file per time step. The specification is spec/specification.md in the encoding3d repository. // // read_frame(path) -> Frame write_frame(path, frame) // from_bytes(data) -> Frame to_bytes(frame) -> bytes @@ -13,9 +13,9 @@ #include #include -#include "fspm_tf/model.hpp" +#include "encoding3d/model.hpp" -namespace fspm_tf { +namespace encoding3d { // ── The GLB container ───────────────────────────────────────────────────── @@ -70,8 +70,8 @@ std::vector read_sequence(const std::string &path); /// A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md): /// every table's columns as lists, NaN as null, ±∞ as "Infinity" / "-Infinity", positions as their float values. json frame_to_json(const Frame &frame); -/// The differences between two JSON values, as `fspm-tf compare` finds them: objects by key, arrays in order, +/// The differences between two JSON values, as `encoding3d compare` finds them: objects by key, arrays in order, /// numbers exactly (1 and 1.0 equal), as "path: what differs" lines (none when equal). std::vector compare(const json &actual, const json &expected, const std::string &path = "$"); -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/include/fspm_tf/model.hpp b/bindings/cpp/include/encoding3d/model.hpp similarity index 98% rename from bindings/cpp/include/fspm_tf/model.hpp rename to bindings/cpp/include/encoding3d/model.hpp index 4936ecd..28813eb 100644 --- a/bindings/cpp/include/fspm_tf/model.hpp +++ b/bindings/cpp/include/encoding3d/model.hpp @@ -1,4 +1,4 @@ -// The fspm-TF data model: a frame's data structures, their tables (columns of equal length), grids and meshes. +// The encoding3d data model: a frame's data structures, their tables (columns of equal length), grids and meshes. // // Frame one time step of a scene // DataStructure one plant population's graph, or one environment model's grid @@ -27,7 +27,7 @@ #include -namespace fspm_tf { +namespace encoding3d { using json = nlohmann::ordered_json; @@ -41,7 +41,7 @@ inline const std::vector EDGE_TYPES = {"none", "successor", "branch inline const std::vector BASE_SCALES = {"Plant", "Axis", "GrowthUnit", "Phytomer", "Organ", "SubOrgan", "Layer", "Cell", "Compartment", "Connection"}; -/// What the specification forbids, or a file that is not fspm-TF. +/// What the specification forbids, or a file that is not encoding3d. struct Error : std::runtime_error { using std::runtime_error::runtime_error; }; @@ -259,4 +259,4 @@ struct Frame { DataStructure &data_structure(const std::string &id); }; -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/src/bytes.hpp b/bindings/cpp/src/bytes.hpp index b7e0abb..c0bead8 100644 --- a/bindings/cpp/src/bytes.hpp +++ b/bindings/cpp/src/bytes.hpp @@ -7,9 +7,9 @@ #include #include -#include "fspm_tf/model.hpp" +#include "encoding3d/model.hpp" -namespace fspm_tf::detail { +namespace encoding3d::detail { inline bool machine_is_little_endian() { const std::uint16_t probe = 1; @@ -53,4 +53,4 @@ inline std::uint32_t read_word(const std::vector &data, std::size_ inline std::size_t padding(std::size_t length) { return (8 - length % 8) % 8; } -} // namespace fspm_tf::detail +} // namespace encoding3d::detail diff --git a/bindings/cpp/src/codec.cpp b/bindings/cpp/src/codec.cpp index ef45148..9170adf 100644 --- a/bindings/cpp/src/codec.cpp +++ b/bindings/cpp/src/codec.cpp @@ -2,7 +2,7 @@ // meshes carry EXT_mesh_features feature IDs, and the FSPM_scene root extension holds what these two cannot: data // structures, entity sets and their ranks, relations, grids, mappings, the frame's time, and the variables' units, // standard names, kinds and roles. -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" #include #include @@ -16,7 +16,7 @@ #include "bytes.hpp" -namespace fspm_tf { +namespace encoding3d { namespace { @@ -405,7 +405,7 @@ std::vector to_bytes(const Frame &frame) { json schema = {{"id", "fspm_scene"}, {"classes", json::object()}, {"enums", {{EDGE_TYPE_ENUM, {{"valueType", "UINT8"}, {"values", edge_values}}}}}}; json tables = json::array(), variables = json::object(); - json document = {{"asset", {{"version", "2.0"}, {"generator", std::string("fspm_tf (C++) ") + SPECIFICATION_VERSION}}}, + json document = {{"asset", {{"version", "2.0"}, {"generator", std::string("encoding3d (C++) ") + SPECIFICATION_VERSION}}}, {"scene", 0}, {"scenes", json::array({{{"nodes", json::array({0})}}})}, {"nodes", json::array({{{"name", "fspm_scene"}, {"matrix", Z_UP_TO_Y_UP}, {"children", json::array()}}})}}; @@ -567,7 +567,7 @@ struct ViewBytes { ViewBytes view_bytes(const json &document, const std::vector &binary, std::int64_t index) { const json &view = document.at("bufferViews").at(static_cast(index)); if (get(view, "buffer", 0).get() != 0) - throw Error("fspm-TF files keep every buffer view in the GLB binary chunk"); + throw Error("encoding3d files keep every buffer view in the GLB binary chunk"); const auto start = get(view, "byteOffset", 0).get(); const auto length = view.at("byteLength").get(); if (start + length > binary.size()) throw Error("a buffer view runs past the binary chunk"); @@ -612,7 +612,7 @@ Column decode_scalar_of(const std::string &component, const ViewBytes &raw, cons if (component == "UINT64") return decode_scalar(raw, property, count, width); if (component == "FLOAT32") return decode_scalar(raw, property, count, width); if (component == "FLOAT64") return decode_scalar(raw, property, count, width); - throw Error("component type " + component + " is not part of fspm-TF 0.1"); + throw Error("component type " + component + " is not part of encoding3d 0.1"); } std::vector read_unsigned(const std::string &type, const ViewBytes &raw, std::size_t count) { @@ -621,7 +621,7 @@ std::vector read_unsigned(const std::string &type, const ViewByte else if (type == "UINT16") for (auto value : read_view(raw, count)) values.push_back(value); else if (type == "UINT32") for (auto value : read_view(raw, count)) values.push_back(value); else if (type == "UINT64") values = read_view(raw, count); - else throw Error("offsets or enum values of type " + type + " are not part of fspm-TF 0.1"); + else throw Error("offsets or enum values of type " + type + " are not part of encoding3d 0.1"); return values; } @@ -678,7 +678,7 @@ void decode_table(const json &document, const std::vector &binary, } else if (type == "SCALAR") { const bool vector = get(property, "array", false).get(); if (vector && !property.contains("count")) - throw Error("'" + class_id + "." + name + "': variable-length arrays are not part of fspm-TF 0.1"); + throw Error("'" + class_id + "." + name + "': variable-length arrays are not part of encoding3d 0.1"); const std::size_t width = vector ? property["count"].get() : 1; const std::string component = property.at("componentType").get(); column = decode_scalar_of(component, raw, property, count, width); @@ -709,7 +709,7 @@ void decode_table(const json &document, const std::vector &binary, if (any) column.missing = missing; } } else { - throw Error("'" + class_id + "." + name + "': property type " + type + " is not part of fspm-TF 0.1"); + throw Error("'" + class_id + "." + name + "': property type " + type + " is not part of encoding3d 0.1"); } table.columns.emplace_back(name, std::move(column)); const json declared = get(get(variables, class_id, json::object()), name); @@ -737,7 +737,7 @@ std::vector accessor_values(const json &document, const std::vector(index)); const std::string type = accessor.at("type").get(); width = type == "SCALAR" ? 1 : type == "VEC2" ? 2 : type == "VEC3" ? 3 : type == "VEC4" ? 4 : 0; - if (width == 0) throw Error("accessor type " + type + " is not used by fspm-TF"); + if (width == 0) throw Error("accessor type " + type + " is not used by encoding3d"); const auto view_index = accessor.at("bufferView").get(); const auto raw = view_bytes(document, binary, view_index); const json &view = document["bufferViews"][static_cast(view_index)]; @@ -758,7 +758,7 @@ std::vector accessor_values(const json &document, const std::vector &binary, const json &mesh_entry, @@ -798,7 +798,7 @@ Mesh decode_mesh(const json &document, const std::vector &binary, Frame from_bytes(const std::vector &data) { auto [document, binary] = unpack(data); const json extensions = get(document, "extensions", json::object()); - if (!extensions.contains(EXTENSION)) throw Error("not an fspm-TF file: no " + EXTENSION + " extension"); + if (!extensions.contains(EXTENSION)) throw Error("not an encoding3d file: no " + EXTENSION + " extension"); const json &scene = extensions[EXTENSION]; const json metadata = get(extensions, METADATA, {{"schema", json::object()}, {"propertyTables", json::array()}}); const json variables = get(scene, "variables", json::object()); @@ -902,4 +902,4 @@ Frame read_frame(const std::string &path) { return from_bytes(data); } -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/src/coupling.cpp b/bindings/cpp/src/coupling.cpp index a5da489..66d6ba8 100644 --- a/bindings/cpp/src/coupling.cpp +++ b/bindings/cpp/src/coupling.cpp @@ -1,6 +1,6 @@ // A participant's side of a coupling session: the folder transport, the participant, and the ZeroMQ transport when -// built with libzmq (FSPM_TF_WITH_ZMQ). -#include "fspm_tf/coupling.hpp" +// built with libzmq (ENCODING3D_WITH_ZMQ). +#include "encoding3d/coupling.hpp" #include #include @@ -11,13 +11,13 @@ #include #include -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" -#ifdef FSPM_TF_WITH_ZMQ +#ifdef ENCODING3D_WITH_ZMQ #include #endif -namespace fspm_tf::coupling { +namespace encoding3d::coupling { namespace { @@ -99,7 +99,7 @@ class FolderChannel : public Channel { std::vector read_files_; // those of the messages read and kept }; -#ifdef FSPM_TF_WITH_ZMQ +#ifdef ENCODING3D_WITH_ZMQ /// A DEALER socket named after the participant, connected to the coordinator's ROUTER socket; a message is one or two /// frames, its header then its payload. class ZmqChannel : public Channel { @@ -158,7 +158,7 @@ std::string environment(const char *name) { } // namespace bool has_zeromq() { -#ifdef FSPM_TF_WITH_ZMQ +#ifdef ENCODING3D_WITH_ZMQ return true; #else return false; @@ -167,10 +167,10 @@ bool has_zeromq() { std::unique_ptr participant_channel(const std::string &endpoint, const std::string &participant) { if (endpoint.rfind("files:", 0) == 0) return std::make_unique(endpoint.substr(6), participant); -#ifdef FSPM_TF_WITH_ZMQ +#ifdef ENCODING3D_WITH_ZMQ return std::make_unique(endpoint, participant); #else - throw Error("the endpoint " + endpoint + " needs ZeroMQ: build fspm_tf with libzmq, or use a folder (files:path)"); + throw Error("the endpoint " + endpoint + " needs ZeroMQ: build encoding3d with libzmq, or use a folder (files:path)"); #endif } @@ -220,4 +220,4 @@ void Participant::send_update(Frame frame) { void Participant::fail(const std::string &message) { channel_->send({{"type", "error"}, {"message", message}}); } -} // namespace fspm_tf::coupling +} // namespace encoding3d::coupling diff --git a/bindings/cpp/src/dump.cpp b/bindings/cpp/src/dump.cpp index b31fd59..436a5c3 100644 --- a/bindings/cpp/src/dump.cpp +++ b/bindings/cpp/src/dump.cpp @@ -1,11 +1,11 @@ // A frame as plain JSON values, in the layout of the conformance files' expected values (conformance/README.md), and -// the comparison `fspm-tf compare` makes. -#include "fspm_tf/fspm_tf.hpp" +// the comparison `encoding3d compare` makes. +#include "encoding3d/encoding3d.hpp" #include #include -namespace fspm_tf { +namespace encoding3d { namespace { @@ -180,4 +180,4 @@ std::vector compare(const json &actual, const json &expected, const return {}; } -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/src/glb.cpp b/bindings/cpp/src/glb.cpp index 410d3bc..287b777 100644 --- a/bindings/cpp/src/glb.cpp +++ b/bindings/cpp/src/glb.cpp @@ -1,10 +1,10 @@ // The GLB container: a 12-byte header (magic, version 2, total length), a JSON chunk padded with spaces, a binary // chunk padded with zeros, each 8-byte aligned (as EXT_structural_metadata requires). Little-endian throughout. -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" #include "bytes.hpp" -namespace fspm_tf { +namespace encoding3d { namespace { constexpr std::uint32_t GLB_MAGIC = 0x46546C67; // "glTF" @@ -69,4 +69,4 @@ std::pair> unpack(const std::vector #include -namespace fspm_tf { +namespace encoding3d { std::size_t Column::size() const { return std::visit([](const auto &column_values) { return column_values.size(); }, values); @@ -145,4 +145,4 @@ DataStructure &Frame::data_structure(const std::string &id) { return const_cast(static_cast(*this).data_structure(id)); } -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/src/sequence.cpp b/bindings/cpp/src/sequence.cpp index 6537fa7..c364619 100644 --- a/bindings/cpp/src/sequence.cpp +++ b/bindings/cpp/src/sequence.cpp @@ -1,5 +1,5 @@ // Sequences: a folder of frames (one GLB each) and their index, scene.fspm.json. -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" #include #include @@ -7,7 +7,7 @@ #include #include -namespace fspm_tf { +namespace encoding3d { void write_index(const std::string &path, const Index &index) { json frames = json::array(); @@ -62,4 +62,4 @@ std::vector read_sequence(const std::string &path) { return frames; } -} // namespace fspm_tf +} // namespace encoding3d diff --git a/bindings/cpp/test/test_fspm_tf.cpp b/bindings/cpp/test/test_encoding3d.cpp similarity index 97% rename from bindings/cpp/test/test_fspm_tf.cpp rename to bindings/cpp/test/test_encoding3d.cpp index 1a38fdd..8a9dab9 100644 --- a/bindings/cpp/test/test_fspm_tf.cpp +++ b/bindings/cpp/test/test_encoding3d.cpp @@ -1,7 +1,7 @@ // The C++ codec's tests: the conformance files read as their expected values, written and read back the same, every // column type, meshes, sequences, and what the specification forbids refused. // -// fspm_tf_tests CONFORMANCE +// encoding3d_tests CONFORMANCE #include #include #include @@ -12,10 +12,10 @@ #include #include -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" namespace filesystem = std::filesystem; -using namespace fspm_tf; +using namespace encoding3d; namespace { @@ -190,7 +190,7 @@ void test_refused() { } void test_sequence(const filesystem::path &conformance) { - const auto folder = filesystem::temp_directory_path() / "fspm_tf_cpp_sequence"; + const auto folder = filesystem::temp_directory_path() / "encoding3d_cpp_sequence"; filesystem::remove_all(folder); std::vector frames; for (int step = 0; step < 3; ++step) { @@ -239,7 +239,7 @@ void test_partial_frame() { int main(int argument_count, char **arguments) { if (argument_count != 2) { - std::cerr << "usage: fspm_tf_tests CONFORMANCE\n"; + std::cerr << "usage: encoding3d_tests CONFORMANCE\n"; return 2; } const filesystem::path conformance = arguments[1]; diff --git a/bindings/cpp/tools/conformance_run.cpp b/bindings/cpp/tools/conformance_run.cpp index d101562..dfbc0b9 100644 --- a/bindings/cpp/tools/conformance_run.cpp +++ b/bindings/cpp/tools/conformance_run.cpp @@ -2,18 +2,18 @@ // it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb). // Then .github/actions/conformance compares the dumps with the expected values and validates the written files. // -// fspm_tf_conformance_run CONFORMANCE OUT +// encoding3d_conformance_run CONFORMANCE OUT #include #include #include -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/encoding3d.hpp" namespace filesystem = std::filesystem; int main(int argument_count, char **arguments) { if (argument_count != 3) { - std::cerr << "usage: fspm_tf_conformance_run CONFORMANCE OUT\n"; + std::cerr << "usage: encoding3d_conformance_run CONFORMANCE OUT\n"; return 2; } const filesystem::path conformance = arguments[1], out = arguments[2]; @@ -21,18 +21,18 @@ int main(int argument_count, char **arguments) { if (!entry.is_regular_file() || entry.path().extension() != ".glb") continue; if (filesystem::relative(entry.path(), conformance).begin()->string() == "coupling") continue; // sessions const auto relative = filesystem::relative(entry.path(), conformance); - const auto frame = fspm_tf::read_frame(entry.path().string()); + const auto frame = encoding3d::read_frame(entry.path().string()); auto dump = out / "dumps" / relative; dump.replace_extension(".json"); const auto written = out / "written" / relative; filesystem::create_directories(dump.parent_path()); filesystem::create_directories(written.parent_path()); - std::ofstream(dump, std::ios::binary) << fspm_tf::frame_to_json(frame).dump(1) << "\n"; - fspm_tf::write_frame(written.string(), frame); // written again from what was read + std::ofstream(dump, std::ios::binary) << encoding3d::frame_to_json(frame).dump(1) << "\n"; + encoding3d::write_frame(written.string(), frame); // written again from what was read } - const auto index = conformance / "growing_sequence" / fspm_tf::INDEX_FILE; + const auto index = conformance / "growing_sequence" / encoding3d::INDEX_FILE; if (filesystem::exists(index)) - filesystem::copy_file(index, out / "written" / "growing_sequence" / fspm_tf::INDEX_FILE, + filesystem::copy_file(index, out / "written" / "growing_sequence" / encoding3d::INDEX_FILE, filesystem::copy_options::overwrite_existing); return 0; } diff --git a/bindings/cpp/tools/coupling_conformance.cpp b/bindings/cpp/tools/coupling_conformance.cpp index 4d5f718..918d322 100644 --- a/bindings/cpp/tools/coupling_conformance.cpp +++ b/bindings/cpp/tools/coupling_conformance.cpp @@ -1,17 +1,17 @@ // The C++ participant under test in the coupling conformance sessions (conformance/coupling, specification §9): -// launched by the coordinator as `fspm_tf_coupling_conformance SESSION OUT`, it joins with the session's +// launched by the coordinator as `encoding3d_coupling_conformance SESSION OUT`, it joins with the session's // subject/declaration.json, writes what it reads from each turn to OUT/turn_.json, and sends // subject/update_.glb as read and written again by this library. // -// fspm-tf couple-conformance conformance/coupling --participant build/fspm_tf_coupling_conformance +// encoding3d couple-conformance conformance/coupling --participant build/encoding3d_coupling_conformance #include #include #include #include #include -#include "fspm_tf/coupling.hpp" -#include "fspm_tf/fspm_tf.hpp" +#include "encoding3d/coupling.hpp" +#include "encoding3d/encoding3d.hpp" namespace filesystem = std::filesystem; @@ -23,20 +23,20 @@ std::string numbered(std::int64_t step) { int main(int count, char **arguments) { if (count != 3) { - std::cerr << "usage: fspm_tf_coupling_conformance SESSION OUT\n"; + std::cerr << "usage: encoding3d_coupling_conformance SESSION OUT\n"; return 2; } const filesystem::path session = arguments[1], out = arguments[2]; filesystem::create_directories(out); std::ifstream declaration_file(session / "subject" / "declaration.json"); - fspm_tf::coupling::Participant participant(fspm_tf::json::parse(declaration_file), "", fspm_tf::json::array(), + encoding3d::coupling::Participant participant(encoding3d::json::parse(declaration_file), "", encoding3d::json::array(), 120.0); while (auto turn = participant.next_turn()) { std::ofstream dump(out / ("turn_" + numbered(turn->step) + ".json")); - dump << fspm_tf::frame_to_json(turn->frame).dump(1); + dump << encoding3d::frame_to_json(turn->frame).dump(1); dump.close(); - const auto update = fspm_tf::read_frame((session / "subject" / ("update_" + numbered(turn->step) + ".glb")).string()); - participant.send_update(fspm_tf::from_bytes(fspm_tf::to_bytes(update))); + const auto update = encoding3d::read_frame((session / "subject" / ("update_" + numbered(turn->step) + ".glb")).string()); + participant.send_update(encoding3d::from_bytes(encoding3d::to_bytes(update))); } return 0; } diff --git a/bindings/java/README.md b/bindings/java/README.md index 6f83f73..89702fa 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -1,20 +1,20 @@ -# fspm-tf (Java) +# encoding3d (Java) -The Java codec of fspm-TF (Java 17, Maven): reading and writing frames and sequences (spec 0.3), with one dependency, +The Java codec of encoding3d (Java 17, Maven): reading and writing frames and sequences (spec 0.3), with one dependency, Jackson. GroIMP's adapter is in `groimp/`. -- `src/main/java/org/openalea/fspmtf/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB +- `src/main/java/org/openalea/encoding3d/`: the data model (`Frame`, `DataStructure`, `EntitySet`, `Column`, …), the GLB container (`Glb`), reading and writing (`Codec`), sequences (`Sequences`), the conformance dump (`Dump`) and tool (`ConformanceRun`). - `src/test/`: the tests, the conformance files included. -- `groimp/`: GroIMP's adapter (`org.openalea.fspmtf.groimp.GroImp`, Java 21), on GroIMP's Graph module. +- `groimp/`: GroIMP's adapter (`org.openalea.encoding3d.groimp.GroImp`, Java 21), on GroIMP's Graph module. ```bash mvn -f bindings/java/pom.xml verify ``` -`org.openalea.fspmtf.coupling` is a participant of coupling sessions (its ZeroMQ transport with JeroMQ, an optional +`org.openalea.encoding3d.coupling` is a participant of coupling sessions (its ZeroMQ transport with JeroMQ, an optional dependency). -Licence: GPL-3.0-or-later (`LICENSE`), as GroIMP; the rest of fspm-TF is CeCILL-C. See +Licence: GPL-3.0-or-later (`LICENSE`), as GroIMP; the rest of encoding3d is CeCILL-C. See [the documentation](../../doc/java.md). diff --git a/bindings/java/groimp-plugin/README.md b/bindings/java/groimp-plugin/README.md index bda19e3..793ec7d 100644 --- a/bindings/java/groimp-plugin/README.md +++ b/bindings/java/groimp-plugin/README.md @@ -1,7 +1,7 @@ -# GroIMP's fspm-TF plugin +# GroIMP's encoding3d plugin -A GroIMP plugin (`org.openalea.fspmtf`, GroIMP 3.0): **File > Export** of the 3D view to a `.glb` file, or the CLI's -`export3d file.glb`, writes the project as an fspm-TF frame: +A GroIMP plugin (`org.openalea.encoding3d`, GroIMP 3.0): **File > Export** of the 3D view to a `.glb` file, or the CLI's +`export3d file.glb`, writes the project as an encoding3d frame: - the graph below the project's root: its scales, topology and the nodes' fields (`GroImp.toDataStructure`, `../groimp`); @@ -12,10 +12,10 @@ gitlab.com/grogra/groimp-plugins/gltf), which it requires. ```bash mvn -f bindings/java/pom.xml install && mvn -f bindings/java/groimp/pom.xml install -mvn -f bindings/java/groimp-plugin/pom.xml package # the plugin's folder: target/plugin/fspmTF +mvn -f bindings/java/groimp-plugin/pom.xml package # the plugin's folder: target/plugin/encoding3d ``` -To install it, copy `target/plugin/fspmTF` into GroIMP's `plugins/` folder, next to the glTF plugin's. +To install it, copy `target/plugin/encoding3d` into GroIMP's `plugins/` folder, next to the glTF plugin's. `test/run_in_groimp.sh` does this in a GroIMP install, grows an example headless, exports it and checks the frame. It is run by the `groimp-plugin` job of `.github/workflows/java-codec.yml`. diff --git a/bindings/java/groimp-plugin/pom.xml b/bindings/java/groimp-plugin/pom.xml index 47ec604..d360437 100644 --- a/bindings/java/groimp-plugin/pom.xml +++ b/bindings/java/groimp-plugin/pom.xml @@ -1,10 +1,10 @@ - 4.0.0 org.openalea - fspm-tf-groimp-plugin + encoding3d-groimp-plugin 0.1.0 jar - fspm-TF, GroIMP's plugin + encoding3d, GroIMP's plugin GPL-3.0-or-later @@ -39,7 +39,7 @@ org.openalea - fspm-tf-groimp + encoding3d-groimp 0.1.0 @@ -87,7 +87,7 @@ - fspmTF + encoding3d org.apache.maven.plugins @@ -106,7 +106,7 @@ package copy-dependencies - ${project.build.directory}/plugin/fspmTF/ext + ${project.build.directory}/plugin/encoding3d/ext runtime com.fasterxml.jackson.core @@ -124,8 +124,8 @@ run - - + + diff --git a/bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java b/bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java similarity index 92% rename from bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java rename to bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java index 93411fa..643c8dc 100644 --- a/bindings/java/groimp-plugin/src/main/java/org/openalea/fspmtf/groimp/plugin/FspmTfExport.java +++ b/bindings/java/groimp-plugin/src/main/java/org/openalea/encoding3d/groimp/plugin/Encoding3dExport.java @@ -1,4 +1,4 @@ -package org.openalea.fspmtf.groimp.plugin; +package org.openalea.encoding3d.groimp.plugin; import java.io.File; import java.io.IOException; @@ -7,12 +7,12 @@ import java.util.List; import java.util.Map; -import org.openalea.fspmtf.Codec; -import org.openalea.fspmtf.DataStructure; -import org.openalea.fspmtf.EntitySet; -import org.openalea.fspmtf.Frame; -import org.openalea.fspmtf.Mesh; -import org.openalea.fspmtf.groimp.GroImp; +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Mesh; +import org.openalea.encoding3d.groimp.GroImp; import de.grogra.gltf.exporter.GLTFExport; import de.grogra.gltf.exporter.GLTFNodeWrapper; @@ -31,17 +31,17 @@ import de.javagl.jgltf.model.impl.DefaultNodeModel; /** - * The 3D view of a GroIMP project as an fspm-TF frame (a .glb file): the graph below the project's root (its scales, + * The 3D view of a GroIMP project as an encoding3d frame (a .glb file): the graph below the project's root (its scales, * topology and fields, by {@link GroImp#toDataStructure}) and the scene's geometry as one mesh, each triangle carrying * the rows of its node and of the node's complexes. The shapes are meshed by GroIMP's glTF plugin (its per-shape * exporters, as for a .gltf file); this class collects their triangles in the scene's frame (GroIMP's: z up), node by * node, instead of writing glTF. */ -public class FspmTfExport extends GLTFExport { +public class Encoding3dExport extends GLTFExport { /** The id of no node, for meshes whose node is not an entity (e.g. the scene's ground). */ private static final long NO_NODE = -1; - public FspmTfExport(FilterItem item, FilterSource source) { + public Encoding3dExport(FilterItem item, FilterSource source) { super(item, source); } @@ -63,7 +63,7 @@ public void write(File file) throws IOException { Frame frame = new Frame(); frame.dataStructures.add(plants); - frame.writer.put("platform", "GroIMP").put("adapter", "org.openalea.fspmtf.groimp.plugin"); + frame.writer.put("platform", "GroIMP").put("adapter", "org.openalea.encoding3d.groimp.plugin"); frame.vocabularyVersion = "0.2"; Codec.write(file.toPath(), frame); } @@ -145,7 +145,7 @@ private static int[] indices(AccessorData data) { return values; } - /** The triangles as an fspm-TF mesh, three vertices each, with each vertex's row in every entity set: its node's, + /** The triangles as an encoding3d mesh, three vertices each, with each vertex's row in every entity set: its node's, * and its node's complexes' up the scales (NO_ID where it has none). */ private static Mesh mesh(DataStructure plants, List positions, List nodes) { Map where = new HashMap<>(); // id -> (entity set, row) diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.properties b/bindings/java/groimp-plugin/src/main/resources/plugin.properties index 8a0c5f7..63a920b 100644 --- a/bindings/java/groimp-plugin/src/main/resources/plugin.properties +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.properties @@ -1,4 +1,4 @@ -pluginName = fspm-TF -provider = OpenAlea (fspm-TF) -/io/filetypes/fspmTF.Name = fspm-TF frame (*.glb) -/io/mimetypes/application/x-java-jvm-local-objectref/fspmTFExport.Name = fspm-TF frame +pluginName = encoding3d +provider = OpenAlea (encoding3d) +/io/filetypes/encoding3d.Name = encoding3d frame (*.glb) +/io/mimetypes/application/x-java-jvm-local-objectref/encoding3dExport.Name = encoding3d frame diff --git a/bindings/java/groimp-plugin/src/main/resources/plugin.xml b/bindings/java/groimp-plugin/src/main/resources/plugin.xml index dc01b69..013a83d 100644 --- a/bindings/java/groimp-plugin/src/main/resources/plugin.xml +++ b/bindings/java/groimp-plugin/src/main/resources/plugin.xml @@ -1,13 +1,13 @@ - - - + + @@ -15,14 +15,14 @@ - + - + output="model/encoding3d; io=fwriter" + create="org.openalea.encoding3d.groimp.plugin.Encoding3dExport"> "$out/groimp.log" 2>&1 test -s "$frame" || { echo "no frame written"; tail -40 "$out/groimp.log"; exit 1; } -fspm-tf validate "$frame" +encoding3d validate "$frame" python - "$frame" <<'PYTHON' import sys import numpy as np -from openalea import fspm_tf -plants = fspm_tf.read(sys.argv[1]).data_structure("plants") +from openalea import encoding3d +plants = encoding3d.read(sys.argv[1]).data_structure("plants") mesh = plants.meshes[0] rows = mesh.feature_rows[plants.entity_sets[-1].name] assert len(mesh.positions) > 0 and (rows >= 0).all(), "every vertex carries its node" @@ -45,7 +45,7 @@ mkdir -p "$log" python - "$log" <<'PYTHON' import os import sys -from openalea.fspm_tf import read, write, write_index +from openalea.encoding3d import read, write, write_index folder = sys.argv[1] files = sorted(name for name in os.listdir(folder) if name.endswith(".glb")) assert len(files) == 4, f"four frames expected, {files}" @@ -57,4 +57,4 @@ write_index(os.path.join(folder, "scene.fspm.json"), [{"time": float(step), "step": step, "file": name} for step, name in enumerate(files)], time_unit="d") print(f"GroIMP's log: {len(files)} frames in {folder}") PYTHON -fspm-tf validate "$log/scene.fspm.json" +encoding3d validate "$log/scene.fspm.json" diff --git a/bindings/java/groimp/pom.xml b/bindings/java/groimp/pom.xml index a249847..6a23aa4 100644 --- a/bindings/java/groimp/pom.xml +++ b/bindings/java/groimp/pom.xml @@ -1,5 +1,5 @@ - + + 4.0.0 + org.openalea.tutorials + encoding3d-tutorial-groimp + 0.1.0 + + + 21 + UTF-8 + + + + + groimp + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d-groimp + 0.1.0 + + + de.grogra + graph + 3.0.0 + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + tutorial.GroImpTutorial + outputs + + + + + diff --git a/tutorials/02d_groimp/run.sh b/tutorials/02d_groimp/run.sh new file mode 100755 index 0000000..d39e544 --- /dev/null +++ b/tutorials/02d_groimp/run.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Runs tutorial 2d (README.md): Java 21 and Maven; encoding3d's Java codec and GroIMP's adapter installed in Maven's +# local repository first (from the repository's root, once). Its files in outputs/. +set -euo pipefail +cd "$(dirname "$0")" +mvn -q -f ../../bindings/java/pom.xml install -DskipTests +mvn -q -f ../../bindings/java/groimp/pom.xml install -DskipTests +mvn -q compile exec:java +if command -v encoding3d > /dev/null; then encoding3d validate outputs/frames/scene.fspm.json | head -1; fi diff --git a/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java new file mode 100644 index 0000000..d5da424 --- /dev/null +++ b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java @@ -0,0 +1,114 @@ +package tutorial; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.openalea.encoding3d.Codec; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Sequences; +import org.openalea.encoding3d.Variable; +import org.openalea.encoding3d.groimp.GroImp; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** Tutorial 2d: a GroIMP graph written and read as encoding3d (README.md explains each part). */ +public final class GroImpTutorial { + private GroImpTutorial() {} + + // ── 1. A GroIMP plant: node classes with managed fields, as an RGG model declares them ────────────────── + public static final class Plant extends Node { + String species = "Wheat"; + public static final NType $TYPE = Managed.type(Plant.class, "species"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Plant(); } + } + + public static final class Internode extends Node { + double length = 0.02; + float radius = 0.002f; + public static final NType $TYPE = Managed.type(Internode.class, "length", "radius"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Internode(); } + } + + public static final class Leaf extends Node { + double length = 0.01; + public static final NType $TYPE = Managed.type(Leaf.class, "length"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Leaf(); } + } + + /** The plant grows as an RGG rule would make it: a new internode bearing a leaf on top, every leaf longer. */ + static Internode grow(Internode top, List leaves) { + for (Leaf leaf : leaves) leaf.length += 0.02; + Internode next = new Internode(); + Leaf leaf = new Leaf(); + top.addEdgeBitsTo(next, Graph.SUCCESSOR_EDGE, null); // GroIMP's '>' edge + next.addEdgeBitsTo(leaf, Graph.BRANCH_EDGE, null); // GroIMP's '+' edge + leaves.add(leaf); + return next; + } + + /** How the graph is written: its scale below the plant is the vocabulary's Organ, lengths in metres. */ + static GroImp.Options options() { + GroImp.Options options = new GroImp.Options(); + options.id = "plant"; + options.scaleNames.put(2, "Organ"); + options.scaleRanks.put(2, 5L); + options.scaleScales.put(2, "Organ"); + options.labels.put("Internode", "StemInternode"); + options.variables.put("length", new Variable("m", "length", "extensive", "state")); + options.variables.put("radius", new Variable("m", "radius", "intensive", "state")); + options.variables.put("species", new Variable("1", null, "descriptor", "parameter")); + return options; + } + + public static void main(String[] arguments) throws Exception { + Path out = Path.of(arguments.length > 0 ? arguments[0] : "outputs"); + GraphManager graph = new GraphManager(new ServerConnection(null), "tutorial", false, false); + Node root = new Node(); + Plant plant = new Plant(); + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Internode top = new Internode(); + plant.addEdgeBitsTo(top, Graph.REFINEMENT_EDGE, null); // GroIMP's '/' edge: the plant is made of them + graph.setRoot(Graph.MAIN_GRAPH, root); + List leaves = new ArrayList<>(); + + // ── 2. A frame per step ───────────────────────────────────────────────────────────────────────────── + List frames = new ArrayList<>(); + for (int step = 0; step < 6; step++) { + if (step > 0) top = grow(top, leaves); + DataStructure written = GroImp.toDataStructure(root, options()); + Frame frame = new Frame(); + frame.dataStructures.add(written); + frame.time = step; + frame.step = step; + frames.add(frame); + EntitySet organs = written.entitySet("Organ"); + System.out.println("step " + step + ": " + organs.rowCount() + " organs, scales " + + written.entitySets.stream().map(set -> set.name).toList()); + } + Path index = Sequences.writeSequence(out.resolve("frames"), frames); + System.out.println("written: " + index); + + // ── 3. Back into GroIMP nodes ───────────────────────────────────────────────────────────────────────── + DataStructure last = Codec.read(out.resolve("frames").resolve("frame_000005.glb")).dataStructures.get(0); + Map> classes = Map.of("Plant", Plant::new, "Internode", Internode::new, "Leaf", Leaf::new); + Map nodes = new HashMap<>(); + Node rebuilt = GroImp.toGraph(last, label -> classes.get(label).get(), nodes); + new GraphManager(new ServerConnection(null), "rebuilt", false, false).setRoot(Graph.MAIN_GRAPH, rebuilt); + double leafLength = nodes.values().stream().filter(node -> node instanceof Leaf) + .mapToDouble(node -> ((Leaf) node).length).sum(); + System.out.println("rebuilt as GroIMP nodes: " + nodes.size() + " nodes, leaves " + Math.round(leafLength * 100) + + " cm long in all"); + } +} diff --git a/tutorials/02d_groimp/src/main/java/tutorial/Managed.java b/tutorials/02d_groimp/src/main/java/tutorial/Managed.java new file mode 100644 index 0000000..259fca7 --- /dev/null +++ b/tutorials/02d_groimp/src/main/java/tutorial/Managed.java @@ -0,0 +1,77 @@ +package tutorial; + +import java.lang.reflect.Field; + +import de.grogra.graph.impl.Node; +import de.grogra.graph.impl.Node.NType; +import de.grogra.reflect.Type; + +/** + * GroIMP's "managed fields": the fields of a node class GroIMP knows by name, as its XL compiler declares them for the + * classes of an RGG model. GroIMP's adapter writes them as columns. Here they are declared from plain Java fields by + * reflection, so that the tutorial's node classes stay short. + */ +final class Managed { + private Managed() {} + + /** The NType of {@code type}, with its fields {@code names} managed. */ + static NType type(Class type, String... names) { + NType nodeType = new NType(type); + for (String name : names) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + nodeType.addManagedField(new ManagedField(nodeType, field)); + } catch (NoSuchFieldException error) { + throw new IllegalArgumentException(type.getSimpleName() + " has no field " + name, error); + } + } + nodeType.validate(); + return nodeType; + } + + private static final class ManagedField extends NType.Field { + private final Field field; + + ManagedField(NType type, Field field) { + super(type, field.getName(), PUBLIC | SCO, typeOf(field.getType()), null); + this.field = field; + } + + private static Type typeOf(Class type) { + if (type == double.class) return Type.DOUBLE; + if (type == float.class) return Type.FLOAT; + if (type == int.class) return Type.INT; + if (type == boolean.class) return Type.BOOLEAN; + if (type == String.class) return Type.STRING; + throw new IllegalArgumentException("a managed field of type " + type); + } + + private Object read(Object node) { + try { + return field.get(node); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + private void write(Object node, Object value) { + try { + field.set(node, value); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + @Override public double getDouble(Object node) { return (double) read(node); } + @Override public void setDouble(Object node, double value) { write(node, value); } + @Override public float getFloat(Object node) { return (float) read(node); } + @Override public void setFloat(Object node, float value) { write(node, value); } + @Override public int getInt(Object node) { return (int) read(node); } + @Override public void setInt(Object node, int value) { write(node, value); } + @Override public boolean getBoolean(Object node) { return (boolean) read(node); } + @Override public void setBoolean(Object node, boolean value) { write(node, value); } + @Override public Object getObject(Object node) { return read(node); } + @Override protected void setObjectImpl(Object node, Object value) { write(node, value); } + } +} diff --git a/tutorials/README.md b/tutorials/README.md index 5c1c026..8bc7e98 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -9,6 +9,7 @@ its explanation (`README.md`), its code, and `run.sh`, which runs it from start | `02a_metafspm` | a metafspm scene recorded as frames, read by encoding3d alone, logged smaller, rebuilt in metafspm | metafspm's environment | | `02b_virtualplantlab` | a VirtualPlantLab plant grown by rules, written with its geometry, rebuilt and grown on; an MTG to and from encoding3d | Julia 1.12, VirtualPlantLab's packages, MTG.jl | | `02c_cplantbox` | a CPlantBox plant written day by day with its geometry, soil grid and mapping; frames back as CPlantBox root systems | C++17, CMake, CPlantBox built | +| `02d_groimp` | a GroIMP graph written step by step and rebuilt as GroIMP nodes; GroIMP's plugin exporting a project with its geometry | Java 21, Maven (GroIMP's Graph module) | **encoding3d depends on no platform.** Tutorial 1 needs encoding3d only, and encoding3d's CI runs it. The tutorials using a platform (metafspm, VirtualPlantLab, CPlantBox, GroIMP) list what they need in their own folder, outside From a1e9a72f30096d5fe4b0ed83560d2b85290e8a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Thu, 5 Nov 2026 12:38:21 +0100 Subject: [PATCH 37/39] Tutorial 3 (metafspm and VirtualPlantLab on one plant: metafspm's sugar-limited root growth and VirtualPlantLab's carbon model splitting the segments into layers by a PlantGraphs rule, a coupling session over ZeroMQ, the coordinator converting units, copying and averaging between scales, its record read back), by hand --- doc/tutorials.md | 6 + .../Project.toml | 12 + .../03_metafspm_and_virtualplantlab/README.md | 32 ++ .../03_metafspm_and_virtualplantlab/carbon.jl | 69 ++++ .../coupling.ipynb | 319 ++++++++++++++++++ .../03_metafspm_and_virtualplantlab/run.sh | 6 + tutorials/README.md | 1 + 7 files changed, 445 insertions(+) create mode 100644 tutorials/03_metafspm_and_virtualplantlab/Project.toml create mode 100644 tutorials/03_metafspm_and_virtualplantlab/README.md create mode 100644 tutorials/03_metafspm_and_virtualplantlab/carbon.jl create mode 100644 tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb create mode 100755 tutorials/03_metafspm_and_virtualplantlab/run.sh diff --git a/doc/tutorials.md b/doc/tutorials.md index 1953451..8974fbf 100644 --- a/doc/tutorials.md +++ b/doc/tutorials.md @@ -36,3 +36,9 @@ end. This page gathers their explanations. ```{include} ../tutorials/02d_groimp/README.md :start-line: 2 ``` + +## Tutorial 3: metafspm and VirtualPlantLab, one plant + +```{include} ../tutorials/03_metafspm_and_virtualplantlab/README.md +:start-line: 2 +``` diff --git a/tutorials/03_metafspm_and_virtualplantlab/Project.toml b/tutorials/03_metafspm_and_virtualplantlab/Project.toml new file mode 100644 index 0000000..a8460d2 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/Project.toml @@ -0,0 +1,12 @@ +[deps] +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[sources] +encoding3d = {path = "../../bindings/julia/encoding3d"} + +[compat] +PlantGraphs = "1" +ZMQ = "1" +julia = "1.12" diff --git a/tutorials/03_metafspm_and_virtualplantlab/README.md b/tutorials/03_metafspm_and_virtualplantlab/README.md new file mode 100644 index 0000000..176275c --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/README.md @@ -0,0 +1,32 @@ +# Tutorial 3: metafspm and VirtualPlantLab, one plant + +**What you learn:** how two platforms, each in its own language and process, work on the same plant and exchange +their values at each step through encoding3d: a coupling session. + +- **metafspm** (Python) grows the roots: each tip elongates as fast as its sugar allows (structural: it creates the + segments). +- **VirtualPlantLab** (Julia, `carbon.jl`) is the carbon model: it splits each segment into two tissue layers, grown + by a PlantGraphs rule, and computes their sugar from the segment's mass (functional, adding entities of its own at + a finer scale). +- encoding3d's **coordinator** runs the session: it keeps the shared plant, passes each model the values it needs + (converting grams to milligrams, copying a segment's mass to its layers, averaging the layers' sugar back to their + segment), and tells each model when to run. + +Both models are toys: the tutorial shows the coupling, not plant science. + +**What you need** (not encoding3d's requirements): metafspm's environment, as in tutorial 2a, and Julia 1.12 or later +for VirtualPlantLab's side (this folder's `Project.toml`: encoding3d.jl from this repository, PlantGraphs, ZMQ). + +**How to run it:** `./run.sh` (`JULIA=/path/to/julia ./run.sh` if Julia is not on the path). It prepares the Julia +project, then runs `coupling.ipynb`, which starts the coordinator, launches `carbon.jl` and runs metafspm's scene. +Its files are in `outputs/`: the coordinator's record of the shared plant (`record/`) and metafspm's own frames +(`metafspm/`). The notebook explains: + +1. metafspm's root growth, with its sugar as an input; +2. VirtualPlantLab's model seen from metafspm: a `RemoteComponent`, computing nothing, whose state variables are the + other model's outputs, with metafspm's rights on the plant's scales and the links between their names; +3. the session: the coordinator, VirtualPlantLab's model launched, metafspm's scene run; +4. what happened, from the coordinator's record. + +Open `outputs/metafspm/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1) and colour the roots +by `C_hexose_root`. diff --git a/tutorials/03_metafspm_and_virtualplantlab/carbon.jl b/tutorials/03_metafspm_and_virtualplantlab/carbon.jl new file mode 100644 index 0000000..72da792 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/carbon.jl @@ -0,0 +1,69 @@ +# Tutorial 3, VirtualPlantLab's side: a carbon model (a toy) as a participant of an encoding3d coupling session. +# Launched by the coordinator (the notebook), which gives it the session's address. It splits each of metafspm's root +# segments into two layers, grown by a PlantGraphs rule, and gives each layer a sugar (hexose) concentration from its +# segment's structural mass in mg (metafspm sends grams: the coordinator converts): (rank + 1) / (1 + mass), computed +# on the graph by a query. +using encoding3d +using PlantGraphs +using ZMQ # the ZeroMQ transport (a folder works without it) + +struct Tissue <: Node end +struct Bud <: Node end +mutable struct Layer <: Node + segment::Int64 # its segment's id in the session + rank::Int # 0: the outer layer, 1: the inner one + hexose::Float64 +end +const NEW_SEGMENTS = Int64[] # the segments to split at the next rewriting + +# the rule: the bud adds two layers for each new segment, and stays to add more later +split_segments = Rule(Bud, rhs = bud -> begin + isempty(NEW_SEGMENTS) ? Bud() : Bud() + Tuple(Layer(segment, 0, 0.0) + Layer(segment, 1, 0.0) for segment in NEW_SEGMENTS) +end) +tissue = Graph(axiom = Tissue() + Bud(), rules = split_segments) + +# what it does in the session: it follows metafspm's segments, adds its own layers under them, needs their structural +# mass, gives their hexose concentration +declaration = Declaration("carbon"; platform = "VirtualPlantLab", + entity_sets = [EntitySetSpec("plant", "SubOrgan"), EntitySetSpec("plant", "Layer"; rights = "refines")], + variables = [VariableSpec("plant", "Layer", "struct_mass", "requires"; unit = "mg", kind = "extensive"), + VariableSpec("plant", "Layer", "hexose", "provides"; unit = "mol.g-1", kind = "massic_concentration")]) +participant = Participant(declaration; timeout = 600) +offset = participant.id_offset # the ids of the layers it creates start there +split = Set{Int64}() + +layers_of(tissue) = entity_set(from_graph(tissue; id = "plant", entity_set = "Layer", rank = 7, + keep = node -> node isa Layer), "Layer") + +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + # each segment's structural mass (mg), from its layers (the coordinator copies a segment's mass to its layers) + mass = Dict{Int64,Float64}() + if any(set -> set.name == "Layer", plant.entity_sets) + received, mine = entity_set(plant, "Layer"), layers_of(tissue) + segment_of = Dict(id + offset => segment for (id, segment) in zip(mine.columns["id"], mine.columns["segment"])) + for (layer, value) in zip(received.columns["id"], received.columns["struct_mass"]) + isnan(value) || (mass[segment_of[layer]] = value) + end + end + # the new segments split by the rule, then every layer's hexose by a query + empty!(NEW_SEGMENTS) + append!(NEW_SEGMENTS, [segment for segment in entity_set(plant, "SubOrgan").columns["id"] if !(segment in split)]) + union!(split, NEW_SEGMENTS) + rewrite!(tissue) + for layer in apply(tissue, Query(Layer)) + layer.hexose = (layer.rank + 1) / (1 + get(mass, layer.segment, 0.0)) + end + # its layers sent: ids in its range, under their segment + layers = layers_of(tissue) + columns = encoding3d.Columns( + "id" => layers.columns["id"] .+ offset, + "complex" => Vector{Int64}(layers.columns["segment"]), + "parent" => [parent == NO_ID ? NO_ID : parent + offset for parent in layers.columns["parent"]], + "edge_type" => layers.columns["edge_type"], + "label" => layers.columns["label"], + "hexose" => Vector{Float64}(layers.columns["hexose"])) + send_update(participant, Frame(data_structures = [DataStructure("plant"; + entity_sets = [EntitySet("Layer"; columns = columns, rank = 7)])])) +end +println("carbon: the session ended (", participant.stopped, ")") diff --git a/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb b/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb new file mode 100644 index 0000000..335614c --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/coupling.ipynb @@ -0,0 +1,319 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# Tutorial 3: metafspm and VirtualPlantLab, one plant\n", + "\n", + "Two platforms work on the same root system, each in its own language and process, exchanging their values at each\n", + "step through encoding3d:\n", + "\n", + "- **metafspm** (Python) grows the roots: each segment's tip elongates as fast as its sugar allows (a **structural**\n", + " model: it creates the segments);\n", + "- **VirtualPlantLab** (Julia) is the carbon model: it splits each segment into two tissue layers, grown by a\n", + " PlantGraphs rule, and computes their sugar from the segment's mass (a **functional** model, which also adds\n", + " entities of its own, at a finer scale).\n", + "\n", + "Both models are toys, written for the tutorial: they show the coupling, not plant science.\n", + "\n", + "A **coordinator** (encoding3d's, in Python) runs the session: it keeps the shared plant, passes each model the values\n", + "it needs (converting units, copying a segment's mass to its layers, averaging the layers' sugar back to the segment),\n", + "and tells each model when to run. This notebook starts the coordinator, launches VirtualPlantLab's model\n", + "(`carbon.jl`), and runs metafspm's." + ] + }, + { + "cell_type": "code", + "id": "cell-1", + "metadata": {}, + "source": [ + "import os\n", + "import shutil\n", + "import threading\n", + "from dataclasses import dataclass\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from openalea import encoding3d\n", + "from openalea.encoding3d.coupling import Coordinator, Link\n", + "from openalea.metafspm.coupling.component import StructuralComponent, input_variable, parameter, state_variable\n", + "from openalea.metafspm.data_structure.configs import PropsConfig, ScalesConfig as scales\n", + "from openalea.metafspm.io.coupling import RemoteComponent\n", + "from openalea.metafspm.scene.scene import Scene, SceneRecorder\n", + "from openalea.metafspm.solve.decorator import actual, segmentation\n", + "\n", + "OUTPUTS = \"outputs\"\n", + "JULIA = os.environ.get(\"JULIA\", \"julia\") # the Julia that runs VirtualPlantLab's model\n", + "STEPS = 8" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## 1. metafspm's root growth\n", + "\n", + "As in tutorial 2a, a component grows one root axis, but its tip now elongates in proportion to its sugar,\n", + "`C_hexose_root`, an **input**: metafspm does not compute it, VirtualPlantLab will. Each segment's structural mass is\n", + "proportional to its length." + ] + }, + { + "cell_type": "code", + "id": "cell-3", + "metadata": {}, + "source": [ + "SEGMENT = 0.01 # m\n", + "DENSITY = 0.1 # g of structure per m of root\n", + "UNIT = dict(unit_comment=\"\", value_comment=\"\", references=\"\", DOI=[], min_value=None, max_value=None)\n", + "\n", + "\n", + "def segment_variable(unit, description, default=0., kind=\"descriptor\"):\n", + " return state_variable(unit=unit, description=description, default=default, scale=scales.SubOrgan, kind=kind, **UNIT)\n", + "\n", + "\n", + "@dataclass\n", + "class SugarLimitedRoot(StructuralComponent):\n", + " \"\"\"A root axis whose tip elongates with its sugar, cut every 1 cm (a toy).\"\"\"\n", + " C_hexose_root: float = input_variable(unit=\"mol.g-1\", description=\"sugar concentration\", default=1.,\n", + " scale=scales.SubOrgan, kind=\"massic_concentration\", **UNIT)\n", + " length: float = segment_variable(\"m\", \"the segment's length\", kind=\"extensive\")\n", + " struct_mass: float = segment_variable(\"g\", \"the segment's structural mass\", kind=\"extensive\")\n", + " radius: float = segment_variable(\"m\", \"the segment's radius\", default=0.0005, kind=\"intensive\")\n", + " is_apex: float = segment_variable(\"adim\", \"1 for the tip\")\n", + " x1: float = segment_variable(\"m\", \"proximal end\")\n", + " y1: float = segment_variable(\"m\", \"proximal end\")\n", + " z1: float = segment_variable(\"m\", \"proximal end\")\n", + " x2: float = segment_variable(\"m\", \"distal end\")\n", + " y2: float = segment_variable(\"m\", \"distal end\")\n", + " z2: float = segment_variable(\"m\", \"distal end\")\n", + " elongation_rate: float = parameter(unit=\"m.s-1.g.mol-1\", description=\"elongation per unit of sugar\",\n", + " default=2e-7, **UNIT)\n", + "\n", + " @classmethod\n", + " def initiate_plant(cls, g, plant, scenario):\n", + " s = g.scales\n", + " parent = plant\n", + " for scale, label in ((s.Axis, g.labels.Axis.Root), (s.GrowthUnit, g.labels.GrowthUnit.Root),\n", + " (s.Phytomer, g.labels.Phytomer.Root), (s.Organ, g.labels.Organ.RootInternode)):\n", + " parent = g.add_component(parent, **PropsConfig(scale=scale, edge_type=\"/\", label=label))\n", + " tip = g.add_component(parent, **PropsConfig(scale=s.SubOrgan, edge_type=\"/\", label=g.labels.SubOrgan.RootSegment))\n", + " x, y, z = (g.property(name)[plant] for name in (\"x\", \"y\", \"z\"))\n", + " for name, value in ((\"length\", 0.005), (\"struct_mass\", DENSITY * 0.005), (\"radius\", 0.0005), (\"is_apex\", 1.),\n", + " (\"x1\", x), (\"y1\", y), (\"z1\", z), (\"x2\", x), (\"y2\", y), (\"z2\", z - 0.005)):\n", + " g.property(name)[tip] = value\n", + "\n", + " def _tips(self):\n", + " return [v for v, flag in self.mtg.property(\"is_apex\").items() if flag]\n", + "\n", + " @actual\n", + " def _elongation(self):\n", + " g = self.mtg\n", + " length, mass, z2, sugar = (g.property(name) for name in (\"length\", \"struct_mass\", \"z2\", \"C_hexose_root\"))\n", + " rate = dict(zip(self.data_structure.entity_ids(\"node\").tolist(),\n", + " self.parameter_values(\"elongation_rate\").tolist()))\n", + " for tip in self._tips():\n", + " growth = rate[tip] * sugar[tip] * self.dt\n", + " length[tip] += growth\n", + " mass[tip] = DENSITY * length[tip]\n", + " z2[tip] -= growth\n", + "\n", + " @segmentation\n", + " def _segmentation(self):\n", + " g = self.mtg\n", + " props = {name: g.property(name) for name in (\"length\", \"struct_mass\", \"is_apex\", \"radius\", \"C_hexose_root\",\n", + " \"x1\", \"y1\", \"z1\", \"x2\", \"y2\", \"z2\")}\n", + " for tip in self._tips():\n", + " if props[\"length\"][tip] <= SEGMENT:\n", + " continue\n", + " rest = props[\"length\"][tip] - SEGMENT\n", + " props[\"length\"][tip], props[\"struct_mass\"][tip], props[\"is_apex\"][tip] = SEGMENT, DENSITY * SEGMENT, 0.\n", + " props[\"z2\"][tip] = props[\"z1\"][tip] - SEGMENT\n", + " x, y, z = props[\"x2\"][tip], props[\"y2\"][tip], props[\"z2\"][tip]\n", + " g.add_child(tip, **PropsConfig(scale=g.scales.SubOrgan, edge_type=\"<\", label=g.labels.SubOrgan.RootSegment,\n", + " length=rest, struct_mass=DENSITY * rest, is_apex=1.,\n", + " radius=props[\"radius\"][tip], C_hexose_root=props[\"C_hexose_root\"][tip],\n", + " x1=x, y1=y, z1=z, x2=x, y2=y, z2=z - rest))" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## 2. VirtualPlantLab's model, seen from metafspm\n", + "\n", + "Inside metafspm, the other platform's model is a **RemoteComponent**: a component that computes nothing. Its state\n", + "variables are what the remote model gives (`C_hexose_root`), its inputs what metafspm sends it (`struct_mass`). It\n", + "also says what metafspm does with the plant's scales: metafspm **owns** the plant down to the segments (it creates\n", + "them); it **follows** the layers VirtualPlantLab adds (they are created in metafspm's graph too). And it **links**\n", + "metafspm's names to the other model's: the segment's sugar is the mean of its layers' `hexose`.\n", + "\n", + "Calling it at the end of each step sends metafspm's outputs and brings back the next values." + ] + }, + { + "cell_type": "code", + "id": "cell-5", + "metadata": {}, + "source": [ + "@dataclass\n", + "class VirtualPlantLabCarbon(RemoteComponent):\n", + " \"\"\"VirtualPlantLab's carbon model (carbon.jl), as a metafspm component.\"\"\"\n", + " participant = \"metafspm\"\n", + " rights = {\"Plant\": \"owns\", \"Axis\": \"owns\", \"GrowthUnit\": \"owns\", \"Phytomer\": \"owns\", \"Organ\": \"owns\",\n", + " \"SubOrgan\": \"owns\", \"Layer\": \"follows\"}\n", + " links = [Link(\"metafspm\", \"C_hexose_root\", \"carbon\", \"hexose\", aggregation=\"mean\")]\n", + " C_hexose_root: float = state_variable(unit=\"mol.g-1\", description=\"sugar concentration\", default=1.,\n", + " scale=scales.SubOrgan, kind=\"massic_concentration\", **UNIT)\n", + " struct_mass: float = input_variable(unit=\"g\", description=\"structural mass\", default=0., scale=scales.SubOrgan,\n", + " kind=\"extensive\", **UNIT)\n", + "\n", + "\n", + "class CoupledRoot:\n", + " \"\"\"The plant model: metafspm's growth, then the exchange with VirtualPlantLab.\"\"\"\n", + " initiators = (SugarLimitedRoot,)\n", + "\n", + " def __init__(self, data_structure, time_step):\n", + " self.growth = SugarLimitedRoot(data_structure=data_structure)\n", + " self.carbon = VirtualPlantLabCarbon(data_structure=data_structure)\n", + " self.components = [self.growth, self.carbon]\n", + " self.carbon.start() # joins the session: the initial state exchanged\n", + "\n", + " def run(self):\n", + " self.growth()\n", + " self.carbon()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-6", + "metadata": {}, + "source": [ + "## 3. The session\n", + "\n", + "The coordinator gets the session's settings: how many steps, how long each is, the links it applies itself (the\n", + "layers need their segment's structural mass: `copy` it down), and a folder where it **records** the whole shared\n", + "plant at each step, as an encoding3d sequence. Then it launches VirtualPlantLab's model, which joins, and waits for\n", + "metafspm, which joins when its scene is built.\n", + "\n", + "The participants talk over ZeroMQ (in memory). `Coordinator(\"files:outputs/messages\", ...)` would make them\n", + "exchange files in a folder instead, with no other change." + ] + }, + { + "cell_type": "code", + "id": "cell-7", + "metadata": {}, + "source": [ + "for folder in (\"record\", \"metafspm\"):\n", + " shutil.rmtree(os.path.join(OUTPUTS, folder), ignore_errors=True)\n", + "coordinator = Coordinator(\"tcp://127.0.0.1:*\", steps=STEPS, interval=6., time_unit=\"h\", expect=[\"metafspm\"],\n", + " links=[Link(\"carbon\", \"struct_mass\", \"metafspm\", \"struct_mass\", aggregation=\"copy\")],\n", + " record=os.path.join(OUTPUTS, \"record\"), timeout=600)\n", + "coordinator.launch(\"carbon\", [JULIA, \"--project=.\", \"carbon.jl\"])\n", + "session = threading.Thread(target=coordinator.run)\n", + "session.start()\n", + "VirtualPlantLabCarbon.endpoint = coordinator.endpoint\n", + "print(\"coordinator at\", coordinator.endpoint)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-8", + "metadata": {}, + "source": [ + "metafspm's scene: one plant, six-hour steps, its own frames recorded too. Each `scene.run()` is one step of the\n", + "session." + ] + }, + { + "cell_type": "code", + "id": "cell-9", + "metadata": {}, + "source": [ + "planting = pd.DataFrame([dict(plant=\"root\", model=CoupledRoot, x=0., y=0., z=0., rotation=0., scenario={})])\n", + "planting.attrs.update(xrange=0.1, yrange=0.1)\n", + "scene = Scene(planting=planting, time_step=6 * 3600.,\n", + " recorder=SceneRecorder(os.path.join(OUTPUTS, \"metafspm\"), summaries=False, frame_period=1))\n", + "for _ in range(STEPS):\n", + " scene.run()\n", + "session.join() # the coordinator stops the session after the last step\n", + "print(\"the session has ended\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-10", + "metadata": {}, + "source": [ + "## 4. What happened\n", + "\n", + "The coordinator's record holds the shared plant at each step: metafspm's segments with their mass, and\n", + "VirtualPlantLab's layers with their sugar." + ] + }, + { + "cell_type": "code", + "id": "cell-11", + "metadata": {}, + "source": [ + "rows = []\n", + "for frame in encoding3d.read_sequence(os.path.join(OUTPUTS, \"record\", \"scene.fspm.json\")):\n", + " plant = frame.data_structure(\"plant\")\n", + " segments, layers = plant.entity_set(\"SubOrgan\"), plant.entity_set(\"Layer\")\n", + " rows.append(dict(time_h=frame.time, segments=segments.count, layers=layers.count,\n", + " total_mass_mg=1000 * np.nansum(segments.columns[\"struct_mass\"]),\n", + " mean_hexose=np.nanmean(layers.columns[\"hexose\"]) if layers.count else np.nan))\n", + "pd.DataFrame(rows)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-12", + "metadata": {}, + "source": [ + "The sugar of a layer falls as its segment gets heavier, so the roots grow more slowly; new segments start with the\n", + "most sugar (their layers are new: their mass reaches VirtualPlantLab one step later, the coupling being explicit).\n", + "metafspm counts the mass in grams and VirtualPlantLab in milligrams: the coordinator converts, as it declares. metafspm's own frames\n", + "(`outputs/metafspm/frames`) hold the roots with their geometry and sugar: open them in ParaView with encoding3d's\n", + "plugin and colour the roots by `C_hexose_root`.\n", + "\n", + "## Try it yourself\n", + "\n", + "- Exchange through a folder: `Coordinator(\"files:outputs/messages\", ...)`. The messages are deleted once read; add\n", + " `keep_messages=True` to keep them and look at them.\n", + "- Change VirtualPlantLab's sugar rule in `carbon.jl`, and see metafspm's roots grow faster or slower.\n", + "- Run two plants in the scene: VirtualPlantLab's model splits the segments of both." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/03_metafspm_and_virtualplantlab/run.sh b/tutorials/03_metafspm_and_virtualplantlab/run.sh new file mode 100755 index 0000000..f248391 --- /dev/null +++ b/tutorials/03_metafspm_and_virtualplantlab/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# Runs tutorial 3 (README.md): metafspm's environment, and Julia 1.12 (JULIA, default julia) for VirtualPlantLab's side. +set -euo pipefail +cd "$(dirname "$0")" +"${JULIA:-julia}" --project=. -e 'using Pkg; Pkg.instantiate(); using encoding3d, PlantGraphs, ZMQ' +python -m pytest --nbmake coupling.ipynb diff --git a/tutorials/README.md b/tutorials/README.md index 8bc7e98..4ba8f74 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -10,6 +10,7 @@ its explanation (`README.md`), its code, and `run.sh`, which runs it from start | `02b_virtualplantlab` | a VirtualPlantLab plant grown by rules, written with its geometry, rebuilt and grown on; an MTG to and from encoding3d | Julia 1.12, VirtualPlantLab's packages, MTG.jl | | `02c_cplantbox` | a CPlantBox plant written day by day with its geometry, soil grid and mapping; frames back as CPlantBox root systems | C++17, CMake, CPlantBox built | | `02d_groimp` | a GroIMP graph written step by step and rebuilt as GroIMP nodes; GroIMP's plugin exporting a project with its geometry | Java 21, Maven (GroIMP's Graph module) | +| `03_metafspm_and_virtualplantlab` | a coupling session: metafspm's root growth and VirtualPlantLab's carbon model on one plant, over ZeroMQ, recorded | metafspm's environment, Julia 1.12 | **encoding3d depends on no platform.** Tutorial 1 needs encoding3d only, and encoding3d's CI runs it. The tutorials using a platform (metafspm, VirtualPlantLab, CPlantBox, GroIMP) list what they need in their own folder, outside From 968e5705e9b3024914ad0306ca369268267d022e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Thu, 5 Nov 2026 12:49:07 +0100 Subject: [PATCH 38/39] Tutorial 4, four platforms on one plant: CPlantBox's roots and GroIMP's shoot (structural, owning their entity sets), metafspm's root water uptake (functional, its data structure built from CPlantBox's roots by RemoteComponent.join) and VirtualPlantLab's light and photosynthesis (functional, on a PlantGraphs graph rebuilt by to_graph); the coordinator linking them by standard name, name, a sum and a copy between scales, three stages, the plant recorded with its views; each side in its own process, run by hand --- doc/tutorials.md | 6 + tutorials/04_four_platforms/README.md | 58 ++++++++ .../cplantbox/CMakeLists.txt | 19 +++ .../04_four_platforms/cplantbox/roots.cpp | 90 ++++++++++++ tutorials/04_four_platforms/groimp/pom.xml | 44 ++++++ .../src/main/java/tutorial/Managed.java | 77 +++++++++++ .../groimp/src/main/java/tutorial/Shoot.java | 128 ++++++++++++++++++ .../04_four_platforms/metafspm/uptake.py | 62 +++++++++ tutorials/04_four_platforms/run.sh | 17 +++ tutorials/04_four_platforms/session.py | 44 ++++++ .../virtualplantlab/Project.toml | 12 ++ .../virtualplantlab/light.jl | 60 ++++++++ tutorials/README.md | 1 + 13 files changed, 618 insertions(+) create mode 100644 tutorials/04_four_platforms/README.md create mode 100644 tutorials/04_four_platforms/cplantbox/CMakeLists.txt create mode 100644 tutorials/04_four_platforms/cplantbox/roots.cpp create mode 100644 tutorials/04_four_platforms/groimp/pom.xml create mode 100644 tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java create mode 100644 tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java create mode 100644 tutorials/04_four_platforms/metafspm/uptake.py create mode 100755 tutorials/04_four_platforms/run.sh create mode 100644 tutorials/04_four_platforms/session.py create mode 100644 tutorials/04_four_platforms/virtualplantlab/Project.toml create mode 100644 tutorials/04_four_platforms/virtualplantlab/light.jl diff --git a/doc/tutorials.md b/doc/tutorials.md index 8974fbf..5f0e294 100644 --- a/doc/tutorials.md +++ b/doc/tutorials.md @@ -42,3 +42,9 @@ end. This page gathers their explanations. ```{include} ../tutorials/03_metafspm_and_virtualplantlab/README.md :start-line: 2 ``` + +## Tutorial 4: four platforms, one plant + +```{include} ../tutorials/04_four_platforms/README.md +:start-line: 2 +``` diff --git a/tutorials/04_four_platforms/README.md b/tutorials/04_four_platforms/README.md new file mode 100644 index 0000000..6987279 --- /dev/null +++ b/tutorials/04_four_platforms/README.md @@ -0,0 +1,58 @@ +# Tutorial 4: four platforms, one plant + +**What you learn:** how four platforms, in four languages, run one plant together through encoding3d, each with the +model it is good at, two building the plant's structure and two computing its functions. + +| Platform | Its model (a toy) | Kind | In the shared plant | It gives | It needs | +|---|---|---|---|---|---| +| **CPlantBox** (C++, `cplantbox/roots.cpp`) | the root system, grown by CPlantBox's own rules (Anagallis' parameters) | structural | **owns** the plant, its root organs and their segments | each segment's length, radius and ends (cm) | — | +| **GroIMP** (Java, `groimp/.../Shoot.java`) | the shoot, a GroIMP graph of phytomers: a new one each time the roots have taken up 8 cm³ of water | structural | **owns** the phytomers, under CPlantBox's plant | each leaf's area (m²) and height (m) | the plant's water uptake (cm³/d) | +| **metafspm** (Python, `metafspm/uptake.py`) | root water uptake: through each segment's surface, more when the plant has more carbon | functional | **follows** the plant, organs and segments | each segment's uptake (cm³/d) | each segment's length and radius; the plant's assimilation | +| **VirtualPlantLab** (Julia, `virtualplantlab/light.jl`) | light (Beer–Lambert down the canopy) and photosynthesis (a light-use efficiency), on a PlantGraphs graph | functional | **follows** the plant and the phytomers | each leaf's absorbed light and assimilation; the plant's total (µmol/s) | each leaf's area and height | + +The loop: the leaves' light → their assimilation → the roots' uptake → the water the shoot has → new leaves → more +light. The models are toys, labelled as such: the tutorial shows how platforms work together, not plant science. + +## How they fit together + +- **One plant, shared.** The session's data structure, `plant`, holds every platform's entities: CPlantBox's plant, + root organs and segments, GroIMP's phytomers under the same plant. A platform declares the entity sets it works + on, and what it may do with them: **own** them (create them), or **follow** them (read them). No platform needs to + know the others' code or language. +- **Links, made by encoding3d's coordinator** (`session.py`): + - metafspm's `length` and `radius` are found by their **standard name** among CPlantBox's outputs (the same unit + here, cm; a different one would be converted); + - VirtualPlantLab's `area` and `height` by their **name** among GroIMP's; + - GroIMP's `water_uptake` of the plant is the **sum** of metafspm's uptake over the plant's segments (they are in + the plant, through their organs); + - metafspm's `plant_assimilation` of each segment is a **copy** of the plant's, given by VirtualPlantLab. +- **Stages.** At each daily step, CPlantBox grows the roots first, then GroIMP the shoot (it needs the plant + CPlantBox created), then metafspm and VirtualPlantLab compute together. +- **What each platform does with what it receives:** CPlantBox's adapter writes its plant; GroIMP's writes its graph; + metafspm builds its own graph from CPlantBox's roots (`RemoteComponent.join`) and computes with a metafspm + component; VirtualPlantLab rebuilds the phytomers as a PlantGraphs graph (`to_graph`) and queries it. + +## What you need + +Not encoding3d's requirements: everything tutorials 2a to 2d need. CPlantBox built (tutorial 2c, `CPLANTBOX` and +`CPLANTBOX_LIBRARY`), Java 21 and Maven (tutorial 2d), Julia 1.12 or later (`JULIA` if not on the path), and +metafspm's environment active (tutorial 2a), whose Python runs the coordinator and metafspm's side. + +## How to run it + +```bash +CPLANTBOX=/path/to/CPlantBox CPLANTBOX_LIBRARY=/path/to/CPlantBox-build/src/libCPlantBox.a ./run.sh +``` + +It builds CPlantBox's and GroIMP's sides, prepares the Julia project, then runs `session.py`, which launches the four +models, each in its own process, for ten days. Each prints what it does at each step, and the coordinator prints how +it linked them. The whole plant is recorded at each step in `outputs/record/`, with a `views.json` colouring the +roots by uptake: open `outputs/record/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1) and press +*Play*. + +## Try it yourself + +- Make phytomers cheaper (`WATER_PER_PHYTOMER` in `Shoot.java`) and watch the canopy close: assimilation saturates. +- Give metafspm's uptake a soil water potential, from a fifth participant owning a soil grid, linked to the roots + by a mapping (the coupling documentation, "Coupling platforms"). +- Replace one platform by the record of an earlier run: `encoding3d couple --replay` (the coupling documentation). diff --git a/tutorials/04_four_platforms/cplantbox/CMakeLists.txt b/tutorials/04_four_platforms/cplantbox/CMakeLists.txt new file mode 100644 index 0000000..f485b5a --- /dev/null +++ b/tutorials/04_four_platforms/cplantbox/CMakeLists.txt @@ -0,0 +1,19 @@ +# Tutorial 4, CPlantBox's side: the root architecture as a coupling session's participant (../README.md). +# +# cmake -S . -B build -DCPLANTBOX_SOURCE_DIR=/path/to/CPlantBox -DCPLANTBOX_LIBRARY=/path/to/libCPlantBox.a +cmake_minimum_required(VERSION 3.16) +project(encoding3d_tutorial_four_platforms_cplantbox LANGUAGES CXX) + +set(CPLANTBOX_SOURCE_DIR "" CACHE PATH "CPlantBox's source folder") +set(CPLANTBOX_LIBRARY "" CACHE FILEPATH "CPlantBox's built library (libCPlantBox.a)") +if(NOT CPLANTBOX_SOURCE_DIR OR NOT CPLANTBOX_LIBRARY) + message(FATAL_ERROR "give CPlantBox: -DCPLANTBOX_SOURCE_DIR=... -DCPLANTBOX_LIBRARY=... (../README.md)") +endif() +set(ENCODING3D_CPLANTBOX_SOURCE_DIR "${CPLANTBOX_SOURCE_DIR}" CACHE PATH "" FORCE) +set(ENCODING3D_CPLANTBOX_LIBRARY "${CPLANTBOX_LIBRARY}" CACHE FILEPATH "" FORCE) +set(ENCODING3D_BUILD_TESTS OFF CACHE BOOL "" FORCE) +add_subdirectory(../../../bindings/cpp encoding3d) + +add_executable(roots roots.cpp) +target_link_libraries(roots PRIVATE encoding3d::cplantbox) +target_compile_definitions(roots PRIVATE CPLANTBOX_PARAMETERS="${CPLANTBOX_SOURCE_DIR}/modelparameter/structural") diff --git a/tutorials/04_four_platforms/cplantbox/roots.cpp b/tutorials/04_four_platforms/cplantbox/roots.cpp new file mode 100644 index 0000000..959a502 --- /dev/null +++ b/tutorials/04_four_platforms/cplantbox/roots.cpp @@ -0,0 +1,90 @@ +// Tutorial 4, CPlantBox's side: the root architecture (structural). CPlantBox grows a root system by its own rules; +// each turn, the plant, its root organs and their segments are sent to the session (CPlantBox's adapter writes them, +// their ids renumbered in this participant's range), with each segment's length, radius and ends. +#include +#include +#include + +#include "encoding3d/coupling.hpp" +#include "encoding3d/cplantbox.hpp" +#include "encoding3d/encoding3d.hpp" + +#include "Plant.h" + +using namespace encoding3d; + +namespace { + +/// The declaration: CPlantBox owns the plant, its organs and segments, and provides the segments' sizes and ends. +json declaration() { + json variables = json::array(); + auto provides = [&](const char *name, const char *unit, const char *standard_name, const char *kind) { + variables.push_back({{"dataStructure", "plant"}, {"entitySet", "SubOrgan"}, {"name", name}, + {"direction", "provides"}, {"unit", unit}, {"standardName", standard_name}, + {"kind", kind}}); + }; + provides("length", "cm", "length", "extensive"); + provides("radius", "cm", "radius", "intensive"); + for (const auto &[name, standard_name] : {std::pair{"x1", "proximal_end_x"}, {"y1", "proximal_end_y"}, + {"z1", "proximal_end_z"}, {"x2", "distal_end_x"}, + {"y2", "distal_end_y"}, {"z2", "distal_end_z"}}) + provides(name, "cm", standard_name, "descriptor"); + json sets = json::array(); + for (const auto &[name, rank] : {std::pair{"Plant", 1}, {"Organ", 5}, {"SubOrgan", 6}}) + sets.push_back({{"dataStructure", "plant"}, {"name", name}, {"rank", rank}, {"rights", "owns"}}); + return {{"participant", "cplantbox"}, {"platform", "CPlantBox"}, {"entitySets", sets}, {"variables", variables}}; +} + +/// The session's id of an adapter's id: this participant's range, in order of appearance (stable while it grows). +std::int64_t session_id(std::map &ids, std::int64_t offset, std::int64_t adapter) { + if (adapter == NO_ID) return NO_ID; + auto found = ids.find(adapter); + if (found == ids.end()) found = ids.emplace(adapter, offset + std::int64_t(ids.size())).first; + return found->second; +} + +/// An entity set of the adapter's as the session's: its ids renumbered, its topology, the variables in *keep*. +EntitySet renumbered(const EntitySet &set, std::map &ids, std::int64_t offset, + const std::vector &keep) { + EntitySet out(set.name, set.rank); + for (const char *name : {"id", "complex", "parent"}) { + if (!set.has_column(name)) continue; + std::vector values; + for (const auto value : set.column(name).as()) values.push_back(session_id(ids, offset, value)); + out.set(name, Column(values)); + } + for (const char *name : {"label", "edge_type"}) + if (set.has_column(name)) out.set(name, set.column(name)); + for (const auto &name : keep) + if (set.has_column(name)) out.set(name, set.column(name)); + return out; +} + +} // namespace + +int main() { + auto plant = std::make_shared(1); + plant->readParameters(std::string(CPLANTBOX_PARAMETERS) + "/plant/Anagallis_femina_Leitner_2010.xml", "plant", + true, false); + plant->initialize(false); + plant->simulate(3., false); // a young root system to start from + + coupling::Participant participant(declaration(), "", json::array(), 600.0); + std::map ids; + while (auto turn = participant.next_turn()) { + if (turn->phase == "step") plant->simulate(turn->until - turn->time, false); // days, the session's unit + const auto written = cplantbox::to_data_structure(*plant); + DataStructure update("plant"); + update.entity_sets.push_back(renumbered(written.entity_set("Plant"), ids, participant.id_offset, {})); + update.entity_sets.push_back(renumbered(written.entity_set("Organ"), ids, participant.id_offset, {})); + update.entity_sets.push_back(renumbered(written.entity_set("SubOrgan"), ids, participant.id_offset, + {"length", "radius", "x1", "y1", "z1", "x2", "y2", "z2"})); + Frame frame; + frame.data_structures.push_back(update); + participant.send_update(frame); + std::cout << "cplantbox: day " << plant->getSimTime() << ", " << plant->getNumberOfSegments() << " segments" + << std::endl; + } + std::cout << "cplantbox: the session ended (" << participant.stopped.value_or("") << ")" << std::endl; + return 0; +} diff --git a/tutorials/04_four_platforms/groimp/pom.xml b/tutorials/04_four_platforms/groimp/pom.xml new file mode 100644 index 0000000..6632b7c --- /dev/null +++ b/tutorials/04_four_platforms/groimp/pom.xml @@ -0,0 +1,44 @@ + + + + 4.0.0 + org.openalea.tutorials + encoding3d-tutorial-shoot + 0.1.0 + + + 21 + UTF-8 + + + + + groimp + https://gitlab.com/api/v4/groups/55048746/-/packages/maven + + + + + + org.openalea + encoding3d-groimp + 0.1.0 + + + de.grogra + graph + 3.0.0 + + + org.zeromq + jeromq + 0.6.0 + + + diff --git a/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java new file mode 100644 index 0000000..259fca7 --- /dev/null +++ b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Managed.java @@ -0,0 +1,77 @@ +package tutorial; + +import java.lang.reflect.Field; + +import de.grogra.graph.impl.Node; +import de.grogra.graph.impl.Node.NType; +import de.grogra.reflect.Type; + +/** + * GroIMP's "managed fields": the fields of a node class GroIMP knows by name, as its XL compiler declares them for the + * classes of an RGG model. GroIMP's adapter writes them as columns. Here they are declared from plain Java fields by + * reflection, so that the tutorial's node classes stay short. + */ +final class Managed { + private Managed() {} + + /** The NType of {@code type}, with its fields {@code names} managed. */ + static NType type(Class type, String... names) { + NType nodeType = new NType(type); + for (String name : names) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + nodeType.addManagedField(new ManagedField(nodeType, field)); + } catch (NoSuchFieldException error) { + throw new IllegalArgumentException(type.getSimpleName() + " has no field " + name, error); + } + } + nodeType.validate(); + return nodeType; + } + + private static final class ManagedField extends NType.Field { + private final Field field; + + ManagedField(NType type, Field field) { + super(type, field.getName(), PUBLIC | SCO, typeOf(field.getType()), null); + this.field = field; + } + + private static Type typeOf(Class type) { + if (type == double.class) return Type.DOUBLE; + if (type == float.class) return Type.FLOAT; + if (type == int.class) return Type.INT; + if (type == boolean.class) return Type.BOOLEAN; + if (type == String.class) return Type.STRING; + throw new IllegalArgumentException("a managed field of type " + type); + } + + private Object read(Object node) { + try { + return field.get(node); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + private void write(Object node, Object value) { + try { + field.set(node, value); + } catch (IllegalAccessException error) { + throw new IllegalStateException(error); + } + } + + @Override public double getDouble(Object node) { return (double) read(node); } + @Override public void setDouble(Object node, double value) { write(node, value); } + @Override public float getFloat(Object node) { return (float) read(node); } + @Override public void setFloat(Object node, float value) { write(node, value); } + @Override public int getInt(Object node) { return (int) read(node); } + @Override public void setInt(Object node, int value) { write(node, value); } + @Override public boolean getBoolean(Object node) { return (boolean) read(node); } + @Override public void setBoolean(Object node, boolean value) { write(node, value); } + @Override public Object getObject(Object node) { return read(node); } + @Override protected void setObjectImpl(Object node, Object value) { write(node, value); } + } +} diff --git a/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java new file mode 100644 index 0000000..4d9a1e2 --- /dev/null +++ b/tutorials/04_four_platforms/groimp/src/main/java/tutorial/Shoot.java @@ -0,0 +1,128 @@ +package tutorial; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.openalea.encoding3d.Column; +import org.openalea.encoding3d.DataStructure; +import org.openalea.encoding3d.EntitySet; +import org.openalea.encoding3d.Frame; +import org.openalea.encoding3d.Variable; +import org.openalea.encoding3d.coupling.Participant; +import org.openalea.encoding3d.coupling.Turn; +import org.openalea.encoding3d.groimp.GroImp; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import de.grogra.graph.Graph; +import de.grogra.graph.impl.GraphManager; +import de.grogra.graph.impl.Node; +import de.grogra.persistence.ServerConnection; + +/** + * Tutorial 4, GroIMP's side: the shoot architecture (structural). The shoot is a GroIMP graph of phytomers (an + * internode and its leaf), one on top of the other; a new one grows each time the plant's roots have taken up enough + * water since the last, and every leaf expands. Each turn, GroIMP's adapter writes the graph, and its phytomers are + * sent to the session under the plant CPlantBox created. + */ +public final class Shoot { + private Shoot() {} + + static final double WATER_PER_PHYTOMER = 8.; // cm3 of water taken up for a new phytomer (a toy) + + /** A phytomer, as an RGG model declares it: its leaf's area and its height, managed by GroIMP. */ + public static final class Phytomer extends Node { + double area = 1e-4; // m2 + double height; // m + public static final NType $TYPE = Managed.type(Phytomer.class, "area", "height"); + @Override protected NType getNTypeImpl() { return $TYPE; } + @Override protected Node newInstance() { return new Phytomer(); } + } + + static ObjectNode declaration(ObjectMapper json) { + ObjectNode declaration = json.createObjectNode().put("participant", "groimp").put("platform", "GroIMP"); + ArrayNode sets = declaration.putArray("entitySets"); + sets.addObject().put("dataStructure", "plant").put("name", "Plant").put("rank", 1).put("rights", "follows"); + sets.addObject().put("dataStructure", "plant").put("name", "Phytomer").put("rank", 4).put("rights", "owns"); + ArrayNode variables = declaration.putArray("variables"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Phytomer").put("name", "area") + .put("direction", "provides").put("unit", "m2").put("kind", "extensive"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Phytomer").put("name", "height") + .put("direction", "provides").put("unit", "m").put("kind", "descriptor"); + variables.addObject().put("dataStructure", "plant").put("entitySet", "Plant").put("name", "water_uptake") + .put("direction", "requires").put("unit", "cm3/d").put("kind", "extensive"); + return declaration; + } + + static GroImp.Options options() { + GroImp.Options options = new GroImp.Options(); + options.id = "plant"; + options.scaleNames.put(2, "Phytomer"); + options.scaleRanks.put(2, 4L); + options.variables.put("area", new Variable("m2", null, "extensive", "state")); + options.variables.put("height", new Variable("m", null, "descriptor", "state")); + return options; + } + + public static void main(String[] arguments) throws Exception { + ObjectMapper json = new ObjectMapper(); + GraphManager graph = new GraphManager(new ServerConnection(null), "shoot", false, false); + Node root = new Node(); + Node plant = new Node(); // the plant node GroIMP's graph hangs the phytomers on + root.addEdgeBitsTo(plant, Graph.BRANCH_EDGE, null); + Phytomer top = new Phytomer(); + plant.addEdgeBitsTo(top, Graph.REFINEMENT_EDGE, null); + graph.setRoot(Graph.MAIN_GRAPH, root); + List phytomers = new ArrayList<>(List.of(top)); + Map ids = new HashMap<>(); // GroIMP node id: session id + double water = 0; + + try (Participant participant = new Participant(declaration(json), null, null, 600.0)) { + for (Turn turn; (turn = participant.nextTurn()) != null; ) { + EntitySet plants = turn.frame().dataStructure("plant").entitySet("Plant"); + long plantId = plants.column("id").longs()[0]; // CPlantBox's plant + if (turn.phase().equals("step")) { + double uptake = plants.column("water_uptake").doubles()[0]; + water += Double.isNaN(uptake) ? 0 : uptake * (turn.until() - turn.time()); + for (Phytomer phytomer : phytomers) phytomer.area *= 1.15; // the leaves expand + while (water >= WATER_PER_PHYTOMER) { // a new phytomer on top + water -= WATER_PER_PHYTOMER; + Phytomer next = new Phytomer(); + next.height = top.height + 0.03; + top.addEdgeBitsTo(next, Graph.SUCCESSOR_EDGE, null); + phytomers.add(next); + top = next; + } + } + // the graph written by GroIMP's adapter, its phytomers sent under the session's plant + DataStructure written = GroImp.toDataStructure(root, options()); + EntitySet set = written.entitySet("Phytomer"); + long[] nodes = set.column("id").longs(); + long[] session = new long[nodes.length], complexes = new long[nodes.length]; + long[] parents = set.column("parent").longs().clone(); + for (int row = 0; row < nodes.length; row++) { + final long node = nodes[row]; + session[row] = ids.computeIfAbsent(node, key -> participant.idOffset + ids.size()); + complexes[row] = plantId; + } + for (int row = 0; row < parents.length; row++) if (parents[row] >= 0) parents[row] = ids.get(parents[row]); + EntitySet out = new EntitySet("Phytomer", 4L); + out.set("id", Column.of(session)).set("complex", Column.of(complexes)).set("parent", Column.of(parents)) + .set("edge_type", set.column("edge_type")).set("label", set.column("label")) + .set("area", set.column("area")).set("height", set.column("height")); + DataStructure update = new DataStructure("plant"); + update.entitySets.add(out); + Frame frame = new Frame(); + frame.dataStructures.add(update); + participant.sendUpdate(frame); + System.out.println("groimp: " + phytomers.size() + " phytomers, water left " + Math.round(water * 1000) / 1000.0 + + " cm3"); + } + System.out.println("groimp: the session ended (" + participant.stopped + ")"); + } + } +} diff --git a/tutorials/04_four_platforms/metafspm/uptake.py b/tutorials/04_four_platforms/metafspm/uptake.py new file mode 100644 index 0000000..337c07a --- /dev/null +++ b/tutorials/04_four_platforms/metafspm/uptake.py @@ -0,0 +1,62 @@ +""" +Tutorial 4, metafspm's side: root water uptake (functional, a toy). metafspm has no plant of its own here: it joins +the session and builds its data structure from the roots CPlantBox grows (RemoteComponent.join), then its component +computes each segment's uptake from its surface and the plant's carbon status (the more the leaves assimilate, the more +the roots take up), at each step. +""" +from dataclasses import dataclass + +import numpy as np + +from openalea.metafspm.coupling.component import FunctionalComponent, input_variable, state_variable +from openalea.metafspm.data_structure.configs import ScalesConfig as scales +from openalea.metafspm.io.coupling import RemoteComponent +from openalea.metafspm.solve.decorator import actual + +UNIT = dict(unit_comment="", value_comment="", references="", DOI=[], min_value=None, max_value=None) +CONDUCTANCE = 0.5 # cm3 of water per cm2 of root surface per day, at full carbon status (a toy) +HALF_ASSIMILATION = 5. # umol/s of assimilation for half the full uptake + + +def segment(variable, unit, description, kind): + return variable(unit=unit, description=description, default=0., scale=scales.SubOrgan, kind=kind, **UNIT) + + +@dataclass +class OtherPlatforms(RemoteComponent): + """The other platforms' models, as seen from metafspm: CPlantBox's roots (followed: their segments' sizes) and + VirtualPlantLab's photosynthesis (the plant's assimilation, copied to each segment); metafspm's uptake sent.""" + participant = "metafspm" + rights = {"Plant": "follows", "Organ": "follows", "SubOrgan": "follows"} + length: float = segment(state_variable, "cm", "the segment's length (CPlantBox's)", "extensive") + radius: float = segment(state_variable, "cm", "the segment's radius (CPlantBox's)", "intensive") + plant_assimilation: float = segment(state_variable, "umol/s", "the plant's assimilation (VirtualPlantLab's)", + "extensive") + water_uptake: float = segment(input_variable, "cm3/d", "the segment's water uptake (metafspm's)", "extensive") + + +@dataclass +class RootWaterUptake(FunctionalComponent): + """Each segment takes up water through its surface, more when the plant has more carbon.""" + length: float = segment(input_variable, "cm", "the segment's length", "extensive") + radius: float = segment(input_variable, "cm", "the segment's radius", "intensive") + plant_assimilation: float = segment(input_variable, "umol/s", "the plant's assimilation", "extensive") + water_uptake: float = segment(state_variable, "cm3/d", "the segment's water uptake", "extensive") + + @actual + def _water_uptake(self, length, radius, plant_assimilation): + carbon_status = plant_assimilation / (plant_assimilation + HALF_ASSIMILATION) + return CONDUCTANCE * 2 * np.pi * radius * length * np.nan_to_num(carbon_status) + + +if __name__ == "__main__": + data_structure = OtherPlatforms.join() # the session's address from the coordinator, which launched it + uptake = RootWaterUptake(data_structure=data_structure) + others = OtherPlatforms(data_structure=data_structure) + others.start() + while others.stopped is None: + uptake() + print(f"metafspm: {data_structure.n_nodes()} segments, uptake {np.sum(data_structure.get('water_uptake')):.4f} " + "cm3/d", flush=True) + others() + print(f"metafspm: the session ended ({others.stopped})") diff --git a/tutorials/04_four_platforms/run.sh b/tutorials/04_four_platforms/run.sh new file mode 100755 index 0000000..8aba9ae --- /dev/null +++ b/tutorials/04_four_platforms/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Runs tutorial 4 (README.md): builds the CPlantBox and GroIMP participants, prepares the Julia project, then runs the +# session. Needs: CPLANTBOX and CPLANTBOX_LIBRARY (as tutorial 2c), Java 21 and Maven, Julia 1.12 (JULIA, default +# julia), and metafspm's environment active (its python runs the coordinator and metafspm's side). +set -euo pipefail +cd "$(dirname "$0")" +: "${CPLANTBOX:?set CPLANTBOX to the CPlantBox source folder}" "${CPLANTBOX_LIBRARY:?set CPLANTBOX_LIBRARY to libCPlantBox.a}" +cmake -S cplantbox -B cplantbox/build -DCMAKE_BUILD_TYPE=Release -DCPLANTBOX_SOURCE_DIR="$CPLANTBOX" \ + -DCPLANTBOX_LIBRARY="$CPLANTBOX_LIBRARY" > /dev/null +cmake --build cplantbox/build -j 4 > /dev/null +mvn -q -f ../../bindings/java/pom.xml install -DskipTests +mvn -q -f ../../bindings/java/groimp/pom.xml install -DskipTests +mvn -q -f groimp/pom.xml package dependency:copy-dependencies -DincludeScope=runtime +"${JULIA:-julia}" --project=virtualplantlab -e 'using Pkg; Pkg.instantiate(); using encoding3d, PlantGraphs, ZMQ' +python session.py "$PWD/cplantbox/build/roots" "$PWD/groimp/target/classes:$PWD/groimp/target/dependency/*" \ + "${JULIA:-julia}" "$(command -v python)" +if command -v encoding3d > /dev/null; then encoding3d validate outputs/record/scene.fspm.json | head -1; fi diff --git a/tutorials/04_four_platforms/session.py b/tutorials/04_four_platforms/session.py new file mode 100644 index 0000000..31475cb --- /dev/null +++ b/tutorials/04_four_platforms/session.py @@ -0,0 +1,44 @@ +""" +Tutorial 4: the session. encoding3d's coordinator launches the four platforms' models, each in its own process, and +runs ten daily steps: CPlantBox grows the roots, then GroIMP the shoot, then metafspm (root water uptake) and +VirtualPlantLab (light and photosynthesis) compute together. The shared plant is recorded at each step in +outputs/record (an encoding3d sequence). + + python session.py CPLANTBOX JAVA_CLASSPATH JULIA METAFSPM_PYTHON +""" +import os +import sys + +from openalea.encoding3d import read_index, read +from openalea.encoding3d.coupling import Coordinator, Link +from openalea.encoding3d.views import VIEWS_FILE, check_views, write_views + +here = os.path.dirname(os.path.abspath(__file__)) +cplantbox, classpath, julia, python = sys.argv[1:5] + +coordinator = Coordinator( + "tcp://127.0.0.1:*", steps=10, interval=1., time_unit="d", timeout=600, + stages=[["cplantbox"], ["groimp"], ["metafspm", "virtualplantlab"]], + links=[Link("metafspm", "plant_assimilation", "virtualplantlab", "plant_assimilation", aggregation="copy"), + Link("groimp", "water_uptake", "metafspm", "water_uptake", aggregation="sum")], + record=os.path.join(here, "outputs", "record")) +coordinator.launch("cplantbox", [cplantbox]) +coordinator.launch("groimp", ["java", "-cp", classpath, "tutorial.Shoot"]) +coordinator.launch("virtualplantlab", [julia, f"--project={os.path.join(here, 'virtualplantlab')}", + os.path.join(here, "virtualplantlab", "light.jl")]) +coordinator.launch("metafspm", [python, os.path.join(here, "metafspm", "uptake.py")]) +store = coordinator.run() +for coupling in coordinator.couplings: + source = f"{coupling.provider}.{coupling.source.name}" if coupling.source else "-" + print(f" {coupling.receiver}.{coupling.variable.name} <- {source} ({coupling.how}" + f"{', ' + coupling.aggregation if coupling.aggregation else ''}, factor {coupling.factor:g})") +# how to look at the record: the roots (CPlantBox's segments) as tubes, coloured by metafspm's uptake +record = os.path.join(here, "outputs", "record") +views = {"encoding3d_views": 1, "active": "uptake", "configurations": {"uptake": {"layers": [ + {"name": "roots", "source": {"data_structure": "plant", "kind": "segments", "entity_set": "SubOrgan"}, + "color_by": {"entity_set": "SubOrgan", "variable": "water_uptake"}, "colormap": "Viridis (matplotlib)", + "tube": {"radius": "radius", "scale": 1}}]}}} +write_views(os.path.join(record, VIEWS_FILE), views) +last = read(read_index(os.path.join(record, "scene.fspm.json"))["frames"][-1]["path"]) +print(check_views(views, last)) +print("recorded in", record) diff --git a/tutorials/04_four_platforms/virtualplantlab/Project.toml b/tutorials/04_four_platforms/virtualplantlab/Project.toml new file mode 100644 index 0000000..8e1400e --- /dev/null +++ b/tutorials/04_four_platforms/virtualplantlab/Project.toml @@ -0,0 +1,12 @@ +[deps] +PlantGraphs = "615ad455-9aac-4340-9247-71ef610f781a" +ZMQ = "c2297ded-f4af-51ae-bb23-16f91089e4e1" +encoding3d = "b7fd6adf-cc0c-448b-9b1a-23349074d866" + +[sources] +encoding3d = {path = "../../../bindings/julia/encoding3d"} + +[compat] +PlantGraphs = "1" +ZMQ = "1" +julia = "1.12" diff --git a/tutorials/04_four_platforms/virtualplantlab/light.jl b/tutorials/04_four_platforms/virtualplantlab/light.jl new file mode 100644 index 0000000..171adb5 --- /dev/null +++ b/tutorials/04_four_platforms/virtualplantlab/light.jl @@ -0,0 +1,60 @@ +# Tutorial 4, VirtualPlantLab's side: light and photosynthesis (functional, a toy). Each turn, the shoot's phytomers +# (GroIMP's, sent by the session) become a PlantGraphs graph (to_graph); a query then gives each leaf the light left +# under the leaves above it (Beer-Lambert) and its photosynthesis (a light-use efficiency). The plant's total +# assimilation goes to the plant, for metafspm. +using encoding3d +using PlantGraphs +using ZMQ + +const INCIDENT = 1000.0 # umol/s of light (PAR) falling on the plant's ground area +const GROUND = 0.01 # m2: the plant's ground area +const EXTINCTION = 0.6 # the canopy's light extinction coefficient +const EFFICIENCY = 0.05 # mol CO2 fixed per mol of light absorbed + +mutable struct Leaf <: Node # a phytomer's leaf, as the session sends it (to_graph fills its fields) + area::Float64 # m2 + height::Float64 # m +end + +declaration = Declaration("virtualplantlab"; platform = "VirtualPlantLab", + entity_sets = [EntitySetSpec("plant", "Plant"; rank = 1), EntitySetSpec("plant", "Phytomer"; rank = 4)], + variables = [VariableSpec("plant", "Phytomer", "area", "requires"; unit = "m2", kind = "extensive"), + VariableSpec("plant", "Phytomer", "height", "requires"; unit = "m", kind = "descriptor"), + VariableSpec("plant", "Phytomer", "absorbed_light", "provides"; unit = "umol/s", kind = "extensive"), + VariableSpec("plant", "Phytomer", "assimilation", "provides"; unit = "umol/s", kind = "extensive"), + VariableSpec("plant", "Plant", "plant_assimilation", "provides"; unit = "umol/s", kind = "extensive")]) +participant = Participant(declaration; timeout = 600) + +for turn in turns(participant) + plant = data_structure(turn.frame, "plant") + if !any(set -> set.name == "Phytomer", plant.entity_sets) || row_count(entity_set(plant, "Phytomer")) == 0 + send_update(participant, Frame()) + continue + end + shoot = entity_set(plant, "Phytomer") + # the phytomers as a VirtualPlantLab graph: each row a Leaf node, its fields from the columns + static, graph_ids = to_graph(plant; entity_set = "Phytomer", types = Dict("Phytomer" => Leaf)) + canopy = Graph(axiom = static) + # Beer-Lambert down the canopy, by a query: a leaf receives the light the leaves above it let through + leaves = sort(apply(canopy, Query(Leaf)); by = leaf -> -leaf.height) + absorbed, assimilation = IdDict{Leaf,Float64}(), IdDict{Leaf,Float64}() + area_above = 0.0 + for leaf in leaves + light = INCIDENT * exp(-EXTINCTION * area_above / GROUND) + absorbed[leaf] = light * (1 - exp(-EXTINCTION * leaf.area / GROUND)) + assimilation[leaf] = EFFICIENCY * absorbed[leaf] + area_above += leaf.area + end + # sent: each phytomer's light and assimilation (its row's leaf, by its id), the plant's total + leaf_of = [static_graph(canopy)[graph_ids[id]].data for id in shoot.columns["id"]] # the graph holds a copy + phytomers = EntitySet("Phytomer"; rank = 4, columns = encoding3d.Columns( + "id" => shoot.columns["id"], + "absorbed_light" => [absorbed[leaf] for leaf in leaf_of], + "assimilation" => [assimilation[leaf] for leaf in leaf_of])) + total = sum(values(assimilation)) + plants = EntitySet("Plant"; rank = 1, columns = encoding3d.Columns( + "id" => entity_set(plant, "Plant").columns["id"], "plant_assimilation" => [total])) + send_update(participant, Frame(data_structures = [DataStructure("plant"; entity_sets = [plants, phytomers])])) + println("virtualplantlab: ", length(leaves), " leaves, assimilation ", round(total; digits = 2), " umol/s") +end +println("virtualplantlab: the session ended (", participant.stopped, ")") diff --git a/tutorials/README.md b/tutorials/README.md index 4ba8f74..178cfa2 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -11,6 +11,7 @@ its explanation (`README.md`), its code, and `run.sh`, which runs it from start | `02c_cplantbox` | a CPlantBox plant written day by day with its geometry, soil grid and mapping; frames back as CPlantBox root systems | C++17, CMake, CPlantBox built | | `02d_groimp` | a GroIMP graph written step by step and rebuilt as GroIMP nodes; GroIMP's plugin exporting a project with its geometry | Java 21, Maven (GroIMP's Graph module) | | `03_metafspm_and_virtualplantlab` | a coupling session: metafspm's root growth and VirtualPlantLab's carbon model on one plant, over ZeroMQ, recorded | metafspm's environment, Julia 1.12 | +| `04_four_platforms` | CPlantBox's roots and GroIMP's shoot (structural), metafspm's water uptake and VirtualPlantLab's light and photosynthesis (functional), one plant, one session | everything tutorials 2a to 2d need | **encoding3d depends on no platform.** Tutorial 1 needs encoding3d only, and encoding3d's CI runs it. The tutorials using a platform (metafspm, VirtualPlantLab, CPlantBox, GroIMP) list what they need in their own folder, outside From 4d88aaa8488e7108b6453bbee41198ab1c832e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tristan=20G=C3=A9rault?= Date: Thu, 5 Nov 2026 12:54:22 +0100 Subject: [PATCH 39/39] Tutorial 2d: the frames drawable: GroIMP's graph module has no geometry (the GroIMP application's shape nodes do, which the plugin meshes), so each organ keeps its two ends (x1 ... z2, the vocabulary's segment ends), the internodes stacked, the leaves leaning out to alternate sides; drawn by ParaView's reader --- tutorials/02d_groimp/README.md | 7 ++++ .../main/java/tutorial/GroImpTutorial.java | 34 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tutorials/02d_groimp/README.md b/tutorials/02d_groimp/README.md index 1777997..1e3043c 100644 --- a/tutorials/02d_groimp/README.md +++ b/tutorials/02d_groimp/README.md @@ -16,11 +16,18 @@ parts: GroIMP manages (`Managed.java` declares them, as GroIMP's XL compiler does for a model's classes). The plant is made of its internodes (a `/` edge), each internode followed by the next (`>`) and bearing a leaf (`+`). It grows by code standing for an RGG rule: a new internode and leaf on top, every leaf longer. + GroIMP's graph module has no geometry: in the GroIMP application, shape nodes (cylinders, …) place the organs in + space, and the plugin meshes them (below). So that viewers can draw this plant, each organ keeps its two ends + (`x1` … `z2`, declared as the vocabulary's `proximal_end_x` … `distal_end_z`): the internodes stacked upwards, the + leaves leaning out to alternate sides. 2. **A frame per step.** `GroImp.toDataStructure` writes the plant and its organs as two entity sets, the edges as `parent` and `edge_type`, the managed fields as columns (`length` and `radius` declared in metres). Six steps make a sequence. 3. **Back into GroIMP.** `GroImp.toGraph` rebuilds each row as a node of its class, its fields set. +Open `outputs/frames/scene.fspm.json` in ParaView with encoding3d's plugin (tutorial 1): the stem and its leaves, +drawn from their ends, grow step by step. + ## From GroIMP itself: the plugin GroIMP's encoding3d plugin (`bindings/java/groimp-plugin/`, also a proposal for GroIMP) exports a GroIMP project with diff --git a/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java index d5da424..1c51d2f 100644 --- a/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java +++ b/tutorials/02d_groimp/src/main/java/tutorial/GroImpTutorial.java @@ -32,26 +32,49 @@ public static final class Plant extends Node { @Override protected Node newInstance() { return new Plant(); } } + // GroIMP's graph module has no geometry: in the GroIMP application, shape nodes (cylinders...) place the organs in + // space, and the plugin meshes them. Here each organ keeps its two ends (x1 ... z2), so that viewers draw it. public static final class Internode extends Node { double length = 0.02; float radius = 0.002f; - public static final NType $TYPE = Managed.type(Internode.class, "length", "radius"); + double x1, y1, z1, x2, y2, z2 = 0.02; // m: its lower and upper ends + public static final NType $TYPE = Managed.type(Internode.class, "length", "radius", + "x1", "y1", "z1", "x2", "y2", "z2"); @Override protected NType getNTypeImpl() { return $TYPE; } @Override protected Node newInstance() { return new Internode(); } } public static final class Leaf extends Node { double length = 0.01; - public static final NType $TYPE = Managed.type(Leaf.class, "length"); + float radius = 0.001f; + double x1, y1, z1, x2, y2, z2; // m: where it leaves its internode, and its tip + double side = 1; // which side it leans to: +1 or -1 (not managed) + public static final NType $TYPE = Managed.type(Leaf.class, "length", "radius", + "x1", "y1", "z1", "x2", "y2", "z2"); + + /** Its tip, from its base: leaning 45 degrees to its side. */ + void place() { + x2 = x1 + side * length * Math.sqrt(0.5); + y2 = y1; + z2 = z1 + length * Math.sqrt(0.5); + } @Override protected NType getNTypeImpl() { return $TYPE; } @Override protected Node newInstance() { return new Leaf(); } } /** The plant grows as an RGG rule would make it: a new internode bearing a leaf on top, every leaf longer. */ static Internode grow(Internode top, List leaves) { - for (Leaf leaf : leaves) leaf.length += 0.02; + for (Leaf leaf : leaves) { + leaf.length += 0.02; + leaf.place(); + } Internode next = new Internode(); + next.x1 = top.x2; next.y1 = top.y2; next.z1 = top.z2; // on top of the previous one + next.x2 = next.x1; next.y2 = next.y1; next.z2 = next.z1 + next.length; Leaf leaf = new Leaf(); + leaf.x1 = next.x2; leaf.y1 = next.y2; leaf.z1 = next.z2; + leaf.side = leaves.size() % 2 == 0 ? 1 : -1; // alternate sides + leaf.place(); top.addEdgeBitsTo(next, Graph.SUCCESSOR_EDGE, null); // GroIMP's '>' edge next.addEdgeBitsTo(leaf, Graph.BRANCH_EDGE, null); // GroIMP's '+' edge leaves.add(leaf); @@ -69,6 +92,11 @@ static GroImp.Options options() { options.variables.put("length", new Variable("m", "length", "extensive", "state")); options.variables.put("radius", new Variable("m", "radius", "intensive", "state")); options.variables.put("species", new Variable("1", null, "descriptor", "parameter")); + String[] ends = {"x1", "y1", "z1", "x2", "y2", "z2"}; + String[] standardNames = {"proximal_end_x", "proximal_end_y", "proximal_end_z", "distal_end_x", "distal_end_y", + "distal_end_z"}; + for (int index = 0; index < ends.length; index++) + options.variables.put(ends[index], new Variable("m", standardNames[index], "descriptor", "state")); return options; }