Develop merge for first alpha release

.github/actions/conformance/action.yml

@@ -0,0 +1,50 @@
+# Checks a codec against fspm-TF's conformance files, in a codec's own CI, once the codec has run:
+#
+#   - uses: ./.github/actions/conformance            (in this repository; elsewhere openalea/fspm-TF/.github/actions/conformance@main)
+#     with:
+#       dumps: out/dumps          # what the codec read: one JSON dump per conformance GLB, x.json for x.expected.json
+#       written: out/written      # what the codec wrote: the conformance scenes, as .glb files (optional)
+#
+# Reading passes when every dump equals its expected values; writing, when every written file passes fspm-tf validate
+# and Khronos' glTF-Validator with no error.
+name: fspm-TF conformance
+description: Compare a codec's reading of the conformance files with their expected values, and validate what it wrote
+inputs:
+  dumps:
+    description: the folder of the codec's JSON dumps of the conformance files
+    required: true
+  written:
+    description: the folder of the .glb files the codec wrote for the conformance scenes (empty to skip)
+    required: false
+    default: ""
+  fspm-tf:
+    description: the path of an fspm-TF checkout (its conformance folder and Python package)
+    required: false
+    default: "."
+  gltf-validator:
+    description: the version of Khronos' glTF-Validator
+    required: false
+    default: "2.0.0-dev.3.10"
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+    - name: Install openalea.fspm_tf
+      shell: bash
+      run: python -m pip install numpy jsonschema "${{ inputs.fspm-tf }}"
+    - name: Reading
+      shell: bash
+      run: fspm-tf compare-all "${{ inputs.dumps }}" "${{ inputs.fspm-tf }}/conformance"
+    - name: Writing, fspm-TF validator
+      if: inputs.written != ''
+      shell: bash
+      run: find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 fspm-tf validate
+    - name: Writing, Khronos' glTF-Validator
+      if: inputs.written != '' && runner.os == 'Linux'
+      shell: bash
+      run: |
+        curl -sSL -o validator.tar.xz "https://github.com/KhronosGroup/glTF-Validator/releases/download/${{ inputs.gltf-validator }}/gltf_validator-${{ inputs.gltf-validator }}-linux64.tar.xz"
+        tar xf validator.tar.xz gltf_validator
+        find "${{ inputs.written }}" -name "*.glb" -print0 | xargs -0 -n1 ./gltf_validator

.github/workflows/conformance.yml

@@ -46,3 +46,20 @@ jobs:
         run: python -m pip install numpy jsonschema sphinx pydata-sphinx-theme myst-parser sphinx-favicon sphinx-copybutton .
       - name: Build the documentation, warnings as errors
         run: python -m sphinx -b html docs docs/_build -W --keep-going
+
+  # The reference codec through the action other codecs use: its dumps of the conformance files, and the files it writes
+  python-codec-conformance:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Dump and write the conformance scenes with the Python codec
+        run: |
+          python -m pip install numpy jsonschema .
+          python bindings/python/conformance_run.py conformance out
+      - uses: ./.github/actions/conformance
+        with:
+          dumps: out/dumps
+          written: out/written

README.md

@@ -1,4 +1,4 @@
-# fspm-TF
+# FSPM-TF - Functional Structural Plant Model Transmission Format - based on GLTF 2.0
 
 An exchange format for whole functional-structural plant model (FSPM) scenes, across platforms and languages
 (openalea.metafspm, CPlantBox, GroIMP, VirtualPlantLab). It holds:
@@ -30,7 +30,7 @@ change there, released as a new version. Everything else implements, checks or d
 | `spec/specification.md` | the specification: data model, glTF layout, sequences, conformance | **yes** |
 | `spec/schema/` | the JSON Schemas of the `FSPM_scene` extension and of a sequence's index | **yes** |
 | `spec/vocabulary/` | the standard names (units, kinds, definitions, and each platform's names for them), the labels with their Plant Ontology terms, the enumerations | **yes** |
-| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code | follows `spec/` |
+| `conformance/` | reference files, each a GLB with its expected values as JSON, written by `generate.py`: the specification as test cases for every platform's code (`conformance/README.md`; checked with `fspm-tf compare-all` and the action `.github/actions/conformance`) | follows `spec/` |
 | `bindings/<language>/` | each language's codec, with its own build, tests and release; `bindings/python/`: `openalea.fspm_tf`, the reference (codec, validator, display copies, command line), which reads `spec/` and ships it | no |
 | `paraview/` | the ParaView reader plugin, and the script checking it in a `pvpython` | no |
 | `docs/design_note.md` | how the decisions were reached | no |

bindings/python/conformance_run.py

@@ -0,0 +1,38 @@
+"""
+The Python codec's run of the conformance check, as every codec does it in its CI: read each conformance GLB and dump
+it as JSON (OUT/dumps/x.json for x.glb), and write the conformance scenes again as GLB files (OUT/written/x.glb).
+Then .github/actions/conformance compares the dumps with the expected values and validates the written files.
+
+    python bindings/python/conformance_run.py CONFORMANCE OUT
+"""
+import json
+import os
+import shutil
+import sys
+
+from openalea import fspm_tf
+from openalea.fspm_tf.dump import frame_to_dict
+
+
+def main(conformance: str, out: str) -> int:
+    for folder, _, files in os.walk(conformance):
+        for name in sorted(files):
+            if not name.endswith(".glb"):
+                continue
+            relative = os.path.relpath(os.path.join(folder, name), conformance)
+            frame = fspm_tf.read(os.path.join(conformance, relative))
+            dump = os.path.join(out, "dumps", relative[:-len(".glb")] + ".json")
+            written = os.path.join(out, "written", relative)
+            for path in (dump, written):
+                os.makedirs(os.path.dirname(path), exist_ok=True)
+            with open(dump, "w", encoding="utf-8") as file:
+                json.dump(frame_to_dict(frame), file, indent=1)
+            fspm_tf.write(written, frame)                       # written again from what was read
+    index = os.path.join(conformance, "growing_sequence", "scene.fspm.json")
+    if os.path.exists(index):
+        shutil.copy(index, os.path.join(out, "written", "growing_sequence", "scene.fspm.json"))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(*sys.argv[1:3]))

bindings/python/src/openalea/fspm_tf/cli.py

@@ -1,10 +1,14 @@
 """
-The fspm-tf command line:
+The fspm-tf command line::
 
     fspm-tf validate FILE...                    check .glb frames or sequences (scene.fspm.json); exit 1 on errors
     fspm-tf info FILE                           what a frame holds
     fspm-tf display FILE NAME... [-o OUT]       add display copies of variables (in place, or to OUT)
     fspm-tf display FILE --remove [-o OUT]      remove them
+    fspm-tf compare ACTUAL EXPECTED             a codec's JSON dump (or a .glb) against expected values; exit 1 if they
+                                                differ
+    fspm-tf compare-all DUMPS CONFORMANCE       every conformance file's expected values against the dump at the same
+                                                path (x.expected.json: DUMPS/x.json); exit 1 if any differ
 """
 import argparse
 import sys
@@ -58,6 +62,31 @@ def _display(arguments) -> int:
     return 0
 
 
+def _print_differences(name: str, differences: list, shown: int = 20) -> None:
+    print(f"{name}: {'equal' if not differences else f'{len(differences)} difference(s)'}")
+    for line in differences[:shown]:
+        print(f"  {line}")
+    if len(differences) > shown:
+        print(f"  ... {len(differences) - shown} more")
+
+
+def _compare(arguments) -> int:
+    from openalea.fspm_tf.compare import compare_files
+    differences = compare_files(arguments.actual, arguments.expected)
+    _print_differences(arguments.actual, differences)
+    return 1 if differences else 0
+
+
+def _compare_all(arguments) -> int:
+    from openalea.fspm_tf.compare import compare_all
+    results = compare_all(arguments.dumps, arguments.conformance)
+    for name, differences in results.items():
+        _print_differences(name, differences)
+    failed = sum(1 for differences in results.values() if differences)
+    print(f"{len(results) - failed} of {len(results)} conformance files read as expected")
+    return 1 if failed or not results else 0
+
+
 def main(argv=None) -> int:
     parser = argparse.ArgumentParser(prog="fspm-tf", description="fspm-TF files: validate, inspect, display")
     commands = parser.add_subparsers(dest="command", required=True)
@@ -74,6 +103,15 @@ def main(argv=None) -> int:
     display.add_argument("--remove", action="store_true", help="remove display copies (all, or the names given)")
     display.add_argument("-o", "--output", default=None, help="write to this file instead of in place")
     display.set_defaults(run=_display)
+    compare = commands.add_parser("compare", help="a codec's JSON dump (or a .glb) against expected values")
+    compare.add_argument("actual")
+    compare.add_argument("expected")
+    compare.set_defaults(run=_compare)
+    compare_all = commands.add_parser("compare-all", help="dumps of every conformance file against their expected "
+                                                          "values")
+    compare_all.add_argument("dumps", help="the folder of the codec's dumps (x.json for x.expected.json)")
+    compare_all.add_argument("conformance", help="fspm-TF's conformance folder")
+    compare_all.set_defaults(run=_compare_all)
     arguments = parser.parse_args(argv)
     return arguments.run(arguments)
 

bindings/python/src/openalea/fspm_tf/codec.py

@@ -30,6 +30,9 @@
 ARRAY_BUFFER, ELEMENT_ARRAY_BUFFER = 34962, 34963
 FLOAT, UNSIGNED_INT = 5126, 5125
 MAX_FLOAT_FEATURE = 2 ** 24                     # feature rows are stored as float32 vertex attributes
+# The scene's frame is z up (metres); glTF viewers expect y up: the root node turns the scene for them (-90° about x,
+# column-major). Readers of the tables use the scene's frame, not this node.
+Z_UP_TO_Y_UP = [1., 0., 0., 0., 0., 0., -1., 0., 0., 1., 0., 0., 0., 0., 0., 1.]
 
 COMPONENT_TYPES = {np.dtype(name): name.upper() for name in (
     "int8", "uint8", "int16", "uint16", "int32", "uint32", "int64", "uint64", "float32", "float64")}
@@ -54,7 +57,8 @@ def to_bytes(frame: Frame) -> bytes:
                                                          for value, name in enumerate(EDGE_TYPES)]}}}
     tables, variables = [], {}
     document = {"asset": {"version": "2.0", "generator": f"fspm-tf {SPECIFICATION_VERSION}"},
-                "scene": 0, "scenes": [{"nodes": []}], "nodes": []}
+                "scene": 0, "scenes": [{"nodes": [0]}],
+                "nodes": [{"name": "fspm_scene", "matrix": Z_UP_TO_Y_UP, "children": []}]}
     accessors, meshes = [], []
 
     def add_table(table: Table, class_id: str) -> int:
@@ -95,7 +99,7 @@ def add_table(table: Table, class_id: str) -> int:
         if not np.array_equal(data_structure.origin, np.eye(4)):
             node["matrix"] = entry["origin"]
         document["nodes"].append(node)
-        document["scenes"][0]["nodes"].append(len(document["nodes"]) - 1)
+        document["nodes"][0]["children"].append(len(document["nodes"]) - 1)
         entry["node"] = len(document["nodes"]) - 1
         if data_structure.meshes:
             node["children"], entry["meshes"] = [], []

bindings/python/src/openalea/fspm_tf/compare.py

@@ -0,0 +1,79 @@
+"""
+Comparing what a codec reads with the conformance files' expected values (``*.expected.json``), whatever the codec's
+language: the codec dumps each GLB it reads as JSON (the layout of fspm_tf.dump), and this compares the dumps::
+
+    compare(actual, expected) -> [differences]         two JSON values: objects by key, arrays in order, numbers
+                                                       exactly (1 and 1.0 equal), null, "Infinity", "-Infinity", text
+    compare_all(dumps, conformance) -> {file: [differences]}
+                                                       every *.expected.json of the conformance folder against the
+                                                       dump at the same relative path (x.expected.json: x.json)
+"""
+import json
+import math
+import os
+
+from openalea.fspm_tf.dump import frame_to_dict
+
+
+def _number(value) -> bool:
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def compare(actual, expected, path: str = "$") -> list:
+    """The differences between two JSON values, as "path: what differs" lines (none when equal)."""
+    if isinstance(expected, dict):
+        if not isinstance(actual, dict):
+            return [f"{path}: an object expected, got {type(actual).__name__}"]
+        found = [f"{path}: missing key '{key}'" for key in expected if key not in actual]
+        found += [f"{path}: unexpected key '{key}'" for key in actual if key not in expected]
+        for key in expected:
+            if key in actual:
+                found += compare(actual[key], expected[key], f"{path}.{key}")
+        return found
+    if isinstance(expected, list):
+        if not isinstance(actual, list):
+            return [f"{path}: an array expected, got {type(actual).__name__}"]
+        if len(actual) != len(expected):
+            return [f"{path}: {len(actual)} items, {len(expected)} expected"]
+        found = []
+        for index, (left, right) in enumerate(zip(actual, expected)):
+            found += compare(left, right, f"{path}[{index}]")
+        return found
+    if _number(expected):
+        if not _number(actual) or float(actual) != float(expected) or (math.isnan(float(actual))):
+            return [f"{path}: {actual!r}, {expected!r} expected"]
+        return []
+    if actual != expected or type(actual) is not type(expected):
+        return [f"{path}: {actual!r}, {expected!r} expected"]
+    return []
+
+
+def load(path: str):
+    """A JSON dump, or a GLB read by this package and dumped (to check this package itself)."""
+    if path.endswith(".glb"):
+        from openalea.fspm_tf import read
+        return json.loads(json.dumps(frame_to_dict(read(path))))
+    with open(path, encoding="utf-8") as file:
+        return json.load(file)
+
+
+def compare_files(actual_path: str, expected_path: str) -> list:
+    return compare(load(actual_path), load(expected_path))
+
+
+def compare_all(dumps: str, conformance: str) -> dict:
+    """{expected file (relative): its differences} for every *.expected.json under *conformance*, against the dump at
+    the same relative path under *dumps* (a missing dump is a difference)."""
+    results = {}
+    for folder, _, files in sorted(os.walk(conformance)):
+        for name in sorted(files):
+            if not name.endswith(".expected.json"):
+                continue
+            expected = os.path.join(folder, name)
+            relative = os.path.relpath(expected, conformance)
+            actual = os.path.join(dumps, relative[:-len(".expected.json")] + ".json")
+            if not os.path.exists(actual):
+                results[relative] = [f"no dump {os.path.relpath(actual, dumps)}"]
+                continue
+            results[relative] = compare_files(actual, expected)
+    return results

bindings/python/src/openalea/fspm_tf/ucum.py

@@ -25,6 +25,7 @@ def _dims(**exponents) -> tuple:
     "lm": _dims(J=1), "lx": _dims(J=1, L=-2), "Bq": _dims(T=-1), "Gy": _dims(L=2, T=-2), "Sv": _dims(L=2, T=-2),
     "L": _dims(L=3), "l": _dims(L=3), "bar": _dims(M=1, L=-1, T=-2), "ar": _dims(L=2), "t": _dims(M=1),
     "eq": _dims(N=1), "Cel": _dims(Th=1),
+    "m[H2O]": _dims(M=1, L=-1, T=-2), "m[Hg]": _dims(M=1, L=-1, T=-2),     # pressures as heights: cm[H2O], mm[Hg]
 }
 ATOMS = dict(METRIC_ATOMS, **{
     "min": _dims(T=1), "h": _dims(T=1), "d": _dims(T=1), "wk": _dims(T=1), "mo": _dims(T=1), "a": _dims(T=1),
@@ -33,7 +34,7 @@ def _dims(**exponents) -> tuple:
 SPECIAL = {"Cel"}                                   # an offset scale: not allowed in products, quotients, powers
 PREFIXES = ("da", "Y", "Z", "E", "P", "T", "G", "M", "k", "h", "d", "c", "m", "u", "n", "p", "f", "a", "z", "y")
 
-_TOKEN = re.compile(r"\{[^{}]*\}|\(|\)|\.|/|[+-]?\d+|\[[^\]]+\]|%|[A-Za-z]+")
+_TOKEN = re.compile(r"\{[^{}]*\}|\(|\)|\.|/|[+-]?\d+|[A-Za-z]+\[[^\]]+\]|\[[^\]]+\]|%|[A-Za-z]+")
 
 
 class UnitError(ValueError):

bindings/python/src/openalea/fspm_tf/validate.py

@@ -16,7 +16,8 @@
 import numpy as np
 
 from openalea.fspm_tf import schema as schemas, ucum, vocabulary
-from openalea.fspm_tf.codec import (DTYPES, EDGE_TYPE_ENUM, EXTENSION, FEATURES, METADATA, from_bytes)
+from openalea.fspm_tf.codec import (DTYPES, EDGE_TYPE_ENUM, EXTENSION, FEATURES, METADATA, Z_UP_TO_Y_UP,
+                                    from_bytes)
 from openalea.fspm_tf.glb import unpack
 from openalea.fspm_tf.model import EDGE_TYPES, NO_ID
 
@@ -95,6 +96,11 @@ def _check_document(document: dict, binary: bytes, report: Report) -> bool:
             report.error("extensionsUsed", f"must list {name}")
     if EXTENSION in document.get("extensionsRequired", []):
         report.warning("extensionsRequired", f"{EXTENSION} should not be required, so that any viewer opens the file")
+    roots = [document.get("nodes", [])[index] for index in document.get("scenes", [{}])[0].get("nodes", [])
+             if index < len(document.get("nodes", []))]
+    if not any(node.get("name") == "fspm_scene" and node.get("matrix") == Z_UP_TO_Y_UP for node in roots):
+        report.warning("scenes", "no root node fspm_scene turning the scene (z up) to glTF's y up: viewers will show it "
+                                 "lying down")
     extensions = document.get("extensions", {})
     if EXTENSION not in extensions:
         report.error("extensions", f"no {EXTENSION} extension: not an fspm-TF file")