diff --git a/.gitignore b/.gitignore index c1f4cd12..0435eff4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -# xtend -**/src-gen/ -**/xtend-gen/ # Compiled Java class files *.class # Compiled Python bytecode @@ -41,8 +38,6 @@ Thumbs.db *.wmv # Jupyter Notebook .ipynb_checkpoints -# some other generated files -*._trace # Python __pycache__ .pyenv/ @@ -53,12 +48,9 @@ __pycache__ **/.checkstyle # project **/*.egg-info -**/rune.runtime-*.*.*-py3-none-any.whl **/test_helper-0.0.0-py3-none-any.whl **/python-test/serialization-tests/rune-common **/python-test/cdm-tests/rosetta/ src/generated/ *.todo /build/ -test/*** -**/python-test/serialization-tests/error-handling diff --git a/build/example_function/AddBusinessDays.py b/build/example_function/AddBusinessDays.py deleted file mode 100644 index 6ec97359..00000000 --- a/build/example_function/AddBusinessDays.py +++ /dev/null @@ -1,77 +0,0 @@ -# pylint: disable=invalid-name, missing-module-docstring -import sys -import inspect -import datetime -from rosetta.runtime.func_proxy import replaceable, create_module_attr_guardian -from rosetta.runtime.utils import if_cond_fn, rosetta_resolve_attr -from cdm.base.datetime.BusinessCenterEnum import BusinessCenterEnum -from cdm.base.datetime.functions.IsBusinessDay import IsBusinessDay -from cdm.base.datetime.functions.AddDays import AddDays - - -@replaceable -def AddBusinessDays(originalDate: datetime.date, offsetBusinessDays: int, - businessCenters: list[BusinessCenterEnum] | None) -> datetime.date: - ''' Returns a good business date that has been offset by the given number - of business days given the supplied business centers. A negative value - implies an earlier date (before the supplied originalDate), and a - positive value a later date (after the supplied date). - - Parameters - ---------- - originalDate : datetime.date - date to be shifted. If not a good business day, - a supplied shift of 0 will shift it to the next business day - - offsetBusinessDays : int - number of business days to shift the original date - - businessCenters : list[BusinessCenterEnum] | None - business centers to use in the shifting - - Returns - ------- - shiftedDate : datetime.date - - ''' - self = inspect.currentframe() - - def _then_fn0(): - return -1 - - def _else_fn0(): - return 1 - - def _then_fn1(): - return rosetta_resolve_attr(self, "shift") - - def _else_fn1(): - return 0 - - def _then_fn2(): - return 0 - - def _else_fn2(): - return rosetta_resolve_attr(self, "offsetBusinessDays") - rosetta_resolve_attr(self, "newShift") - - def _then_fn3(): - return rosetta_resolve_attr(self, "originalDate") - - def _else_fn3(): - return AddBusinessDays(rosetta_resolve_attr(self, "shiftedByOne"), rosetta_resolve_attr(self, "newOffset"), rosetta_resolve_attr(self, "businessCenters")) - - isGoodBusinessDay = IsBusinessDay(rosetta_resolve_attr(self, "originalDate"), rosetta_resolve_attr(self, "businessCenters")) - shift = if_cond_fn(((rosetta_resolve_attr(self, "offsetBusinessDays")) < 0), _then_fn0, _else_fn0) - shiftedByOne = AddDays(rosetta_resolve_attr(self, "originalDate"), rosetta_resolve_attr(self, "shift")) - isShiftedGood = IsBusinessDay(rosetta_resolve_attr(self, "shiftedByOne"), rosetta_resolve_attr(self, "businessCenters")) - newShift = if_cond_fn((rosetta_resolve_attr(self, "isShiftedGood")), _then_fn1, _else_fn1) - newOffset = if_cond_fn(((rosetta_resolve_attr(self, "offsetBusinessDays")) == 0), _then_fn2, _else_fn2) - done = ((rosetta_resolve_attr(self, "offsetBusinessDays") == 0) and (rosetta_resolve_attr(ctx.f_locals, "isGoodBusinessDay") == True)) - newDate = if_cond_fn((rosetta_resolve_attr(self, "done")), _then_fn3, _else_fn3) - shiftedDate = rosetta_resolve_attr(self, "newDate") - return rosetta_resolve_attr(self, "shiftedDate") - - -sys.modules[__name__].__class__ = create_module_attr_guardian(sys.modules[__name__].__class__) - -# EOF diff --git a/build/github_actions_cleanup.sh b/build/github_actions_cleanup.sh deleted file mode 100755 index ffedc1a4..00000000 --- a/build/github_actions_cleanup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -repo=$1 - -# Ensure the repo argument is provided -if [[ -z "$repo" ]]; then - echo "Error: Repository not specified." - echo "Usage: gh_bulk_delete_workflow_runs " - exit 1 -fi - -# Fetch all workflow runs for the given repository -runs=$(gh api repos/$repo/actions/runs --paginate | jq -r '.workflow_runs[] | .id') - -if [[ -z "$runs" ]]; then - echo "No workflow runs found for $repo." - exit 0 -fi - -while IFS= read -r run; do - echo "Deleting run $run..." - gh api -X DELETE repos/$repo/actions/runs/$run --silent -done <<< "$runs" - -echo "All workflow runs for $repo have been deleted." \ No newline at end of file diff --git a/docs/BUILD_AND_TEST.md b/docs/BUILD_AND_TEST.md index 4aa58520..982813a6 100644 --- a/docs/BUILD_AND_TEST.md +++ b/docs/BUILD_AND_TEST.md @@ -29,6 +29,18 @@ mvn clean package The JUnit tests compare generated Python output against expected results. All tests should pass. +### Known Build Warnings + +These warnings come from upstream dependencies and don't affect the build. They are left visible rather than suppressed. + +- **`jvnet-nexus-releases` (Maven 3.10 and later):** + + ``` + [WARNING] Could not transfer metadata /.meta/prefixes.txt from/to jvnet-nexus-releases (https://maven.java.net/content/repositories/releases/): (certificate_expired) PKIX path validation failed + ``` + + `maven-checkstyle-plugin` 3.6.0 depends on `javax.xml.bind:jaxb-api` 2.3.1, whose parent POM declares the retired `maven.java.net` repository. Maven 3.10 checks that repository and reports the failure; every file still resolves from Maven Central. The dependency is removed upstream for `maven-checkstyle-plugin` 3.6.1 ([apache/maven-checkstyle-plugin#736](https://github.com/apache/maven-checkstyle-plugin/issues/736)), which is not yet released. Tracked in [#335](https://github.com/finos/rune-python-generator/issues/335). + --- ## Run Python Unit Tests diff --git a/pom.xml b/pom.xml index 335b5c0b..bcd30c92 100644 --- a/pom.xml +++ b/pom.xml @@ -269,10 +269,6 @@ 1.5.3 - - org.eclipse.emf - org.eclipse.emf.codegen.ecore - com.google.inject guice @@ -377,6 +373,10 @@ src/generated/** + + **/.pyenv/** + **/.profile-pyenv/** + **/__pycache__/** @@ -385,7 +385,9 @@ check-license-headers - verify + + validate check @@ -419,6 +421,12 @@ + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/python-test/cdm-tests/conftest.py b/python-test/cdm-tests/conftest.py new file mode 100644 index 00000000..2aff88b9 --- /dev/null +++ b/python-test/cdm-tests/conftest.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''Collects round-trip results and prints a summary grouped by error type.''' +from collections import Counter, defaultdict + +import pytest + +_ROUNDTRIP_KEY = pytest.StashKey[dict]() + + +def pytest_configure(config): + config.stash[_ROUNDTRIP_KEY] = {} + + +@pytest.fixture +def roundtrip_results(request) -> dict: + '''Maps each sample to its problems, as (error type, example) pairs: the + field for a difference, or the sample for a read or write failure. An empty + list means the sample came back unchanged.''' + return request.config.stash[_ROUNDTRIP_KEY] + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + '''Prints one row per error type, with the number of samples it affects.''' + del exitstatus # unused + results = config.stash.get(_ROUNDTRIP_KEY, {}) + if not results: + return + + files = defaultdict(set) + occurrences = Counter() + details = defaultdict(Counter) + for sample, problems in results.items(): + for category, detail in problems: + files[category].add(sample) + occurrences[category] += 1 + details[category][detail] += 1 + + unchanged = sum(1 for problems in results.values() if not problems) + write = terminalreporter.write_line + terminalreporter.write_sep('=', 'FpML 5.10 round-trip summary') + write(f'Samples: {len(results)} unchanged: {unchanged} ' + f'with problems: {len(results) - unchanged}') + if not files: + return + + rows = [(category, str(len(files[category])), str(occurrences[category]), + details[category].most_common(1)[0][0]) + for category in sorted(files, key=lambda c: (-len(files[c]), c))] + headers = ('Error type', 'Samples', 'Occurrences', 'Most common field or example sample') + widths = [max(len(row[i]) for row in rows + [headers]) for i in range(3)] + write('') + write(' '.join(h.ljust(w) for h, w in zip(headers, widths)) + ' ' + headers[3]) + write(' '.join('-' * w for w in widths) + ' ' + '-' * len(headers[3])) + for row in rows: + write(' '.join(cell.ljust(w) for cell, w in zip(row, widths)) + ' ' + row[3]) diff --git a/python-test/cdm-tests/java-validation/README.md b/python-test/cdm-tests/java-validation/README.md new file mode 100644 index 00000000..3b19dfb7 --- /dev/null +++ b/python-test/cdm-tests/java-validation/README.md @@ -0,0 +1,56 @@ +# CDM ingestion samples: Java validation + +Reads, round-trips and validates CDM's FpML ingestion samples with CDM's own Java code, to produce a Java baseline for comparison with the generated Python. + +The samples are the ones listed in CDM's test-pack configs, `rosetta-source/src/main/resources/ingest/config/test-pack-*.json`. They cover FpML 5.10, 5.12 and 5.13, trade states and workflow steps, including the `incomplete` and `invalid` packs. + +Run it by hand; it isn't part of the build or CI. It builds CDM Java from source, which takes a few minutes and needs network access the first time. + +## Requirements + +- Java 21 and Maven +- Python 3.11+ +- A local git clone of [finos/common-domain-model](https://github.com/finos/common-domain-model) + +## Run + +From the project root: + +```sh +python-test/cdm-tests/java-validation/run_java_validation.sh +``` + +- ``: path to the CDM clone. +- ``: the CDM commit, branch or tag to test, for example `upstream/master`. Fetch it first. +- ``: a directory **outside this repository**. The script builds CDM there and writes the results to `/results/`. + +For each sample, `ValidateSamples`: + +1. reads the sample's output JSON with `RuneJsonObjectMapper`, as a `TradeState`, or a `WorkflowStep` for the `workflow-step` packs; +2. writes it back and compares it with the original, ignoring `@version` and comparing numbers by value; +3. resolves its references with `ReferenceResolverProcessStep`; +4. validates it with `RosettaTypeValidator`. + +## Results + +| File | One row per | Columns | +|---|---|---| +| `all-samples.tsv` | sample | test pack, output path, `modelValidationFailures` and `runtimeError` from the config | +| `java-summary.tsv` | sample | test pack, sample, config failure count, Java failure count, round trip (`same` or `DIFF`), read error | +| `java-failures.tsv` | validation failure | test pack, sample, type (`CARDINALITY` or `DATA_RULE`), object, rule, path, reason | + +A sample with a read error has no failure count or round-trip result. An empty output file in CDM shows up as a read error. + +## Other tools + +- **`undefined_keys.py ...`** lists the references in a sample whose key isn't defined anywhere in the same file. Java leaves such a reference's value empty without reporting a failure, so a sample can pass Java validation with data missing. +- **`java/ShowReferences.java`** prints what Java holds, after resolving references, for the `dateRelativeTo` and business center references of a trade state's first interest rate payout. Compile and run it with the classpath `run_java_validation.sh` builds: + + ```sh + CP="/cdm-java/rosetta-source/target/classes:$(cat /cp.txt)" + java -cp "/classes:$CP" ShowReferences + ``` + +## Comparing with Python + +`python-test/cdm-tests/run_fpml510_tests.sh` and `test_fpml510_samples.py` read and round-trip the same samples with the generated Python. Use the same CDM commit for both, then compare per sample: whether it reads, whether the round trip is unchanged, and the validation failures. diff --git a/python-test/cdm-tests/java-validation/java/ShowReferences.java b/python-test/cdm-tests/java-validation/java/ShowReferences.java new file mode 100644 index 00000000..b40255d1 --- /dev/null +++ b/python-test/cdm-tests/java-validation/java/ShowReferences.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +import com.google.inject.*; +import com.regnosys.rosetta.common.hashing.ReferenceResolverProcessStep; +import com.rosetta.model.lib.RosettaModelObject; +import org.finos.cdm.CdmRuntimeModule; +import org.finos.cdm.reference.CdmReferenceConfig; +import org.finos.rune.mapper.RuneJsonObjectMapper; +import java.nio.file.*; +public class ShowReferences { public static void main(String[] a) throws Exception { + Guice.createInjector(new CdmRuntimeModule()); + cdm.event.common.TradeState m=new RuneJsonObjectMapper().readValue(Files.readString(Paths.get(a[0])),cdm.event.common.TradeState.class); + RosettaModelObject b=m.toBuilder(); new ReferenceResolverProcessStep(CdmReferenceConfig.get()).runProcessStep(b.getType(),b); + var t=(cdm.event.common.TradeState)b.build(); + var p=t.getTrade().getProduct().getEconomicTerms().getPayout().get(0).getInterestRatePayout(); + var r=p.getResetDates().getFixingDates().getDateRelativeTo(); + System.out.println("dateRelativeTo: "+r+" | value="+r.getValue()); + var bc=p.getCalculationPeriodDates().getCalculationPeriodDatesAdjustments().getBusinessCenters(); + System.out.println("businessCenters: reference="+bc.getBusinessCentersReference()+" value="+bc.getBusinessCentersReference().getValue()+" businessCenter="+bc.getBusinessCenter()); +}} diff --git a/python-test/cdm-tests/java-validation/java/ValidateSamples.java b/python-test/cdm-tests/java-validation/java/ValidateSamples.java new file mode 100644 index 00000000..76f73acf --- /dev/null +++ b/python-test/cdm-tests/java-validation/java/ValidateSamples.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +import com.fasterxml.jackson.databind.*; +import com.google.inject.*; +import com.regnosys.rosetta.common.hashing.ReferenceResolverProcessStep; +import com.regnosys.rosetta.common.validation.*; +import com.rosetta.model.lib.RosettaModelObject; +import com.rosetta.model.lib.validation.ValidationResult; +import org.finos.cdm.CdmRuntimeModule; +import org.finos.cdm.reference.CdmReferenceConfig; +import org.finos.rune.mapper.RuneJsonObjectMapper; +import java.nio.file.*; import java.io.*; import java.util.*; +/** args: resourcesDir samples.tsv summary.tsv failures.tsv */ +public class ValidateSamples { + static final Comparator NUM = (x, y) -> { + if (x.equals(y)) return 0; + if (x.isNumber() && y.isNumber()) return x.decimalValue().compareTo(y.decimalValue()); + return 1; + }; + public static void main(String[] a) throws Exception { + Path base = Paths.get(a[0]); + Injector inj = Guice.createInjector(new CdmRuntimeModule()); + RosettaTypeValidator val = new RosettaTypeValidator(); inj.injectMembers(val); + ObjectMapper mapper = new RuneJsonObjectMapper(); + ObjectMapper plain = new ObjectMapper(); + try (PrintWriter sum = new PrintWriter(a[2]); PrintWriter fail = new PrintWriter(a[3])) { + sum.println("pack\tsample\texpected\tjava_failures\troundtrip\terror"); + for (String line : Files.readAllLines(Paths.get(a[1]))) { + String[] t = line.split("\t"); + String pack = t[0], f = t[1], expected = t[2]; + Class c = pack.startsWith("workflow-step") + ? cdm.event.workflow.WorkflowStep.class : cdm.event.common.TradeState.class; + String rt = "", fails = "", err = ""; + try { + String json = Files.readString(base.resolve(f)); + RosettaModelObject m = mapper.readValue(json, c); + JsonNode in = plain.readTree(json), out = plain.readTree(mapper.writeValueAsString(m)); + ((com.fasterxml.jackson.databind.node.ObjectNode) in).remove("@version"); + ((com.fasterxml.jackson.databind.node.ObjectNode) out).remove("@version"); + rt = in.equals(NUM, out) ? "same" : "DIFF"; + RosettaModelObject b = m.toBuilder(); + new ReferenceResolverProcessStep(CdmReferenceConfig.get()).runProcessStep(b.getType(), b); + ValidationReport r = val.runProcessStep(c, b.build()); + List> vf = r.validationFailures(); + fails = String.valueOf(vf.size()); + for (ValidationResult v : vf) + fail.println(String.join("\t", pack, f, String.valueOf(v.getValidationType()), v.getModelObjectName(), + v.getName(), String.valueOf(v.getPath()), v.getFailureReason().orElse("").replaceAll("\\s+", " "))); + } catch (Exception e) { + err = e.getClass().getSimpleName() + ": " + String.valueOf(e.getMessage()).replaceAll("\\s+", " "); + if (err.length() > 300) err = err.substring(0, 300); + } + sum.println(String.join("\t", pack, f, expected, fails, rt, err)); + sum.flush(); + } + } + } +} diff --git a/python-test/cdm-tests/java-validation/make_sample_list.py b/python-test/cdm-tests/java-validation/make_sample_list.py new file mode 100644 index 00000000..0f223aa1 --- /dev/null +++ b/python-test/cdm-tests/java-validation/make_sample_list.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +"""Write one row per ingestion sample: pack, outputPath, modelValidationFailures, runtimeError. + +Reads the test-pack configs from a CDM git ref, so the list matches that commit. + + python3 -I make_sample_list.py +""" +import json +import subprocess +import sys + +repo, ref, out = sys.argv[1:4] +base = "rosetta-source/src/main/resources/ingest/config/" + + +def git(*args): + return subprocess.run(["git", "-C", repo, *args], capture_output=True, text=True, check=True).stdout + + +with open(out, "w") as o: + for f in sorted(git("ls-tree", "-r", "--name-only", ref, base).split()): + if "/test-pack-" not in f: + continue + d = json.loads(git("show", f"{ref}:{f}")) + pack = d["id"].replace("test-pack-translate-fpml-confirmation-to-", "") + for s in d["samples"]: + a = s["assertions"] + print(pack, s["outputPath"], a.get("modelValidationFailures"), a.get("runtimeError"), sep="\t", file=o) diff --git a/python-test/cdm-tests/java-validation/run_java_validation.sh b/python-test/cdm-tests/java-validation/run_java_validation.sh new file mode 100755 index 00000000..97f5e883 --- /dev/null +++ b/python-test/cdm-tests/java-validation/run_java_validation.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +# Build CDM Java at a git ref, then read, round-trip and validate every ingestion sample. +# +# Usage (from the project root): +# python-test/cdm-tests/java-validation/run_java_validation.sh +# +# Writes /results/{all-samples,java-summary,java-failures}.tsv +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ $# -ne 3 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi +CDM_REPO="$1"; REF="$2"; WORK="$3" +B="$WORK/cdm-java"; OUT="$WORK/results" +rm -rf "$B"; mkdir -p "$B" "$OUT" "$WORK/classes" +git -C "$CDM_REPO" archive "$REF" | tar -x -C "$B" +(cd "$B" && mvn -q -B -DskipTests -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true \ + -pl rosetta-source -am package > "$WORK/cdm-java-build.log" 2>&1) +(cd "$B" && mvn -q -B -o -pl rosetta-source -am dependency:build-classpath \ + -Dmdep.outputFile="$WORK/cp.txt" -Dmdep.includeScope=test > /dev/null) +CP="$B/rosetta-source/target/classes:$(cat "$WORK/cp.txt")" +python3 -I "$HERE/make_sample_list.py" "$CDM_REPO" "$REF" "$OUT/all-samples.tsv" +javac -nowarn -cp "$CP" -d "$WORK/classes" "$HERE/java/"*.java +java -cp "$WORK/classes:$CP" ValidateSamples "$B/rosetta-source/src/main/resources" \ + "$OUT/all-samples.tsv" "$OUT/java-summary.tsv" "$OUT/java-failures.tsv" < /dev/null 2>&1 | grep -v SLF4J || true +wc -l "$OUT"/*.tsv diff --git a/python-test/cdm-tests/java-validation/undefined_keys.py b/python-test/cdm-tests/java-validation/undefined_keys.py new file mode 100644 index 00000000..ac0c63d2 --- /dev/null +++ b/python-test/cdm-tests/java-validation/undefined_keys.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +"""List references whose key is not defined anywhere in the same sample. + + python3 -I undefined_keys.py [...] +""" +import json +import sys + + +def scan(path): + keys, refs = set(), [] + + def walk(x, p): + if isinstance(x, dict): + for k, v in x.items(): + if k.startswith("@key") and isinstance(v, str): + keys.add(v) + if k.startswith("@ref") and isinstance(v, str): + refs.append((k, v, p)) + walk(v, f"{p}.{k}") + elif isinstance(x, list): + for v in x: + walk(v, p) + + walk(json.load(open(path)), "") + return refs, [r for r in refs if r[1] not in keys] + + +for f in sys.argv[1:]: + refs, undefined = scan(f) + print(f"{f}: {len(refs)} references, {len(undefined)} undefined") + for k, v, p in undefined: + print(f" {k}={v} at {p}") diff --git a/python-test/cdm-tests/profile_import.py b/python-test/cdm-tests/profile_import.py index 1576c9cc..b85e0e7f 100644 --- a/python-test/cdm-tests/profile_import.py +++ b/python-test/cdm-tests/profile_import.py @@ -3,6 +3,7 @@ # Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC # SPDX-License-Identifier: Apache-2.0 # + """ Profile memory usage of importing CDM TradeState. diff --git a/python-test/cdm-tests/run_cdm_tests.sh b/python-test/cdm-tests/run_cdm_tests.sh index 91142723..0cdb9209 100755 --- a/python-test/cdm-tests/run_cdm_tests.sh +++ b/python-test/cdm-tests/run_cdm_tests.sh @@ -3,6 +3,7 @@ # Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC # SPDX-License-Identifier: Apache-2.0 # + # Run the CDM test suite: import smoke test + deserialization ingestion test. # # Usage (from project root): diff --git a/python-test/cdm-tests/run_fpml510_tests.sh b/python-test/cdm-tests/run_fpml510_tests.sh index 32eb46c4..b12af960 100755 --- a/python-test/cdm-tests/run_fpml510_tests.sh +++ b/python-test/cdm-tests/run_fpml510_tests.sh @@ -3,6 +3,7 @@ # Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC # SPDX-License-Identifier: Apache-2.0 # + # Run deserialization tests against all CDM fpml-5-10-products-* samples. # # Usage (from project root): diff --git a/python-test/cdm-tests/run_memory_profile.sh b/python-test/cdm-tests/run_memory_profile.sh index 1e9cbd0b..a2b282b2 100755 --- a/python-test/cdm-tests/run_memory_profile.sh +++ b/python-test/cdm-tests/run_memory_profile.sh @@ -3,6 +3,7 @@ # Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC # SPDX-License-Identifier: Apache-2.0 # + # Profile CDM import memory usage. # # Creates (or reuses) a lightweight venv with the pre-built CDM wheel and diff --git a/python-test/cdm-tests/test_cdm_packaging_issues.py b/python-test/cdm-tests/test_cdm_packaging_issues.py index a1c0eb76..67254f80 100644 --- a/python-test/cdm-tests/test_cdm_packaging_issues.py +++ b/python-test/cdm-tests/test_cdm_packaging_issues.py @@ -5,14 +5,18 @@ ''' Tests for generator/packaging issues found in CDM 7.7.0 (finos-cdm 7.7.0, -rune.runtime 2.2.0, pydantic 2.14.1). Each test documents one defect and its -workaround. These are generator or CDM-packaging issues, not runtime issues. - -G1 Circular import when Asset is the first CDM import -G2 Placeholder type caching: non-bundle class (Instrument) has a stale pydantic - schema for a bundle class (Security) after bundle rebuild -G3 PriceQuantityTriangulation passes a scalar (0..1) quantity to a function - that iterates over it as a list (same root cause as generator defect #300) +rune.runtime 2.2.0, pydantic 2.14.1). G1, G2 and G3 are fixed, and their tests +guard against a regression. G4 documents one open defect. These are generator or +CDM-packaging issues, not runtime issues. + +Each test imports only the CDM modules it uses, as a user would; none imports the +bundle directly. + +G1 Fixed: circular import when Asset is the first CDM import +G2 Fixed: non-bundle class (Instrument) had a stale pydantic schema for a bundle + class (Security) after bundle rebuild +G3 Fixed: PriceQuantityTriangulation iterated over its scalar (0..1) quantity + as if it were a list G4 Now() has no registered native implementation in the CDM package ''' @@ -25,48 +29,33 @@ # G1: Circular import # --------------------------------------------------------------------------- -def test_circular_import_asset_without_bundle_fails(): - '''Importing Asset as the first CDM import raises ImportError. +def test_asset_can_be_imported_first(): + '''Importing Asset as the first CDM import must not raise ImportError. - Root cause: Asset.py imports Instrument.py, which imports - ListedDerivative.py (a lazy-loader that triggers finos._bundle). - _bundle.py at line 4880 does `from ... Asset import Asset` while Asset.py - is still initialising, returning a partially-initialised module. - - Generator fix: _bundle.py should use lazy attribute access for Asset - rather than a module-level direct import. + Regression test. Types on a path between bundled types used to stay standalone, + so _bundle.py imported Asset while Asset.py was still initialising. The generator + now promotes those types into the bundle, so Asset.py is a lazy stub. ''' result = subprocess.run( [sys.executable, '-c', 'from finos.cdm.base.staticdata.asset.common.Asset import Asset'], capture_output=True, text=True ) - assert result.returncode != 0, 'Expected ImportError but import succeeded' - assert 'ImportError' in result.stderr or 'cannot import name' in result.stderr + assert result.returncode == 0, result.stderr # --------------------------------------------------------------------------- # G2: Placeholder type caching # --------------------------------------------------------------------------- -def test_placeholder_type_schema_stale_in_non_bundle_class(): - '''Constructing Instrument(Security=security) raises ValidationError even - after finos._bundle is loaded. - - Root cause: Instrument.py is NOT in _bundle.py; pydantic builds Instrument's - schema when Instrument.py is first loaded. At that point Security (a bundle - class) has type None placeholders, so Instrument's schema embeds the - placeholder definition for Security.identifier. _bundle.py Phase 3 rebuilds - Security, but Instrument's cached schema is never updated. +def test_instrument_accepts_bundled_security(): + '''Constructing Instrument(Security=security) must not raise ValidationError. - Generator fix: after Phase 3 rebuilds, the generator should also call - model_rebuild(force=True) on non-bundle classes (like Instrument) whose - fields reference bundle classes. - - Workaround: the CDM extension rebuilds all classes (bundle + non-bundle) - with model_rebuild(force=True) at startup. + Regression test. Instrument used to be standalone and built its schema while the + bundle was still loading, copying the None placeholder for Security.identifier. + The generator now promotes types on a path between bundled types into the bundle, + so Instrument is rebuilt with the final annotations. ''' - import finos._bundle from finos.cdm.base.staticdata.asset.common.AssetIdentifier import AssetIdentifier from finos.cdm.base.staticdata.asset.common.AssetIdTypeEnum import AssetIdTypeEnum from finos.cdm.base.staticdata.asset.common.AssetTypeEnum import AssetTypeEnum @@ -74,7 +63,6 @@ def test_placeholder_type_schema_stale_in_non_bundle_class(): from finos.cdm.base.staticdata.asset.common.Security import Security from finos.cdm.base.staticdata.asset.common.Instrument import Instrument from rune.runtime.metadata import StrWithMeta - from pydantic import ValidationError asset_id = AssetIdentifier( identifier=StrWithMeta('AAPL'), @@ -85,30 +73,22 @@ def test_placeholder_type_schema_stale_in_non_bundle_class(): assetType=AssetTypeEnum.SECURITY, securityType=SecurityTypeEnum.EQUITY, ) - # Security itself constructs fine (it is a bundle class, rebuilt in Phase 3). - # Passing it to Instrument triggers re-validation against Instrument's stale - # schema, which still sees Security.identifier as type None. - with pytest.raises(ValidationError, match='Input should be None'): - Instrument(Security=security) + instrument = Instrument(Security=security) + assert instrument.Security.securityType == SecurityTypeEnum.EQUITY # --------------------------------------------------------------------------- -# G3: PriceQuantityTriangulation — scalar passed as list (generator defect #300) +# G3: PriceQuantityTriangulation — extract over a scalar # --------------------------------------------------------------------------- -def test_price_quantity_triangulation_fails_with_scalar_quantity(): - '''PriceQuantityTriangulation iterates over quantity as a list, but CDM's - PriceQuantity.quantity is a scalar (0..1) field. Passing a single - NonNegativeQuantitySchedule raises TypeError. - - Root cause: the generator wraps the function call in `map(lambda item: ..., - rune_resolve_attr(self, "quantity") or [])`, iterating over the scalar value. - Inside rune_cow, this becomes a _COWObject, which is not iterable. +def test_price_quantity_triangulation_accepts_scalar_quantity(): + '''PriceQuantityTriangulation must accept a single NonNegativeQuantitySchedule. - This is the same defect as generator issue #300 (list-wrapping for scalar - constructor fields), manifesting in CDM's own generated code. + Regression test. CDM's PriceQuantity.quantity is a scalar (0..1) field, but the + generated `quantity extract ...` used to iterate over it as a list. Inside rune_cow + the value is a _COWObject, so the function raised TypeError. The generator now + applies extract to a single value directly. ''' - import finos._bundle from finos.cdm.observable.common.functions.PriceQuantityTriangulation import ( PriceQuantityTriangulation, ) @@ -118,8 +98,8 @@ def test_price_quantity_triangulation_fails_with_scalar_quantity(): # Use model_construct to bypass runtime issue R5 (absent-metadata validator) qty = NonNegativeQuantitySchedule.model_construct(value=Decimal('1000')) - with pytest.raises(TypeError, match="'_COWObject' object is not iterable"): - PriceQuantityTriangulation(price=None, quantity=qty) + # With no prices there is nothing to check, so the triangulation holds + assert PriceQuantityTriangulation(price=None, quantity=qty) is True # --------------------------------------------------------------------------- @@ -137,7 +117,6 @@ def test_now_function_raises_not_implemented(): the registration mechanism; the CDM maintainers need to supply the implementation. ''' - import finos._bundle from finos.cdm.base.datetime.functions.Now import Now with pytest.raises(NotImplementedError, match="doesn't have an implementation"): diff --git a/python-test/cdm-tests/test_deserialize_trade_state.py b/python-test/cdm-tests/test_deserialize_trade_state.py index cbea8861..ae734aaa 100644 --- a/python-test/cdm-tests/test_deserialize_trade_state.py +++ b/python-test/cdm-tests/test_deserialize_trade_state.py @@ -1,3 +1,8 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + import logging import os import pytest diff --git a/python-test/cdm-tests/test_fpml510_samples.py b/python-test/cdm-tests/test_fpml510_samples.py index 16feca06..5759b797 100644 --- a/python-test/cdm-tests/test_fpml510_samples.py +++ b/python-test/cdm-tests/test_fpml510_samples.py @@ -1,5 +1,13 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +import json import logging import os +import re +from decimal import Decimal, InvalidOperation from pathlib import Path import pytest from pydantic import BaseModel, ValidationError @@ -49,3 +57,118 @@ def test_deserialize_fpml510_sample(path: str): pytest.fail(f"ValidationError: {e}") except Exception as e: pytest.fail(f"{type(e).__name__}: {e}") + + +# --------------------------------------------------------------------------- +# Round trip: read each sample, write it back and compare with the original +# --------------------------------------------------------------------------- + +# Written by Python from its own package version, so it is expected to differ +_IGNORED_ROOT_FIELDS = ('@version',) +_MAX_REPORTED_DIFFERENCES = 10 +_LABELS = dict(_SAMPLES) + + +def _load_json(text: str): + # Decimal keeps precise numbers exact on both sides of the comparison + document = json.loads(text, parse_float=Decimal) + for field in _IGNORED_ROOT_FIELDS: + document.pop(field, None) + return document + + +def _value_kind(value) -> str: + if isinstance(value, bool): + return 'bool' + if isinstance(value, (int, Decimal)): + return 'number' + return type(value).__name__ + + +def _same_number(expected, actual) -> bool: + '''True for a number and a string holding the same value. Python writes + numbers as strings and Java as numbers; both read back as the same value.''' + if {_value_kind(expected), _value_kind(actual)} != {'number', 'str'}: + return False + try: + return Decimal(str(expected)) == Decimal(str(actual)) + except InvalidOperation: + return False + + +def _compare(expected, actual, path: str = '') -> list[tuple[str, str, object, object]]: + '''Returns (kind, path, expected, actual) for every difference.''' + differences = [] + if isinstance(expected, dict) and isinstance(actual, dict): + for key in sorted(set(expected) | set(actual)): + child = f'{path}.{key}' if path else key + if key not in actual: + differences.append(('missing in output', child, expected[key], None)) + elif key not in expected: + differences.append(('extra in output', child, None, actual[key])) + else: + differences.extend(_compare(expected[key], actual[key], child)) + elif isinstance(expected, list) and isinstance(actual, list): + if len(expected) != len(actual): + differences.append(('list length changed', path, len(expected), len(actual))) + for i, (item_expected, item_actual) in enumerate(zip(expected, actual)): + differences.extend(_compare(item_expected, item_actual, f'{path}[{i}]')) + elif _same_number(expected, actual): + pass + elif _value_kind(expected) != _value_kind(actual): + kind = f'type changed ({_value_kind(expected)} -> {_value_kind(actual)})' + differences.append((kind, path, expected, actual)) + elif expected != actual: + differences.append(('value changed', path, expected, actual)) + return differences + + +def _field_name(path: str) -> str: + '''The last field in a path, without list indices, for grouping.''' + return re.sub(r'\[\d+\]', '', path).rsplit('.', 1)[-1] or '(root)' + + +def _describe_error(error: Exception) -> str: + '''A short form of the error for grouping, without values that vary.''' + if isinstance(error, ValidationError): + error_types = sorted({detail['type'] for detail in error.errors()}) + return f"ValidationError ({', '.join(error_types)})" + message = re.sub(r'\s+', ' ', str(error)) + # Keep the underlying error that serialization wraps + message = re.sub(r'^.*Error calling function `\w+`: ', '', message) + message = re.sub(r'^(Error serializing to JSON: )?(PydanticSerializationError: )?', '', message) + message = re.sub(r', got .*$', '', message) # the value received varies, and may be long + message = re.sub(r"'[^']*'|", "'...'", message) + if not message.startswith(type(error).__name__) and not re.match(r'^\w+Error: ', message): + message = f'{type(error).__name__}: {message}' + return message[:80] + + +@pytest.mark.parametrize("path", _PARAMS, ids=_IDS) +def test_roundtrip_fpml510_sample(path: str, roundtrip_results: dict): + """Read each sample, write it back, and check the JSON is unchanged.""" + if path is None: + pytest.skip("CDM_FPML510_SAMPLES_DIR not set") + label = _LABELS[path] + with open(path, "r", encoding="utf-8") as f: + json_in = f.read() + + try: + obj = finos.rune_deserialize(json_in, validate_model=False) + except Exception as e: + roundtrip_results[label] = [(f'read failed: {_describe_error(e)}', label)] + pytest.fail(f"read failed: {type(e).__name__}: {e}") + + try: + json_out = obj.rune_serialize(validate_model=False) + except Exception as e: + roundtrip_results[label] = [(f'write failed: {_describe_error(e)}', label)] + pytest.fail(f"write failed: {type(e).__name__}: {e}") + + differences = _compare(_load_json(json_in), _load_json(json_out)) + roundtrip_results[label] = [(kind, _field_name(where)) for kind, where, _, _ in differences] + if differences: + shown = '\n'.join(f' {kind}: {where}: {expected!r} -> {actual!r}' + for kind, where, expected, actual + in differences[:_MAX_REPORTED_DIFFERENCES]) + pytest.fail(f"{len(differences)} difference(s) after round trip:\n{shown}") diff --git a/python-test/serialization-tests/error-handling/README.md b/python-test/serialization-tests/error-handling/README.md new file mode 100644 index 00000000..955c9975 --- /dev/null +++ b/python-test/serialization-tests/error-handling/README.md @@ -0,0 +1,32 @@ +# Serialization error-handling cases + +These models and JSON files are copied from +`serialization/src/test/resources/rune-serializer-error-handling-test` in +[finos/rune-common](https://github.com/finos/rune-common), at commit +`cb9c389438cbfd6f78ce38bd4a7d48823c20f42d`. + +One change was made: `metalocation/meta-location.rosetta` declared namespace +`serialization.failing.metalocation`, while its JSON, the other cases and the +removed Java test all use `serialization.test.failing`. The namespace is +corrected to `serialization.test.failing.metalocation`, so the case is rejected +for its dangling address rather than because its type can't be found. + +rune-common removed the suite in `cec660462` ("Disable polymorphic +deserialisation of choice options", #605), along with its Java test, which was +disabled. The cases are kept here because the behaviour they check still +applies to the Python runtime: + +| Folder | Case | Expected | +|---|---|---| +| `cardinality` | `too-many-elements` | More elements than the attribute allows is rejected | +| `metakey` | `dangling-attribute-ref`, `dangling-node-ref` | A reference to a key that isn't defined is rejected | +| `metalocation` | `dangling-address` | An address with no matching location is rejected | +| `parameterised` | `number-too-large`, `number-fraction-too-large` | A number with more digits than its type allows is rejected | +| `parameterised` | `string-too-large`, `string-illegal-pattern` | A string longer than its type allows, or not matching its pattern, is rejected | + +`serialization_test.py` runs each JSON file in `test_should_fail_files`. A case +passes when reading or writing it raises an error, or when the written JSON +differs from the input. + +rune-common is licensed under the Apache License, Version 2.0, as is this +project. diff --git a/python-test/serialization-tests/error-handling/cardinality/cardinality.rosetta b/python-test/serialization-tests/error-handling/cardinality/cardinality.rosetta new file mode 100644 index 00000000..c9d9f6f4 --- /dev/null +++ b/python-test/serialization-tests/error-handling/cardinality/cardinality.rosetta @@ -0,0 +1,7 @@ +namespace serialization.test.failing.cardinality + +annotation rootType: <"Mark a type as a root of the rosetta model"> + +type Root: + [rootType] + stringList string (0..2) diff --git a/python-test/serialization-tests/error-handling/cardinality/too-many-elements.json b/python-test/serialization-tests/error-handling/cardinality/too-many-elements.json new file mode 100644 index 00000000..c213443e --- /dev/null +++ b/python-test/serialization-tests/error-handling/cardinality/too-many-elements.json @@ -0,0 +1,6 @@ +{ + "@model": "serialization", + "@type": "serialization.test.failing.cardinality.Root", + "@version": "0.0.0", + "stringList": ["foo", "bar", "baz"] +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/metakey/dangling-attribute-ref.json b/python-test/serialization-tests/error-handling/metakey/dangling-attribute-ref.json new file mode 100644 index 00000000..d65ec629 --- /dev/null +++ b/python-test/serialization-tests/error-handling/metakey/dangling-attribute-ref.json @@ -0,0 +1,11 @@ +{ + "@model" : "serialization", + "@type" : "serialization.test.failing.metakey.Root", + "@version" : "0.0.0", + "attributeRef" : { + "dateReference" : { + "@ref" : "someKey", + "@ref:external" : "someExternalKey" + } + } +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/metakey/dangling-node-ref.json b/python-test/serialization-tests/error-handling/metakey/dangling-node-ref.json new file mode 100644 index 00000000..892f209c --- /dev/null +++ b/python-test/serialization-tests/error-handling/metakey/dangling-node-ref.json @@ -0,0 +1,11 @@ +{ + "@model" : "serialization", + "@type" : "serialization.test.failing.metakey.Root", + "@version" : "0.0.0", + "nodeRef" : { + "aReference" : { + "@ref" : "someKey", + "@ref:external" : "someExternalKey" + } + } +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/metakey/meta-key.rosetta b/python-test/serialization-tests/error-handling/metakey/meta-key.rosetta new file mode 100644 index 00000000..34de6d85 --- /dev/null +++ b/python-test/serialization-tests/error-handling/metakey/meta-key.rosetta @@ -0,0 +1,23 @@ +namespace serialization.test.failing.metakey + +annotation rootType: <"Mark a type as a root of the rosetta model"> + +type A: + [metadata key] + fieldA string (1..1) + +type NodeRef: + typeA A (0..1) + aReference A (0..1) + [metadata reference] + +type AttributeRef: + dateField date (0..1) + [metadata id] + dateReference date (0..1) + [metadata reference] + +type Root: + [rootType] + nodeRef NodeRef (0..1) + attributeRef AttributeRef (0..1) diff --git a/python-test/serialization-tests/error-handling/metalocation/dangling-address.json b/python-test/serialization-tests/error-handling/metalocation/dangling-address.json new file mode 100644 index 00000000..b1e0f790 --- /dev/null +++ b/python-test/serialization-tests/error-handling/metalocation/dangling-address.json @@ -0,0 +1,8 @@ +{ + "@model" : "serialization", + "@type" : "serialization.test.failing.metalocation.Root", + "@version" : "0.0.0", + "bAddress" : { + "@ref:scoped" : "someLocation" + } +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/metalocation/meta-location.rosetta b/python-test/serialization-tests/error-handling/metalocation/meta-location.rosetta new file mode 100644 index 00000000..491ac753 --- /dev/null +++ b/python-test/serialization-tests/error-handling/metalocation/meta-location.rosetta @@ -0,0 +1,16 @@ +namespace serialization.test.failing.metalocation + +annotation rootType: <"Mark a type as a root of the rosetta model"> + +type A: + b B (1..1) + [metadata location] + +type B: + fieldB string (1..1) + +type Root: + [rootType] + typeA A (0..1) + bAddress B (0..1) + [metadata address "pointsTo"=A->b] diff --git a/python-test/serialization-tests/error-handling/parameterised/number-fraction-too-large.json b/python-test/serialization-tests/error-handling/parameterised/number-fraction-too-large.json new file mode 100644 index 00000000..d4231dfb --- /dev/null +++ b/python-test/serialization-tests/error-handling/parameterised/number-fraction-too-large.json @@ -0,0 +1,6 @@ +{ + "@model": "serialization", + "@type": "serialization.test.failing.parameterised.Root", + "@version": "0.0.0", + "parameterisedNumber": 1.23456789 +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/parameterised/number-too-large.json b/python-test/serialization-tests/error-handling/parameterised/number-too-large.json new file mode 100644 index 00000000..e2eba011 --- /dev/null +++ b/python-test/serialization-tests/error-handling/parameterised/number-too-large.json @@ -0,0 +1,6 @@ +{ + "@model": "serialization", + "@type": "serialization.test.failing.parameterised.Root", + "@version": "0.0.0", + "parameterisedNumber": 123456789 +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/parameterised/parameterised.rosetta b/python-test/serialization-tests/error-handling/parameterised/parameterised.rosetta new file mode 100644 index 00000000..82de581c --- /dev/null +++ b/python-test/serialization-tests/error-handling/parameterised/parameterised.rosetta @@ -0,0 +1,15 @@ +namespace serialization.test.failing.parameterised + +annotation rootType: <"Mark a type as a root of the rosetta model"> + +typeAlias ParameterisedNumberType: + number(digits: 3, fractionalDigits: 2) + +typeAlias ParameterisedStringType: + string(minLength: 1, maxLength: 2, pattern: "[a-zA-Z]") + +type Root: + [rootType] + parameterisedNumber ParameterisedNumberType (0..1) + parameterisedString ParameterisedStringType (0..1) + \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/parameterised/string-illegal-pattern.json b/python-test/serialization-tests/error-handling/parameterised/string-illegal-pattern.json new file mode 100644 index 00000000..e19171ea --- /dev/null +++ b/python-test/serialization-tests/error-handling/parameterised/string-illegal-pattern.json @@ -0,0 +1,6 @@ +{ + "@model": "serialization", + "@type": "serialization.test.failing.parameterised.Root", + "@version": "0.0.0", + "parameterisedString": "$$" +} \ No newline at end of file diff --git a/python-test/serialization-tests/error-handling/parameterised/string-too-large.json b/python-test/serialization-tests/error-handling/parameterised/string-too-large.json new file mode 100644 index 00000000..6f07f90e --- /dev/null +++ b/python-test/serialization-tests/error-handling/parameterised/string-too-large.json @@ -0,0 +1,6 @@ +{ + "@model": "serialization", + "@type": "serialization.test.failing.parameterised.Root", + "@version": "0.0.0", + "parameterisedString": "abcdefg" +} \ No newline at end of file diff --git a/python-test/serialization-tests/run_serialization_tests.sh b/python-test/serialization-tests/run_serialization_tests.sh index bf10decc..fa82389f 100755 --- a/python-test/serialization-tests/run_serialization_tests.sh +++ b/python-test/serialization-tests/run_serialization_tests.sh @@ -9,7 +9,7 @@ function error echo echo "***************************************************************************" echo "* *" - echo "* DEV ENV Initialization FAILED! *" + echo "* Testing failed! *" echo "* *" echo "***************************************************************************" echo @@ -60,6 +60,17 @@ PYTHON_SETUP_PATH="$MY_PATH/../env-setup" JAR_PATH="$PROJECT_ROOT_PATH/target/python-0.0.0.main-SNAPSHOT.jar" SERIALIZATION_SOURCE_ROOT="$MY_PATH/rune-common/serialization/src/test/resources" +# Models to generate, all from different namespaces: +# - the round-trip suite from rune-common; +# - the error-handling cases, kept in this repo since rune-common removed them; +# - the invalid choice option cases from rune-common's choice deserializer suite. +# Other suites in rune-common reuse the round-trip namespaces, so they're left out: +# generating them together rejects those models as duplicates. +ROUND_TRIP_SOURCE_DIR="$SERIALIZATION_SOURCE_ROOT/rune-serializer-round-trip-test" +ERROR_HANDLING_SOURCE_DIR="$MY_PATH/error-handling" +CHOICE_DESERIALIZER_SOURCE_DIR="$SERIALIZATION_SOURCE_ROOT/rune-json-choice-type-deserializer-test" +INVALID_CHOICE_GROUPS=("invalidchoiceoption" "invalidnestedchoiceoption") +GENERATION_SOURCE_DIR="$PROJECT_ROOT_PATH/target/python-tests/serialization_sources" PYTHON_TESTS_TARGET_PATH="$PROJECT_ROOT_PATH/target/python-tests/serialization_unit_tests" PYTHON_PROJECT_NAME="serialization_unit_tests" @@ -68,7 +79,7 @@ if [[ $DO_GET_SERIALIZATION_SOURCE -eq 1 ]]; then echo "***** fetching serialization sources (get_serialization_source.sh)" source "$MY_PATH/get_serialization_source.sh" || error else - echo "***** skipping fetch of serialization sources (pass 'getsource' to fetch)" + echo "***** skipping fetch of serialization sources (pass '--getsource' to fetch)" fi # Validate inputs/existence @@ -84,17 +95,52 @@ if [[ ! -f "$JAR_PATH" ]]; then exit 1 fi fi -if [[ ! -d "$SERIALIZATION_SOURCE_ROOT" ]]; then - echo "Serialization sources not found at: $SERIALIZATION_SOURCE_ROOT" - echo "Run with 'getsource' or ensure the path exists." - exit 1 -fi +for SOURCE_DIR in "$ROUND_TRIP_SOURCE_DIR" "$CHOICE_DESERIALIZER_SOURCE_DIR" "$ERROR_HANDLING_SOURCE_DIR"; do + if [[ ! -d "$SOURCE_DIR" ]]; then + echo "Serialization sources not found at: $SOURCE_DIR" + echo "Run with '--getsource' or ensure the path exists." + exit 1 + fi +done + +# The generator reads one source directory, so stage the models together +echo "***** staging serialization models" +rm -rf "$GENERATION_SOURCE_DIR" +mkdir -p "$GENERATION_SOURCE_DIR" +cp -R "$ROUND_TRIP_SOURCE_DIR" "$GENERATION_SOURCE_DIR/round-trip" || error +cp -R "$ERROR_HANDLING_SOURCE_DIR" "$GENERATION_SOURCE_DIR/error-handling" || error +for GROUP in "${INVALID_CHOICE_GROUPS[@]}"; do + cp -R "$CHOICE_DESERIALIZER_SOURCE_DIR/$GROUP" "$GENERATION_SOURCE_DIR/$GROUP" || error +done + +# ChoiceDeepNested in rune-common's choicetype model reaches the same leaves through two options. +# The DSL warns about that and will reject it; the generator already rejects it. Leave it out of +# the staged copy until rune-common changes the model (https://github.com/finos/rune-dsl/issues/1464); +# serialization_test.py skips its fixtures. +echo "***** removing ChoiceDeepNested from the staged choicetype model" +python3 - "$GENERATION_SOURCE_DIR/round-trip/choicetype/choice.rosetta" <<'EOF' || error +import re +import sys + +path = sys.argv[1] +text = open(path, encoding='utf-8').read() +for name in ('MiddleChoiceA', 'MiddleChoiceB', 'ChoiceDeepNested'): + text, count = re.subn(rf'^choice {name}:\n(?:[ \t]+\S.*\n)*\n?', '', text, flags=re.M) + if count != 1: + sys.exit(f'{path}: choice {name} not found; check whether this step is still needed') +text, count = re.subn(r'^[ \t]+choiceDeepNested ChoiceDeepNested .*\n', '', text, flags=re.M) +if count != 1: + sys.exit(f'{path}: attribute choiceDeepNested not found; check whether this step is still needed') +open(path, 'w', encoding='utf-8').write(text) +EOF # Generate Python serialization tests using the CLI echo "***** generating Python serialization tests" +# Clear earlier output so modules from previous runs aren't packaged +rm -rf "$PYTHON_TESTS_TARGET_PATH" mkdir -p "$PYTHON_TESTS_TARGET_PATH" java -cp "$JAR_PATH" com.regnosys.rosetta.generator.python.PythonCodeGeneratorCLI \ - -s "$SERIALIZATION_SOURCE_ROOT" \ + -s "$GENERATION_SOURCE_DIR" \ -t "$PYTHON_TESTS_TARGET_PATH" \ -p "$PYTHON_PROJECT_NAME" JAVA_EXIT_CODE=$? @@ -128,7 +174,13 @@ if [[ -z "$WHEEL_FILE" ]]; then source "$PYTHON_SETUP_PATH/cleanup_python_env.sh" exit 1 fi -python -m pip install "$WHEEL_FILE" || error +# With a local or GitHub runtime (see setup_python_env.sh), skip dependencies so pip keeps it +# instead of replacing it with a release that satisfies the generated pin +if [ -n "$RUNE_RUNTIME_DIR" ] || [ -n "$RUNE_RUNTIME_REF" ]; then + python -m pip install --no-deps "$WHEEL_FILE" || error +else + python -m pip install "$WHEEL_FILE" || error +fi # run tests echo "***** run tests" diff --git a/python-test/serialization-tests/serialization_test.py b/python-test/serialization-tests/serialization_test.py index 74c3b393..d192aaa9 100644 --- a/python-test/serialization-tests/serialization_test.py +++ b/python-test/serialization-tests/serialization_test.py @@ -17,7 +17,24 @@ from test_helper.test_helper.dict_comp import dict_comp ROUNDTRIP_DIR = 'rune-common/serialization/src/test/resources/rune-serializer-round-trip-test' -SHOULDFAIL_DIR = 'rune-common/serialization/src/test/resources/rune-serializer-error-handling-test' +# ChoiceDeepNested in rune-common's choicetype model reaches the same leaves through two +# options. The DSL warns about that and will reject it; the generator already rejects it, so +# run_serialization_tests.sh leaves it out of the generated model. Skip the fixtures that use +# it until rune-common changes the model (https://github.com/finos/rune-dsl/issues/1464). +SKIPPED_ROUND_TRIP_FILES = { + 'choicetype/choice-deep-nested.json', + 'choicetype/choice-deep-nested-2.json', + 'choicetype/choice-deep-nested-extended.json', +} +# Cases that must fail: the error-handling cases kept in this repo (rune-common +# removed them), and the invalid choice options from rune-common's choice +# deserializer suite +CHOICE_DESERIALIZER_DIR = 'rune-common/serialization/src/test/resources/rune-json-choice-type-deserializer-test' +SHOULDFAIL_DIRS = [ + 'error-handling', + CHOICE_DESERIALIZER_DIR + '/invalidchoiceoption', + CHOICE_DESERIALIZER_DIR + '/invalidnestedchoiceoption', +] def extract_dir_and_file(path_name) -> str: @@ -81,9 +98,18 @@ def process_directory(dir_name): 'failures:', failures) -@pytest.mark.parametrize( - "round_trip_file", - glob.glob(ROUNDTRIP_DIR + os.sep + '**/*.json', recursive=True)) +def _round_trip_files() -> list[str]: + '''the round-trip fixtures, without the skipped ones; fails if a skipped one is missing''' + files = glob.glob(ROUNDTRIP_DIR + os.sep + '**/*.json', recursive=True) + found = {extract_dir_and_file(f) for f in files} + missing = SKIPPED_ROUND_TRIP_FILES - found + if missing and files: + raise RuntimeError(f'skipped round-trip fixtures not found: {sorted(missing)}; ' + 'check whether SKIPPED_ROUND_TRIP_FILES is still needed') + return [f for f in files if extract_dir_and_file(f) not in SKIPPED_ROUND_TRIP_FILES] + + +@pytest.mark.parametrize("round_trip_file", _round_trip_files()) def test_round_trip_files(round_trip_file): '''test round trip files''' try: @@ -93,7 +119,9 @@ def test_round_trip_files(round_trip_file): @pytest.mark.parametrize( "should_fail_file", - glob.glob(SHOULDFAIL_DIR + os.sep + '**/*.json', recursive=True)) + sorted(path_name + for dir_name in SHOULDFAIL_DIRS + for path_name in glob.glob(dir_name + os.sep + '**/*.json', recursive=True))) def test_should_fail_files(should_fail_file): '''test should fail files''' try: diff --git a/python-test/unit-tests/features/expressions/AsInCondition.rosetta b/python-test/unit-tests/features/expressions/AsInCondition.rosetta new file mode 100644 index 00000000..203d9770 --- /dev/null +++ b/python-test/unit-tests/features/expressions/AsInCondition.rosetta @@ -0,0 +1,36 @@ +namespace rosetta_dsl.test.expressions.as_in_condition : <"generate Python unit tests for the as operator in conditions."> + +// DogB and OwnerB refer to each other, so they are bundled, and so is their supertype Pet. +// Shape and Circle are standalone. Checker is a standalone class and OwnerB a bundled one; +// each narrows to a bundled type and to a standalone one. + +type Pet: + +type DogB extends Pet: + owner OwnerB (0..1) + +type OwnerB: + dog DogB (0..1) + pet Pet (0..1) + shape Shape (0..1) + + condition PetIsDog: + if pet exists then (pet as DogB) exists + + condition ShapeIsCircle: + if shape exists then (shape as Circle) exists + +type Shape: + +type Circle extends Shape: + radius number (0..1) + +type Checker: + pet Pet (0..1) + shape Shape (0..1) + + condition PetIsDog: + if pet exists then (pet as DogB) exists + + condition ShapeIsCircle: + if shape exists then (shape as Circle) exists diff --git a/python-test/unit-tests/features/expressions/test_as_in_condition.py b/python-test/unit-tests/features/expressions/test_as_in_condition.py new file mode 100644 index 00000000..547ba2b7 --- /dev/null +++ b/python-test/unit-tests/features/expressions/test_as_in_condition.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''conditions that use `as` to narrow to a data type work in standalone and bundled classes''' +import pytest + +from rosetta_dsl.test.expressions.as_in_condition.Checker import Checker +from rosetta_dsl.test.expressions.as_in_condition.Circle import Circle +from rosetta_dsl.test.expressions.as_in_condition.DogB import DogB +from rosetta_dsl.test.expressions.as_in_condition.OwnerB import OwnerB +from rosetta_dsl.test.expressions.as_in_condition.Pet import Pet +from rosetta_dsl.test.expressions.as_in_condition.Shape import Shape + +# Checker is standalone and OwnerB is bundled. DogB is bundled and Circle is standalone. +OWNERS = [Checker, OwnerB] + + +@pytest.mark.parametrize('owner', OWNERS) +@pytest.mark.parametrize('attr, value', [('pet', DogB), ('shape', Circle)]) +def test_condition_passes_when_value_has_target_type(owner, attr, value): + '''narrowing finds a value of the target type. + + Fails until the runtime keeps a subtype stored in a parent-typed attribute, rather than + turning it into the parent type.''' + assert not owner(**{attr: value()}).validate_conditions(raise_exc=False) + + +@pytest.mark.parametrize('owner', OWNERS) +@pytest.mark.parametrize('attr, value', [('pet', Pet), ('shape', Shape)]) +def test_condition_fails_when_value_lacks_target_type(owner, attr, value): + '''narrowing gives nothing for a value that isn't of the target type''' + assert owner(**{attr: value()}).validate_conditions(raise_exc=False) + + +@pytest.mark.parametrize('owner', OWNERS) +def test_condition_passes_when_value_absent(owner): + '''the conditions only apply when the attribute is set''' + assert not owner().validate_conditions(raise_exc=False) diff --git a/python-test/unit-tests/features/functions/BundledTypeNameTest.rosetta b/python-test/unit-tests/features/functions/BundledTypeNameTest.rosetta new file mode 100644 index 00000000..b2732d95 --- /dev/null +++ b/python-test/unit-tests/features/functions/BundledTypeNameTest.rosetta @@ -0,0 +1,63 @@ +namespace rosetta_dsl.test.functions.bundled_type_name : <"generate Python unit tests from Rosetta."> + +// GroupB1 and GroupB2 form a cycle, so both are bundled. Standalone functions and classes +// build them, and a bundled condition builds a standalone type. + +type GroupB1: + value number (0..1) + groupB2 GroupB2 (0..1) + +type GroupB2: + groupB1 GroupB1 (0..1) + + condition BuildsStandaloneType: <"A bundled condition building a standalone type."> + Leaf { value: 1 } -> value = 1 + +type Leaf: <"Standalone, used only by a bundled condition."> + value number (0..1) + +type Mid: <"Standalone, holds a bundled type."> + groupB1 GroupB1 (0..1) + +type Checker: <"Standalone, its condition builds a bundled type."> + value number (0..1) + + condition BuildsBundledType: + if value exists + then GroupB1 { value: value, ... } -> value = value + +func MakeGroupB1: <"Builds a bundled type with a constructor."> + inputs: + value number (0..1) + output: + result GroupB1 (0..1) + + set result: + GroupB1 { value: value, ... } + +func SetPathGroupB1: <"Builds a bundled type with an ObjectBuilder."> + inputs: + value number (0..1) + output: + result GroupB1 (0..1) + + set result -> value: + value + +func MakeMid: <"Builds a bundled type that appears only in the function body."> + inputs: + value number (0..1) + output: + result Mid (0..1) + + set result: + Mid { groupB1: GroupB1 { value: value, ... } } + +func ValueViaGroupB1: <"Builds a bundled type that appears in neither the signature nor any other type."> + inputs: + value number (0..1) + output: + result number (0..1) + + set result: + GroupB1 { value: value, ... } -> value diff --git a/python-test/unit-tests/features/functions/ConditionFunctionImportTest.rosetta b/python-test/unit-tests/features/functions/ConditionFunctionImportTest.rosetta new file mode 100644 index 00000000..6b2eb26c --- /dev/null +++ b/python-test/unit-tests/features/functions/ConditionFunctionImportTest.rosetta @@ -0,0 +1,35 @@ +namespace rosetta_dsl.test.functions.condition_function_import : <"generate Python unit tests from Rosetta."> + +// GroupB1 and GroupB2 form a cycle, so both are bundled. Conditions on bundled and +// standalone classes call functions that use GroupB1. + +type GroupB1: + value number (0..1) + groupB2 GroupB2 (0..1) + + condition HasItsValue: <"A bundled class whose condition calls a function."> + HasValue(item) + +type GroupB2: + groupB1 GroupB1 (0..1) + +type WithBundledAttribute: <"Standalone, with a bundled attribute, calling a function directly."> + groupB1 GroupB1 (0..1) + + condition UsesFunction: + HasValue(groupB1) + +type WithoutBundledAttribute: <"Standalone, with no bundled attribute; its function uses a bundled type."> + value number (0..1) + + condition UsesFunction: + HasValue(empty) = False + +func HasValue: + inputs: + groupB1 GroupB1 (0..1) + output: + result boolean (1..1) + + set result: + groupB1 -> value exists diff --git a/python-test/unit-tests/features/functions/CountSingleValueTest.rosetta b/python-test/unit-tests/features/functions/CountSingleValueTest.rosetta new file mode 100644 index 00000000..f59ebc92 --- /dev/null +++ b/python-test/unit-tests/features/functions/CountSingleValueTest.rosetta @@ -0,0 +1,50 @@ +namespace rosetta_dsl.test.functions.count_single_value : <"generate Python unit tests from Rosetta."> + +type Inner: + value number (1..1) + name string (0..1) + +type Holder: + lone Inner (0..1) + several Inner (0..*) + + condition AtMostOneSingle: <"count of a single model object is 1, not its number of fields."> + lone count <= 1 + + condition CountInCondition: <"count must agree with the cardinality in a calculation."> + if lone exists + then lone count = 1 + else lone count = 0 + + condition CountOfSeveral: + several count <= 2 + +type Parent: + inner Inner (0..1) + +func CountSingle: <"The input is a proxy for the model object inside a function."> + inputs: + lone Inner (0..1) + output: + result int (1..1) + + set result: + lone count + +func CountSingleAttribute: <"count of a path that ends in a single model object."> + inputs: + parent Parent (0..1) + output: + result int (1..1) + + set result: + parent -> inner count + +func CountSeveral: <"Control: count of a list."> + inputs: + several Inner (0..*) + output: + result int (1..1) + + set result: + several count diff --git a/python-test/unit-tests/features/functions/FilterSingleValueTest.rosetta b/python-test/unit-tests/features/functions/FilterSingleValueTest.rosetta new file mode 100644 index 00000000..4536ccdb --- /dev/null +++ b/python-test/unit-tests/features/functions/FilterSingleValueTest.rosetta @@ -0,0 +1,30 @@ +namespace rosetta_dsl.test.functions.filter_single_value : <"generate Python unit tests from Rosetta."> + +type Leaf: + value number (0..1) + +type Holder: + lone Leaf (0..1) + + condition FilteredCountMatches: <"Filtering a single value keeps it when the test holds."> + if lone -> value exists + then (lone filter value exists then count) = 1 + else (lone filter value exists then count) = 0 + +func FilterSingle: <"A filter on a single value gives that value or nothing."> + inputs: + lone Leaf (0..1) + output: + result Leaf (0..1) + + set result: + lone filter value exists + +func FilterSingleThenCount: + inputs: + lone Leaf (0..1) + output: + result int (1..1) + + set result: + lone filter value exists then count diff --git a/python-test/unit-tests/features/functions/FlattenAndSetOutputTest.rosetta b/python-test/unit-tests/features/functions/FlattenAndSetOutputTest.rosetta new file mode 100644 index 00000000..4ef7d35d --- /dev/null +++ b/python-test/unit-tests/features/functions/FlattenAndSetOutputTest.rosetta @@ -0,0 +1,144 @@ +namespace rosetta_dsl.test.functions.flatten_set_output : <"generate Python unit tests from Rosetta."> + +type Leaf: + value number (1..1) + name string (0..1) + +type Branch: + leaf Leaf (0..1) + leaves Leaf (0..*) + tags string (0..*) + +type Tree: + branches Branch (0..*) + + condition AtMostTwoLeaves: <"flatten of model lists inside a condition."> + branches extract leaves then flatten then count <= 2 + +type TreeOfSetOutputs: + branches Branch (0..*) + + condition AtMostOneLeaf: <"flatten of a function's multi-valued output inside a condition."> + branches extract SetFromSingle(leaf) then flatten then count <= 1 + +func FlattenModelLists: + inputs: + branches Branch (0..*) + output: + result Leaf (0..*) + + add result: + branches extract leaves then flatten + +func FlattenStringLists: <"Strings have __iter__ but must stay whole."> + inputs: + branches Branch (0..*) + output: + result string (0..*) + + add result: + branches extract tags then flatten + +func FlattenConditionalLists: + inputs: + branches Branch (0..*) + output: + result Leaf (0..*) + + add result: + branches + extract + if leaves exists + then leaves + else empty + then flatten + +func AddFromSingle: <"Control: a multi-valued output filled with add."> + inputs: + leaf Leaf (0..1) + output: + result Leaf (0..*) + + add result: + leaf + +func SetFromSingle: <"A multi-valued output filled with set from a single model object."> + inputs: + leaf Leaf (0..1) + output: + result Leaf (0..*) + + set result: + leaf + +func SetFromSingleString: <"A multi-valued output filled with set from a single basic value."> + inputs: + name string (0..1) + output: + result string (0..*) + + set result: + name + +func SetFromList: <"Control: a multi-valued output filled with set from a list."> + inputs: + leaves Leaf (0..*) + output: + result Leaf (0..*) + + set result: + leaves + +func FlattenAddOutputs: + inputs: + branches Branch (0..*) + output: + result Leaf (0..*) + + add result: + branches extract AddFromSingle(leaf) then flatten + +func FlattenSetOutputs: + inputs: + branches Branch (0..*) + output: + result Leaf (0..*) + + add result: + branches extract SetFromSingle(leaf) then flatten + +func FlattenSetListOutputs: <"Control: the function returns a list from set."> + inputs: + branches Branch (0..*) + output: + result Leaf (0..*) + + add result: + branches extract SetFromList(leaves) then flatten + +func ExtractFromSetOutput: <"Other callers rely on the output being a list too."> + inputs: + leaf Leaf (0..1) + output: + result number (0..*) + + add result: + SetFromSingle(leaf) extract value + +func CountSetOutput: + inputs: + leaf Leaf (0..1) + output: + result int (1..1) + + set result: + SetFromSingle(leaf) count + +func FirstOfSetOutput: + inputs: + leaf Leaf (0..1) + output: + result Leaf (0..1) + + set result: + SetFromSingle(leaf) first diff --git a/python-test/unit-tests/features/functions/FunctionCycleTest.rosetta b/python-test/unit-tests/features/functions/FunctionCycleTest.rosetta new file mode 100644 index 00000000..e499073b --- /dev/null +++ b/python-test/unit-tests/features/functions/FunctionCycleTest.rosetta @@ -0,0 +1,59 @@ +namespace rosetta_dsl.test.functions.function_cycle : <"generate Python unit tests from Rosetta."> + +// IsEven and IsOdd call each other. IsEven also calls Helper, which is outside the cycle. +// UsesIsEven calls into the cycle from outside it. UsesHelper has no cycle near it. +// Countdown calls itself. + +func IsEven: + inputs: + n int (1..1) + output: + result boolean (1..1) + + set result: + if n = 0 then True else IsOdd(Helper(n) - 2) + +func IsOdd: + inputs: + n int (1..1) + output: + result boolean (1..1) + + set result: + if n = 0 then False else IsEven(n - 1) + +func Helper: + inputs: + n int (1..1) + output: + result int (1..1) + + set result: + n + 1 + +func UsesHelper: + inputs: + n int (1..1) + output: + result int (1..1) + + set result: + Helper(n) + +func UsesIsEven: + inputs: + n int (1..1) + output: + result boolean (1..1) + + set result: + IsEven(n) + +func Countdown: + inputs: + n int (1..1) + output: + result int (1..1) + + set result: + if n = 0 then 0 else Countdown(n - 1) diff --git a/python-test/unit-tests/features/functions/SingleExtractAndEnumTest.rosetta b/python-test/unit-tests/features/functions/SingleExtractAndEnumTest.rosetta new file mode 100644 index 00000000..d4a586a5 --- /dev/null +++ b/python-test/unit-tests/features/functions/SingleExtractAndEnumTest.rosetta @@ -0,0 +1,105 @@ +namespace rosetta_dsl.test.functions.single_extract_enum : <"generate Python unit tests from Rosetta."> + +enum PriceKind: + AssetPrice + Other + +type Multiplier: + value number (1..1) + +type Quantity: + value number (1..1) + multiplier Multiplier (0..1) + +type Price: + priceKind PriceKind (1..1) + +func ExtractFromSingleValue: <"extract over a (0..1) input produces a single value."> + inputs: + quantity Quantity (0..1) + output: + result number (0..1) + + alias units: + quantity + extract + if multiplier exists + then value * multiplier -> value + else value + + set result: + units + +func UseExtractedSingleValueInArithmetic: <"The extracted single value can be used as a number."> + inputs: + quantity Quantity (0..1) + factor number (1..1) + output: + result number (0..1) + + alias units: + quantity + extract value + + set result: + factor * units + +func CountWithUnqualifiedEnumValue: <"The enum value is written without its type."> + inputs: + prices Price (0..*) + output: + result int (1..1) + + set result: + prices + filter priceKind = AssetPrice + then count + +func CountWithQualifiedEnumValue: <"Control: the enum value is written with its type."> + inputs: + prices Price (0..*) + output: + result int (1..1) + + set result: + prices + filter priceKind = PriceKind -> AssetPrice + then count + +func CountWithEnumValueInThen: <"The walk for imports must enter the body of then."> + inputs: + prices Price (0..*) + output: + result int (1..1) + + set result: + prices + then filter priceKind = AssetPrice + then count + +func ConvertToEnum: <"to-enum refers to the enum by name only."> + inputs: + name string (1..1) + output: + result string (0..1) + + set result: + name to-enum PriceKind to-string + +func EnumValueInSwitch: <"The walk for imports must enter the cases of switch."> + inputs: + name string (1..1) + output: + result string (1..1) + + set result: + name switch + "asset" then PriceKind -> AssetPrice to-string, + default PriceKind -> Other to-string + +type PricedQuantity extends Price: <"The enum belongs to the parent type, so only the condition uses it here."> + value number (0..1) + + condition ValueRequiredForAssetPrice: + if priceKind = AssetPrice + then value exists diff --git a/python-test/unit-tests/features/functions/ToEnumConversionTest.rosetta b/python-test/unit-tests/features/functions/ToEnumConversionTest.rosetta new file mode 100644 index 00000000..5cd9a87f --- /dev/null +++ b/python-test/unit-tests/features/functions/ToEnumConversionTest.rosetta @@ -0,0 +1,85 @@ +namespace rosetta_dsl.test.functions.to_enum_conversion : <"generate Python unit tests from Rosetta."> + +enum Foo: + VALUE1 + VALUE2 displayName "Value 2" + +enum Source: + Red + Green displayName "Leaf green" + +enum Target: + Red + Green + +func StringToFoo: <"to-enum converts a string by display name."> + inputs: + name string (0..1) + output: + result Foo (0..1) + + set result: + name to-enum Foo + +func SourceToTarget: <"to-enum converts an enum by value name."> + inputs: + colour Source (0..1) + output: + result Target (0..1) + + set result: + colour to-enum Target + +func SchemeStringToFoo: <"to-enum converts a string that carries a scheme."> + inputs: + name string (0..1) + [metadata scheme] + output: + result Foo (0..1) + + set result: + name to-enum Foo + +func SchemeSourceToTarget: <"to-enum converts an enum that carries a scheme."> + inputs: + colour Source (0..1) + [metadata scheme] + output: + result Target (0..1) + + set result: + colour to-enum Target + +func StringToSchemeFoo: <"to-enum result stored in an output that allows a scheme."> + inputs: + name string (0..1) + output: + result Foo (0..1) + [metadata scheme] + + set result: + name to-enum Foo + +type Holder: <"Attributes with a scheme, as read from a document."> + name string (0..1) + [metadata scheme] + colour Source (0..1) + [metadata scheme] + +func HolderNameToFoo: <"to-enum on a string attribute that carries a scheme."> + inputs: + holder Holder (0..1) + output: + result Foo (0..1) + + set result: + holder -> name to-enum Foo + +func HolderColourToTarget: <"to-enum on an enum attribute that carries a scheme."> + inputs: + holder Holder (0..1) + output: + result Target (0..1) + + set result: + holder -> colour to-enum Target diff --git a/python-test/unit-tests/features/functions/test_functions_bundled_type_name.py b/python-test/unit-tests/features/functions/test_functions_bundled_type_name.py new file mode 100644 index 00000000..0e9b76d7 --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_bundled_type_name.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''functions and conditions that build types from a circular dependency group work''' +from rosetta_dsl.test.functions.bundled_type_name.Checker import Checker +from rosetta_dsl.test.functions.bundled_type_name.GroupB1 import GroupB1 +from rosetta_dsl.test.functions.bundled_type_name.GroupB2 import GroupB2 +from rosetta_dsl.test.functions.bundled_type_name.functions.MakeGroupB1 import MakeGroupB1 +from rosetta_dsl.test.functions.bundled_type_name.functions.MakeMid import MakeMid +from rosetta_dsl.test.functions.bundled_type_name.functions.SetPathGroupB1 import SetPathGroupB1 +from rosetta_dsl.test.functions.bundled_type_name.functions.ValueViaGroupB1 import ValueViaGroupB1 + + +def test_function_builds_type(): + '''a function returns the type it builds''' + result = MakeGroupB1(1) + assert isinstance(result, GroupB1) + assert result.value == 1 + + +def test_function_sets_attribute_of_output(): + '''a function that sets one attribute of its output returns the type''' + result = SetPathGroupB1(1) + assert isinstance(result, GroupB1) + assert result.value == 1 + + +def test_function_builds_type_inside_another(): + '''a function builds one type inside another''' + assert MakeMid(1).groupB1.value == 1 + + +def test_function_uses_type_only_internally(): + '''a function builds a type it doesn't take or return''' + assert ValueViaGroupB1(1) == 1 + + +def test_standalone_condition_builds_type_from_group(): + '''a condition on a type outside the group builds a type from the group''' + assert not Checker(value=1).validate_conditions(raise_exc=False) + + +def test_group_condition_builds_type_outside_group(): + '''a condition on a type in the group builds a type outside the group''' + assert not GroupB2().validate_conditions(raise_exc=False) diff --git a/python-test/unit-tests/features/functions/test_functions_condition_function_import.py b/python-test/unit-tests/features/functions/test_functions_condition_function_import.py new file mode 100644 index 00000000..7975c046 --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_condition_function_import.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''conditions that call functions must work whichever module is imported first''' +import subprocess +import sys +import textwrap + +import pytest + +NAMESPACE = 'rosetta_dsl.test.functions.condition_function_import' +ENTRY_POINTS = [f'{NAMESPACE}.{name}' for name in + ('GroupB1', 'GroupB2', 'WithBundledAttribute', 'WithoutBundledAttribute', + 'functions.HasValue')] + + +def _run_in_new_process(code: str) -> subprocess.CompletedProcess: + '''Import order is the behaviour under test, so each case needs a fresh + interpreter: modules already loaded by pytest would hide the bug.''' + return subprocess.run([sys.executable, '-c', textwrap.dedent(code)], + capture_output=True, text=True, check=False) + + +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_module_can_be_imported_first(entry_point): + '''no import order may raise a circular ImportError''' + result = _run_in_new_process(f'import {entry_point}') + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_conditions_calling_functions_pass(entry_point): + '''every condition can reach the function it calls''' + result = _run_in_new_process(f''' + import {entry_point} + from {NAMESPACE}.GroupB1 import GroupB1 + from {NAMESPACE}.WithBundledAttribute import WithBundledAttribute + from {NAMESPACE}.WithoutBundledAttribute import WithoutBundledAttribute + group_b1 = GroupB1(value=1) + for obj in (group_b1, WithBundledAttribute(groupB1=group_b1), + WithoutBundledAttribute(value=1)): + failures = obj.validate_conditions(raise_exc=False) + assert not failures, (type(obj).__name__, failures) + ''') + assert result.returncode == 0, result.stderr diff --git a/python-test/unit-tests/features/functions/test_functions_count_single_value.py b/python-test/unit-tests/features/functions/test_functions_count_single_value.py new file mode 100644 index 00000000..c59a51ba --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_count_single_value.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''count of a single model object is 1, not its number of fields''' +from rosetta_dsl.test.functions.count_single_value.Holder import Holder +from rosetta_dsl.test.functions.count_single_value.Inner import Inner +from rosetta_dsl.test.functions.count_single_value.Parent import Parent +from rosetta_dsl.test.functions.count_single_value.functions.CountSeveral import CountSeveral +from rosetta_dsl.test.functions.count_single_value.functions.CountSingle import CountSingle +from rosetta_dsl.test.functions.count_single_value.functions.CountSingleAttribute import ( + CountSingleAttribute) + + +def test_condition_counts_single_object_as_one(): + '''a condition sees the model object itself''' + assert not Holder(lone=Inner(value=1)).validate_conditions(raise_exc=False) + + +def test_condition_counts_absent_object_as_zero(): + '''an absent object counts as 0''' + assert not Holder().validate_conditions(raise_exc=False) + + +def test_condition_counts_list(): + '''control: a list longer than the limit still fails the condition''' + inner = Inner(value=1) + assert len(Holder(several=[inner, inner, inner]).validate_conditions(raise_exc=False)) == 1 + + +def test_function_counts_single_object_as_one(): + '''a function sees a copy-on-write proxy for the object''' + assert CountSingle(Inner(value=1)) == 1 + + +def test_function_counts_absent_object_as_zero(): + '''an absent input counts as 0''' + assert CountSingle(None) == 0 + + +def test_function_counts_attribute_path_to_single_object(): + '''a path ending in a single object counts as 1''' + assert CountSingleAttribute(Parent(inner=Inner(value=1))) == 1 + + +def test_function_counts_list(): + '''control: a list counts its items''' + inner = Inner(value=1) + assert CountSeveral([inner, inner]) == 2 diff --git a/python-test/unit-tests/features/functions/test_functions_filter_single_value.py b/python-test/unit-tests/features/functions/test_functions_filter_single_value.py new file mode 100644 index 00000000..5370573c --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_filter_single_value.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''filter on a single value gives that value, or nothing''' +from rosetta_dsl.test.functions.filter_single_value.Holder import Holder +from rosetta_dsl.test.functions.filter_single_value.Leaf import Leaf +from rosetta_dsl.test.functions.filter_single_value.functions.FilterSingle import FilterSingle +from rosetta_dsl.test.functions.filter_single_value.functions.FilterSingleThenCount import ( + FilterSingleThenCount) + + +def test_filter_keeps_single_value_that_passes(): + '''the value is returned when the test holds''' + result = FilterSingle(Leaf(value=1)) + assert isinstance(result, Leaf) + assert result.value == 1 + + +def test_filter_drops_single_value_that_fails(): + '''nothing is returned when the test fails''' + assert FilterSingle(Leaf()) is None + + +def test_filter_of_absent_value(): + '''an absent value gives nothing''' + assert FilterSingle(None) is None + + +def test_count_after_filter_of_value_that_passes(): + '''the kept value counts as one''' + assert FilterSingleThenCount(Leaf(value=1)) == 1 + + +def test_count_after_filter_of_value_that_fails(): + '''a dropped value counts as none''' + assert FilterSingleThenCount(Leaf()) == 0 + + +def test_condition_with_value_that_passes(): + '''the condition sees one kept value''' + assert not Holder(lone=Leaf(value=1)).validate_conditions(raise_exc=False) + + +def test_condition_with_value_that_fails(): + '''the condition sees no kept value''' + assert not Holder(lone=Leaf()).validate_conditions(raise_exc=False) diff --git a/python-test/unit-tests/features/functions/test_functions_flatten_set_output.py b/python-test/unit-tests/features/functions/test_functions_flatten_set_output.py new file mode 100644 index 00000000..b3c917dc --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_flatten_set_output.py @@ -0,0 +1,136 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''flatten, and multi-valued function outputs filled with set''' +from rosetta_dsl.test.functions.flatten_set_output.Branch import Branch +from rosetta_dsl.test.functions.flatten_set_output.Leaf import Leaf +from rosetta_dsl.test.functions.flatten_set_output.Tree import Tree +from rosetta_dsl.test.functions.flatten_set_output.TreeOfSetOutputs import TreeOfSetOutputs +from rosetta_dsl.test.functions.flatten_set_output.functions.AddFromSingle import AddFromSingle +from rosetta_dsl.test.functions.flatten_set_output.functions.CountSetOutput import CountSetOutput +from rosetta_dsl.test.functions.flatten_set_output.functions.ExtractFromSetOutput import ( + ExtractFromSetOutput) +from rosetta_dsl.test.functions.flatten_set_output.functions.FirstOfSetOutput import FirstOfSetOutput +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenAddOutputs import FlattenAddOutputs +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenConditionalLists import ( + FlattenConditionalLists) +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenModelLists import FlattenModelLists +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenSetListOutputs import ( + FlattenSetListOutputs) +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenSetOutputs import FlattenSetOutputs +from rosetta_dsl.test.functions.flatten_set_output.functions.FlattenStringLists import FlattenStringLists +from rosetta_dsl.test.functions.flatten_set_output.functions.SetFromList import SetFromList +from rosetta_dsl.test.functions.flatten_set_output.functions.SetFromSingle import SetFromSingle +from rosetta_dsl.test.functions.flatten_set_output.functions.SetFromSingleString import ( + SetFromSingleString) + + +def _leaf(value): + return Leaf(value=value) + + +def _values(leaves): + return [leaf.value for leaf in leaves] + + +def test_flatten_model_lists(): + '''lists of models flatten to the models, not to their fields''' + branches = [Branch(leaves=[_leaf(1), _leaf(2)]), Branch(leaves=[_leaf(3)])] + assert _values(FlattenModelLists(branches)) == [1, 2, 3] + + +def test_flatten_with_empty_inner_list(): + '''a branch with no leaves adds nothing''' + assert _values(FlattenModelLists([Branch(), Branch(leaves=[_leaf(1)])])) == [1] + + +def test_flatten_string_lists_keeps_strings_whole(): + '''strings are not split into characters''' + branches = [Branch(tags=['ab', 'cd']), Branch(tags=['e'])] + assert FlattenStringLists(branches) == ['ab', 'cd', 'e'] + + +def test_flatten_conditional_lists(): + '''inner lists may come from a conditional''' + branches = [Branch(leaves=[_leaf(1), _leaf(2)]), Branch()] + assert _values(FlattenConditionalLists(branches)) == [1, 2] + + +def test_flatten_in_condition_passes(): + '''two leaves are within the limit''' + tree = Tree(branches=[Branch(leaves=[_leaf(1), _leaf(2)])]) + assert not tree.validate_conditions(raise_exc=False) + + +def test_flatten_in_condition_fails(): + '''three leaves are over the limit''' + tree = Tree(branches=[Branch(leaves=[_leaf(1), _leaf(2), _leaf(3)])]) + assert len(tree.validate_conditions(raise_exc=False)) == 1 + + +def test_flatten_outputs_filled_with_add(): + '''control: add always produces a list''' + branches = [Branch(leaf=_leaf(1)), Branch(leaf=_leaf(2))] + assert _values(FlattenAddOutputs(branches)) == [1, 2] + + +def test_flatten_outputs_filled_with_set_from_list(): + '''control: set from a list produces a list''' + branches = [Branch(leaves=[_leaf(1), _leaf(2)]), Branch(leaves=[_leaf(3)])] + assert _values(FlattenSetListOutputs(branches)) == [1, 2, 3] + + +def test_flatten_outputs_filled_with_set_from_single(): + '''each inner output has one model, so there is one result per branch''' + branches = [Branch(leaf=_leaf(1)), Branch(leaf=_leaf(2))] + assert _values(FlattenSetOutputs(branches)) == [1, 2] + + +def test_flatten_set_outputs_in_condition(): + '''the same pattern must also work in a class condition''' + tree = TreeOfSetOutputs(branches=[Branch(leaf=_leaf(1))]) + assert not tree.validate_conditions(raise_exc=False) + + +def test_add_from_single_returns_list(): + '''control: a multi-valued output filled with add is a list''' + assert _values(AddFromSingle(_leaf(1))) == [1] + + +def test_set_from_list_returns_list(): + '''control: a multi-valued output set from a list is a list''' + assert _values(SetFromList([_leaf(1), _leaf(2)])) == [1, 2] + + +def test_set_from_single_model_returns_list(): + '''a multi-valued output is a list even when set from one model''' + result = SetFromSingle(_leaf(1)) + assert isinstance(result, list) + assert _values(result) == [1] + + +def test_set_from_single_string_returns_list(): + '''a multi-valued output is a list even when set from one basic value''' + assert SetFromSingleString('a') == ['a'] + + +def test_set_from_absent_single_has_no_items(): + '''an absent input gives an output with no items''' + assert not SetFromSingle(None) + + +def test_extract_from_set_output(): + '''extract treats the declared multi-valued output as a list''' + assert ExtractFromSetOutput(_leaf(1)) == [1] + + +def test_count_set_output(): + '''count sees one item''' + assert CountSetOutput(_leaf(1)) == 1 + + +def test_first_of_set_output(): + '''first returns the model, not one of its fields''' + assert FirstOfSetOutput(_leaf(1)).value == 1 diff --git a/python-test/unit-tests/features/functions/test_functions_function_cycle.py b/python-test/unit-tests/features/functions/test_functions_function_cycle.py new file mode 100644 index 00000000..e781c6f3 --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_function_cycle.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''functions that call each other work whichever one is imported first''' +import subprocess +import sys +import textwrap + +import pytest + +NAMESPACE = 'rosetta_dsl.test.functions.function_cycle.functions' + + +def _run_in_new_process(code: str) -> subprocess.CompletedProcess: + '''Import order is the behaviour under test, so each case needs a fresh + interpreter: modules already loaded by pytest would hide the bug.''' + return subprocess.run([sys.executable, '-c', textwrap.dedent(code)], + capture_output=True, text=True, check=False) + + +@pytest.mark.parametrize('first', ['IsEven', 'IsOdd']) +def test_functions_in_cycle_work_whichever_is_imported_first(first): + '''each function can be imported first, and both then give the right answer''' + result = _run_in_new_process(f''' + from {NAMESPACE}.{first} import {first} + from {NAMESPACE}.IsEven import IsEven + from {NAMESPACE}.IsOdd import IsOdd + assert IsEven(4) is True + assert IsOdd(4) is False + assert IsOdd(3) is True + ''') + assert result.returncode == 0, result.stderr + + +def test_function_calling_into_cycle_imported_first(): + '''a function outside the cycle that calls into it can be imported before the cycle''' + result = _run_in_new_process(f''' + from {NAMESPACE}.UsesIsEven import UsesIsEven + assert UsesIsEven(4) is True + assert UsesIsEven(3) is False + ''') + assert result.returncode == 0, result.stderr + + +def test_recursive_function(): + '''a function that calls itself works''' + result = _run_in_new_process(f''' + from {NAMESPACE}.Countdown import Countdown + assert Countdown(3) == 0 + ''') + assert result.returncode == 0, result.stderr + + +def test_function_outside_cycle(): + '''control: a call outside any cycle still works''' + result = _run_in_new_process(f''' + from {NAMESPACE}.UsesHelper import UsesHelper + assert UsesHelper(1) == 2 + ''') + assert result.returncode == 0, result.stderr diff --git a/python-test/unit-tests/features/functions/test_functions_single_extract_enum.py b/python-test/unit-tests/features/functions/test_functions_single_extract_enum.py new file mode 100644 index 00000000..89af8348 --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_single_extract_enum.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''extract over a single value, and enum values written without their type''' +import pytest + +from rune.runtime.conditions import ConditionViolationError + +from rosetta_dsl.test.functions.single_extract_enum.Multiplier import Multiplier +from rosetta_dsl.test.functions.single_extract_enum.Price import Price +from rosetta_dsl.test.functions.single_extract_enum.PriceKind import PriceKind +from rosetta_dsl.test.functions.single_extract_enum.PricedQuantity import PricedQuantity +from rosetta_dsl.test.functions.single_extract_enum.Quantity import Quantity +from rosetta_dsl.test.functions.single_extract_enum.functions.ConvertToEnum import ( + ConvertToEnum) +from rosetta_dsl.test.functions.single_extract_enum.functions.CountWithEnumValueInThen import ( + CountWithEnumValueInThen) +from rosetta_dsl.test.functions.single_extract_enum.functions.CountWithQualifiedEnumValue import ( + CountWithQualifiedEnumValue) +from rosetta_dsl.test.functions.single_extract_enum.functions.CountWithUnqualifiedEnumValue import ( + CountWithUnqualifiedEnumValue) +from rosetta_dsl.test.functions.single_extract_enum.functions.EnumValueInSwitch import ( + EnumValueInSwitch) +from rosetta_dsl.test.functions.single_extract_enum.functions.ExtractFromSingleValue import ( + ExtractFromSingleValue) +from rosetta_dsl.test.functions.single_extract_enum.functions.UseExtractedSingleValueInArithmetic import ( + UseExtractedSingleValueInArithmetic) + + +def test_extract_from_single_value(): + '''extract applies to the one value rather than iterating over it''' + assert ExtractFromSingleValue(Quantity(value=2)) == 2 + + +def test_extract_from_single_value_uses_its_attributes(): + '''the extract body can read the single value's attributes''' + quantity = Quantity(value=2, multiplier=Multiplier(value=3)) + assert ExtractFromSingleValue(quantity) == 6 + + +def test_extract_from_absent_single_value(): + '''an absent input gives an absent result, not an empty list''' + assert ExtractFromSingleValue(None) is None + + +def test_extracted_single_value_in_arithmetic(): + '''the result of the extract is a number, not a one-item list''' + assert UseExtractedSingleValueInArithmetic(Quantity(value=2), 5) == 10 + + +def test_unqualified_enum_value_is_imported(): + '''an enum value written without its type must still resolve''' + prices = [Price(priceKind=PriceKind.ASSET_PRICE), Price(priceKind=PriceKind.OTHER)] + assert CountWithUnqualifiedEnumValue(prices) == 1 + + +def test_qualified_enum_value_is_imported(): + '''control: an enum value written with its type resolves''' + prices = [Price(priceKind=PriceKind.ASSET_PRICE), Price(priceKind=PriceKind.OTHER)] + assert CountWithQualifiedEnumValue(prices) == 1 + + +def test_enum_value_in_then_body_is_imported(): + '''the enum may be used only inside the body of a then''' + prices = [Price(priceKind=PriceKind.ASSET_PRICE), Price(priceKind=PriceKind.OTHER)] + assert CountWithEnumValueInThen(prices) == 1 + + +def test_enum_value_in_switch_is_imported(): + '''the enum may be used only inside the cases of a switch''' + assert EnumValueInSwitch('asset') == 'AssetPrice' + assert EnumValueInSwitch('other') == 'Other' + + +def test_to_enum_resolves_the_enum(): + '''to-enum must refer to an enum that the module can reach''' + assert ConvertToEnum('AssetPrice') == 'AssetPrice' + + +def test_enum_value_in_inherited_condition_is_imported(): + '''a class condition may be the only use of the enum in its module''' + PricedQuantity(priceKind=PriceKind.ASSET_PRICE, value=1).validate_conditions() + with pytest.raises(ConditionViolationError): + PricedQuantity(priceKind=PriceKind.ASSET_PRICE).validate_conditions() diff --git a/python-test/unit-tests/features/functions/test_functions_to_enum_conversion.py b/python-test/unit-tests/features/functions/test_functions_to_enum_conversion.py new file mode 100644 index 00000000..8e00d526 --- /dev/null +++ b/python-test/unit-tests/features/functions/test_functions_to_enum_conversion.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''to-enum gives the matching enum value, or nothing if there is no match''' +from rune.runtime.metadata import StrWithMeta + +from rosetta_dsl.test.functions.to_enum_conversion.Foo import Foo +from rosetta_dsl.test.functions.to_enum_conversion.Holder import Holder +from rosetta_dsl.test.functions.to_enum_conversion.Source import Source +from rosetta_dsl.test.functions.to_enum_conversion.Target import Target +from rosetta_dsl.test.functions.to_enum_conversion.functions.HolderColourToTarget import ( + HolderColourToTarget) +from rosetta_dsl.test.functions.to_enum_conversion.functions.HolderNameToFoo import HolderNameToFoo +from rosetta_dsl.test.functions.to_enum_conversion.functions.SchemeSourceToTarget import ( + SchemeSourceToTarget) +from rosetta_dsl.test.functions.to_enum_conversion.functions.SchemeStringToFoo import SchemeStringToFoo +from rosetta_dsl.test.functions.to_enum_conversion.functions.SourceToTarget import SourceToTarget +from rosetta_dsl.test.functions.to_enum_conversion.functions.StringToSchemeFoo import StringToSchemeFoo +from rosetta_dsl.test.functions.to_enum_conversion.functions.StringToFoo import StringToFoo + + +def test_string_matching_value_name(): + '''a value without a display name matches its name''' + assert StringToFoo('VALUE1') == Foo.VALUE1 + + +def test_string_matching_display_name(): + '''a value with a display name matches the display name''' + assert StringToFoo('Value 2') == Foo.VALUE2 + + +def test_string_matching_name_instead_of_display_name(): + '''a value with a display name does not match its name''' + assert StringToFoo('VALUE2') is None + + +def test_string_matching_nothing(): + '''a string that matches no value gives nothing''' + assert StringToFoo('not a value') is None + + +def test_absent_string(): + '''an absent string gives nothing''' + assert StringToFoo(None) is None + + +def test_enum_to_enum_by_value_name(): + '''an enum converts to the value with the same name''' + assert SourceToTarget(Source.RED) == Target.RED + + +def test_enum_to_enum_ignores_display_name(): + '''an enum converts by name even when its value has a display name''' + assert SourceToTarget(Source.GREEN) == Target.GREEN + + +def test_absent_enum(): + '''an absent enum gives nothing''' + assert SourceToTarget(None) is None + + +def _holder(**fields): + '''A Holder read from a document, so attributes carry their scheme.''' + return Holder.model_validate( + {name: {'@data': value, '@scheme': 'http://example.org/scheme'} + for name, value in fields.items()}) + + +def test_string_attribute_with_scheme(): + '''a string read with a scheme converts by display name''' + assert HolderNameToFoo(_holder(name='Value 2')) == Foo.VALUE2 + + +def test_string_attribute_with_scheme_matching_nothing(): + '''a string read with a scheme that matches no value gives nothing''' + assert HolderNameToFoo(_holder(name='not a value')) is None + + +def test_absent_attribute_with_scheme(): + '''an absent attribute gives nothing''' + assert HolderNameToFoo(Holder()) is None + + +def test_enum_attribute_with_scheme(): + '''an enum read with a scheme converts by value name''' + assert HolderColourToTarget(_holder(colour='Leaf green')) == Target.GREEN + + +def test_string_input_with_scheme(): + '''a string input declared with a scheme converts by display name''' + assert SchemeStringToFoo(StrWithMeta('Value 2', scheme='http://example.org/scheme')) == Foo.VALUE2 + + +def test_enum_input_with_scheme(): + '''an enum input declared with a scheme converts by value name''' + assert SchemeSourceToTarget(Source.GREEN) == Target.GREEN + + +def test_result_stored_in_output_with_scheme(): + '''the converted value can be returned where the output allows a scheme''' + assert StringToSchemeFoo('Value 2') == Foo.VALUE2 + + +def test_unmatched_result_for_output_with_scheme(): + '''no match gives nothing, even where the output allows a scheme''' + assert StringToSchemeFoo('not a value') is None diff --git a/python-test/unit-tests/features/language/ChoiceSwitchTest.rosetta b/python-test/unit-tests/features/language/ChoiceSwitchTest.rosetta new file mode 100644 index 00000000..3b6a88dc --- /dev/null +++ b/python-test/unit-tests/features/language/ChoiceSwitchTest.rosetta @@ -0,0 +1,75 @@ +namespace rosetta_dsl.test.language.choice_switch : <"generate Python unit tests from Rosetta."> + +// Based on the "Using switch for choice types" example in the Rune documentation. + +choice PoweredVehicle: + PetrolCar + ElectricCar + Motorcycle + +type PetrolCar: + fuelCapacity number (1..1) + +type ElectricCar: + batteryCapacity number (1..1) + +type Motorcycle: + fuelCapacity number (1..1) + +choice Vehicle: + PoweredVehicle + Bicycle + +type Bicycle: + weight number (1..1) + +enum Size: + Small + Large + +func ComputeMileage: <"A case can use the attributes of its option directly."> + inputs: + vehicle PoweredVehicle (0..1) + output: + mileage number (0..1) + + set mileage: + vehicle switch + PetrolCar then 15 * fuelCapacity, + ElectricCar then 5 * batteryCapacity, + default 80 + +func ComputeVehicleMileage: <"A case can name an option of a nested choice."> + inputs: + vehicle Vehicle (1..1) + output: + mileage number (1..1) + + set mileage: + vehicle switch + PetrolCar then 15 * fuelCapacity, + ElectricCar then 5 * batteryCapacity, + PoweredVehicle then 80, + Bicycle then 30 + +func DescribeSize: <"Enum switch with a default."> + inputs: + size Size (0..1) + output: + description string (0..1) + + set description: + size switch + Small then "small", + default "other" + +func DescribeNumber: <"Literal switch with a default."> + inputs: + n int (0..1) + output: + description string (0..1) + + set description: + n switch + 1 then "one", + default "other" diff --git a/python-test/unit-tests/features/language/TypeSwitchTest.rosetta b/python-test/unit-tests/features/language/TypeSwitchTest.rosetta new file mode 100644 index 00000000..080438e3 --- /dev/null +++ b/python-test/unit-tests/features/language/TypeSwitchTest.rosetta @@ -0,0 +1,49 @@ +namespace rosetta_dsl.test.language.type_switch : <"generate Python unit tests from Rosetta."> + +// Based on the "Switch for complex types" example in the Rune documentation. + +type Person: + name string (0..1) + +type Employee extends Person: + salary number (1..1) + +type Contractor extends Person: + hourlyRate number (1..1) + +type Manager extends Employee: + bonus number (1..1) + +func ComputeMonthlyWage: <"A case can use the attributes of its type directly."> + inputs: + person Person (0..1) + output: + monthlyWage number (0..1) + + set monthlyWage: + person switch + Employee then salary, + Contractor then hourlyRate * 8 * 5 * 20, + default 0 + +func DescribeMostSpecificFirst: <"A subtype case comes before its parent type's case; Rune rejects the reverse order."> + inputs: + person Person (1..1) + output: + description string (1..1) + + set description: + person switch + Manager then "manager", + Employee then "employee", + default "person" + +type Payroll: <"A type switch inside a class condition."> + person Person (0..1) + + condition PositivePay: + if person exists + then person switch + Employee then salary > 0, + Contractor then hourlyRate > 0, + default True diff --git a/python-test/unit-tests/features/language/test_choice_switch.py b/python-test/unit-tests/features/language/test_choice_switch.py new file mode 100644 index 00000000..3d1d732f --- /dev/null +++ b/python-test/unit-tests/features/language/test_choice_switch.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''switch narrows to the matching choice option, and gives nothing for an absent value''' +from rosetta_dsl.test.language.choice_switch.Bicycle import Bicycle +from rosetta_dsl.test.language.choice_switch.ElectricCar import ElectricCar +from rosetta_dsl.test.language.choice_switch.Motorcycle import Motorcycle +from rosetta_dsl.test.language.choice_switch.PetrolCar import PetrolCar +from rosetta_dsl.test.language.choice_switch.PoweredVehicle import PoweredVehicle +from rosetta_dsl.test.language.choice_switch.Size import Size +from rosetta_dsl.test.language.choice_switch.Vehicle import Vehicle +from rosetta_dsl.test.language.choice_switch.functions.ComputeMileage import ComputeMileage +from rosetta_dsl.test.language.choice_switch.functions.ComputeVehicleMileage import ComputeVehicleMileage +from rosetta_dsl.test.language.choice_switch.functions.DescribeNumber import DescribeNumber +from rosetta_dsl.test.language.choice_switch.functions.DescribeSize import DescribeSize + + +def test_choice_case_uses_option_attribute(): + '''a petrol car matches its case and can use its fuel capacity''' + assert ComputeMileage(PoweredVehicle(PetrolCar=PetrolCar(fuelCapacity=10))) == 150 + + +def test_choice_case_uses_other_option_attribute(): + '''an electric car matches its case and can use its battery capacity''' + assert ComputeMileage(PoweredVehicle(ElectricCar=ElectricCar(batteryCapacity=20))) == 100 + + +def test_choice_option_without_case_matches_default(): + '''a motorcycle has no case, so it matches the default''' + assert ComputeMileage(PoweredVehicle(Motorcycle=Motorcycle(fuelCapacity=5))) == 80 + + +def test_absent_choice_gives_nothing(): + '''an absent choice matches no case and gives nothing''' + assert ComputeMileage(None) is None + + +def test_nested_choice_option(): + '''a petrol car inside the nested choice matches its own case''' + vehicle = Vehicle(PoweredVehicle=PoweredVehicle(PetrolCar=PetrolCar(fuelCapacity=10))) + assert ComputeVehicleMileage(vehicle) == 150 + + +def test_nested_choice_parent_option(): + '''a motorcycle inside the nested choice matches the PoweredVehicle case''' + vehicle = Vehicle(PoweredVehicle=PoweredVehicle(Motorcycle=Motorcycle(fuelCapacity=5))) + assert ComputeVehicleMileage(vehicle) == 80 + + +def test_nested_choice_other_option(): + '''a bicycle matches its case''' + assert ComputeVehicleMileage(Vehicle(Bicycle=Bicycle(weight=12))) == 30 + + +def test_enum_switch_matches(): + '''control: an enum case matches''' + assert DescribeSize(Size.SMALL) == 'small' + + +def test_absent_enum_gives_nothing(): + '''an absent enum matches no case and gives nothing''' + assert DescribeSize(None) is None + + +def test_literal_switch_matches(): + '''control: a literal case matches''' + assert DescribeNumber(1) == 'one' + + +def test_absent_literal_gives_nothing(): + '''an absent value matches no case and gives nothing''' + assert DescribeNumber(None) is None diff --git a/python-test/unit-tests/features/language/test_type_switch.py b/python-test/unit-tests/features/language/test_type_switch.py new file mode 100644 index 00000000..c7586fb4 --- /dev/null +++ b/python-test/unit-tests/features/language/test_type_switch.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''switch on a type with subtypes matches the value's type''' +from rosetta_dsl.test.language.type_switch.Contractor import Contractor +from rosetta_dsl.test.language.type_switch.Employee import Employee +from rosetta_dsl.test.language.type_switch.Manager import Manager +from rosetta_dsl.test.language.type_switch.Payroll import Payroll +from rosetta_dsl.test.language.type_switch.Person import Person +from rosetta_dsl.test.language.type_switch.functions.ComputeMonthlyWage import ComputeMonthlyWage +from rosetta_dsl.test.language.type_switch.functions.DescribeMostSpecificFirst import ( + DescribeMostSpecificFirst) + + +def test_case_matches_subtype(): + '''an employee matches the Employee case and can use its salary''' + assert ComputeMonthlyWage(Employee(salary=3000)) == 3000 + + +def test_case_uses_attributes_of_its_type(): + '''a contractor matches the Contractor case and can use its hourly rate''' + assert ComputeMonthlyWage(Contractor(hourlyRate=10)) == 8000 + + +def test_parent_type_matches_default(): + '''a person who is neither matches the default''' + assert ComputeMonthlyWage(Person(name='Ann')) == 0 + + +def test_absent_input_gives_nothing(): + '''an absent value matches no case and gives nothing''' + assert ComputeMonthlyWage(None) is None + + +def test_subtype_of_case_type_matches(): + '''a manager is an employee, so it matches the Employee case''' + assert ComputeMonthlyWage(Manager(salary=5000, bonus=100)) == 5000 + + +def test_most_specific_case_first(): + '''with the subtype case first, a manager matches it''' + assert DescribeMostSpecificFirst(Manager(salary=5000, bonus=100)) == 'manager' + assert DescribeMostSpecificFirst(Employee(salary=3000)) == 'employee' + assert DescribeMostSpecificFirst(Person()) == 'person' + + +def test_type_switch_in_condition_passes(): + '''the condition sees the employee's salary''' + assert not Payroll(person=Employee(salary=3000)).validate_conditions(raise_exc=False) + + +def test_type_switch_in_condition_fails(): + '''the condition sees the contractor's rate and rejects it''' + assert len(Payroll(person=Contractor(hourlyRate=0)).validate_conditions(raise_exc=False)) == 1 diff --git a/python-test/unit-tests/features/model-structure/BuiltinNameClash.rosetta b/python-test/unit-tests/features/model-structure/BuiltinNameClash.rosetta new file mode 100644 index 00000000..1290554c --- /dev/null +++ b/python-test/unit-tests/features/model-structure/BuiltinNameClash.rosetta @@ -0,0 +1,27 @@ +namespace rosetta_dsl.test.model_structure.builtin_name_clash : <"generate Python unit tests for attributes named after a Python name their own type annotation uses."> + +choice ChoiceBasic: + string + int + +type Clash: + str string (0..1) + bool boolean (0..1) + +type DateClash: + datetime date (0..1) + +type TimeClash: + datetime time (0..1) + +type DecimalClash: + Decimal number (0..1) + +// BundledChoice and Node refer to each other, so both are bundled. The other types are standalone. +choice BundledChoice: + int + Node + +type Node: + next BundledChoice (0..1) + str string (0..1) diff --git a/python-test/unit-tests/features/model-structure/BundleSchemaOrder.rosetta b/python-test/unit-tests/features/model-structure/BundleSchemaOrder.rosetta new file mode 100644 index 00000000..9c38b168 --- /dev/null +++ b/python-test/unit-tests/features/model-structure/BundleSchemaOrder.rosetta @@ -0,0 +1,26 @@ +namespace rosetta_dsl.test.model.bundle_schema_order : <"generate Python unit tests from Rosetta."> + +// Two independent cycle groups share one _bundle.py, joined by other types: +// GroupA1 -> Outer -> Inner -> GroupB1. The dependency graph has no cycle through +// Outer and Inner, but the files do, so Outer and Inner must also be bundled. + +type GroupA1: <"First cycle group."> + groupA2 GroupA2 (0..1) + outer Outer (0..1) + +type GroupA2: + groupA1 GroupA1 (0..1) + +type Outer: <"On the path between the cycle groups. Used directly by the bundle."> + inner Inner (0..1) + +type Inner: <"On the path between the cycle groups. Uses the bundle directly."> + groupB1 GroupB1 (0..1) + +type GroupB1: <"Second cycle group."> + value int (1..1) + groupB2 GroupB2 (0..1) + +type GroupB2: + value int (1..1) + groupB1 GroupB1 (0..1) diff --git a/python-test/unit-tests/features/model-structure/CircularDependency.rosetta b/python-test/unit-tests/features/model-structure/CircularDependency.rosetta index b09bc7b2..109fccc3 100644 --- a/python-test/unit-tests/features/model-structure/CircularDependency.rosetta +++ b/python-test/unit-tests/features/model-structure/CircularDependency.rosetta @@ -3,6 +3,7 @@ namespace rosetta_dsl.test.model.circular_dependency : <"generate Python unit te type Bar1: <"Test Circular Dependency"> number1 int(1..1) bar2 Bar2(0..1) + leaf BarLeaf (0..1) type Bar2: number2 int(1..1) bar1 Bar1(0..1) @@ -10,3 +11,8 @@ type Bar2: if bar1 exists then bar1->number1 > 0 +type BarLeaf: <"Standalone, used by the bundle."> + value int (0..1) + +type BarContainer: <"Standalone, uses the bundle."> + bar1 Bar1 (0..1) diff --git a/python-test/unit-tests/features/model-structure/test_builtin_name_clash.py b/python-test/unit-tests/features/model-structure/test_builtin_name_clash.py new file mode 100644 index 00000000..0d8b712b --- /dev/null +++ b/python-test/unit-tests/features/model-structure/test_builtin_name_clash.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +"""An attribute named after a Python name its own annotation uses is still validated (#331).""" + +import datetime +from decimal import Decimal + +import pytest +from pydantic import ValidationError + +from rosetta_dsl.test.model_structure.builtin_name_clash.BundledChoice import BundledChoice +from rosetta_dsl.test.model_structure.builtin_name_clash.ChoiceBasic import ChoiceBasic +from rosetta_dsl.test.model_structure.builtin_name_clash.Clash import Clash +from rosetta_dsl.test.model_structure.builtin_name_clash.DateClash import DateClash +from rosetta_dsl.test.model_structure.builtin_name_clash.DecimalClash import DecimalClash +from rosetta_dsl.test.model_structure.builtin_name_clash.Node import Node +from rosetta_dsl.test.model_structure.builtin_name_clash.TimeClash import TimeClash + + +@pytest.mark.parametrize( + "make", + [ + pytest.param(lambda: ChoiceBasic(int="abc"), id="choice-int"), + pytest.param(lambda: Clash(str=5), id="str"), + pytest.param(lambda: Clash(bool="abc"), id="bool"), + pytest.param(lambda: DateClash(datetime="not a date"), id="datetime-date"), + pytest.param(lambda: TimeClash(datetime="not a time"), id="datetime-time"), + pytest.param(lambda: DecimalClash(Decimal="abc"), id="decimal"), + pytest.param(lambda: BundledChoice(int="abc"), id="bundled-choice-int"), + pytest.param(lambda: Node(str=5), id="bundled-str"), + ], +) +def test_invalid_value_is_rejected(make): + """the attribute's own type is enforced""" + with pytest.raises(ValidationError): + make() + + +def test_valid_values_are_accepted(): + """valid values are kept, with the attribute's own type""" + assert ChoiceBasic(int=5).int == 5 + assert Clash(str="x", bool=True).str == "x" + assert Clash(str="x", bool=True).bool is True + assert DecimalClash(Decimal="1.5").Decimal == Decimal("1.5") + assert BundledChoice(int=5).int == 5 + assert Node(str="x").str == "x" + + +def test_datetime_attributes_hold_their_type(): + """a date or time attribute named datetime is defined and converts its value""" + assert DateClash(datetime="2026-11-11").datetime == datetime.date(2026, 11, 11) + assert TimeClash(datetime="10:30:00").datetime == datetime.time(10, 30) diff --git a/python-test/unit-tests/features/model-structure/test_bundle_schema_order.py b/python-test/unit-tests/features/model-structure/test_bundle_schema_order.py new file mode 100644 index 00000000..b5a928a1 --- /dev/null +++ b/python-test/unit-tests/features/model-structure/test_bundle_schema_order.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''schema and import behaviour must not depend on which module loads first''' +import subprocess +import sys +import textwrap + +import pytest + +NAMESPACE = 'rosetta_dsl.test.model.bundle_schema_order' +# Users import the model's modules, never the bundle itself +ENTRY_POINTS = [f'{NAMESPACE}.{name}' for name in ('Inner', 'Outer', 'GroupA1', 'GroupB1')] + + +def _run_in_new_process(code: str) -> subprocess.CompletedProcess: + '''Import order is the behaviour under test, so each case needs a fresh + interpreter: modules already loaded by pytest would hide the bug.''' + return subprocess.run([sys.executable, '-c', textwrap.dedent(code)], + capture_output=True, text=True, check=False) + + +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_module_can_be_imported_first(entry_point): + '''no import order may raise a circular ImportError''' + result = _run_in_new_process(f'import {entry_point}') + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_standalone_type_validates_nested_bundled_type(entry_point): + '''a standalone type must not keep the bundle's `None` placeholders''' + result = _run_in_new_process(f''' + import {entry_point} + from {NAMESPACE}.Inner import Inner + inner = Inner.model_validate( + {{'groupB1': {{'value': 1, 'groupB2': {{'value': 2}}}}}}) + assert inner.groupB1.groupB2.value == 2 + ''') + assert result.returncode == 0, result.stderr diff --git a/python-test/unit-tests/features/model-structure/test_g1_circular_import.py b/python-test/unit-tests/features/model-structure/test_g1_circular_import.py index 97b9acf0..17537374 100644 --- a/python-test/unit-tests/features/model-structure/test_g1_circular_import.py +++ b/python-test/unit-tests/features/model-structure/test_g1_circular_import.py @@ -4,70 +4,52 @@ # """ -## Bug Report +G1 – circular ImportError when a module is imported before the bundle has loaded. -G1 – Circular ImportError when non-bundle class is imported before _bundle initialises +Bar1 and Bar2 form a cycle, so both are generated into _bundle.py. BarLeaf is a +standalone type that the bundle uses, and BarContainer is a standalone type that uses +the bundle. In CDM 7.7.0, importing such a standalone module first made the bundle +import it again while it was still loading: -### Steps to Reproduce: - 1. Generate Python from a Rune model where some types are in a circular-dependency - bundle (Bar1/Bar2) and _bundle.py uses `from … ClassName import ClassName` for - any non-bundle module. - 2. In a fresh Python process import the non-bundle module first; it imports a bundle - type at module level, triggering _bundle to load; _bundle then tries to import the - non-bundle module again from inside its own initialisation. - 3. The circular initialisation raises ImportError. - -### Expected Result: -_bundle.py must use `import module` style (never `from module import ClassName`) for -all external references so that no module-level class import creates a circular chain. - -### Actual Result (bug scenario – CDM 7.7.0): ImportError: cannot import name 'Asset' from partially initialized module 'finos.cdm.base.staticdata.asset.common.asset' -CDM 7.7.0's _bundle.py contained `from … Asset import Asset` (line 4880), while -Asset.py imported bundle types at module level — forming the cycle. - -### Environment: -rune-python-generator; CDM 7.7.0 exhibited this; regression test for the fix. -### Additional Context: -The current generator emits `import module` style for all non-bundle module references -in _bundle.py. This test confirms that pattern is preserved in the generated unit-test -bundle. +Each module must be importable first, and usable afterwards. Import order is the +behaviour under test, so every case runs in a new interpreter: modules already loaded +by pytest would hide the bug. How the generator writes the bundle's imports is checked +by the JUnit tests. """ - -import inspect +import subprocess +import sys +import textwrap import pytest -import rosetta_dsl._bundle as bundle +NAMESPACE = 'rosetta_dsl.test.model.circular_dependency' +ENTRY_POINTS = [f'{NAMESPACE}.{name}' for name in ('Bar1', 'Bar2', 'BarLeaf', 'BarContainer')] + +def _run_in_new_process(code: str) -> subprocess.CompletedProcess: + return subprocess.run([sys.executable, '-c', textwrap.dedent(code)], + capture_output=True, text=True, check=False) -def test_g1_bundle_does_not_use_direct_class_import_for_non_bundle_modules(): - """ - _bundle.py must not contain any `from import ` line for - modules outside the bundle itself. Such a line would create a circular import - if that module imports a bundle type at module level. - The generator must emit `import module` style for enum/non-bundle references. - """ - source = inspect.getsource(bundle) +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_g1_module_can_be_imported_first(entry_point): + '''importing any module first must not raise a circular ImportError''' + result = _run_in_new_process(f'import {entry_point}') + assert result.returncode == 0, result.stderr - # Collect any `from X import Y` lines that are NOT standard-library / runtime imports - forbidden_prefixes = ( - 'rosetta_dsl.test.', - 'finos.', - ) - violations = [] - for line in source.splitlines(): - stripped = line.strip() - if not stripped.startswith('from '): - continue - # Allow runtime / pydantic imports - if any(stripped.startswith(f'from {p}') for p in forbidden_prefixes): - violations.append(stripped) - assert not violations, ( - "G1 regression: _bundle.py uses direct class imports for non-bundle modules:\n" - + "\n".join(violations) - ) +@pytest.mark.parametrize('entry_point', ENTRY_POINTS) +def test_g1_types_are_usable_after_importing_module_first(entry_point): + '''every type works together whichever module was imported first''' + result = _run_in_new_process(f''' + import {entry_point} + from {NAMESPACE}.Bar1 import Bar1 + from {NAMESPACE}.BarContainer import BarContainer + from {NAMESPACE}.BarLeaf import BarLeaf + container = BarContainer(bar1=Bar1(number1=1, leaf=BarLeaf(value=2))) + assert container.bar1.leaf.value == 2 + ''') + assert result.returncode == 0, result.stderr diff --git a/python-test/unit-tests/features/model-structure/test_g2_schema_stale.py b/python-test/unit-tests/features/model-structure/test_g2_schema_stale.py index 0ca98acb..73f167bc 100644 --- a/python-test/unit-tests/features/model-structure/test_g2_schema_stale.py +++ b/python-test/unit-tests/features/model-structure/test_g2_schema_stale.py @@ -35,8 +35,9 @@ ### Additional Context: The current generator emits the lazy __getattr__ pattern for all bundle class modules. -This test confirms that pattern is present and that importing a bundle class via -`from module import ClassName` delivers the fully-rebuilt class, not the placeholder. +That pattern is checked by the JUnit tests. These tests confirm that importing a bundle +class via `from module import ClassName` delivers the fully-rebuilt class, not the +placeholder. """ import sys @@ -52,26 +53,6 @@ def _clear_bundle_modules(): del sys.modules[k] -def test_g2_bundle_class_module_uses_lazy_getattr(): - """ - Each bundle class module must define __getattr__ rather than a top-level - class. A module-level class definition is the old pattern that causes G2: - the importing module gets the class before Phase-3 rebuilds it. - """ - import inspect - import rosetta_dsl.test.model.circular_dependency.Bar1 as bar1_module - import rosetta_dsl.test.model.circular_dependency.Bar2 as bar2_module - - for name, mod in [('Bar1', bar1_module), ('Bar2', bar2_module)]: - source = inspect.getsource(mod) - assert 'def __getattr__' in source, ( - f"G2 regression: {name}.py does not use lazy __getattr__ pattern" - ) - assert 'class ' not in source, ( - f"G2 regression: {name}.py defines the class directly (old pattern)" - ) - - def test_g2_from_import_of_bundle_class_returns_rebuilt_type(): """ `from … Bar1 import Bar1` must return the fully-rebuilt bundle class, not diff --git a/python-test/unit-tests/features/serialization/ChoiceJavaFormat.rosetta b/python-test/unit-tests/features/serialization/ChoiceJavaFormat.rosetta new file mode 100644 index 00000000..fc4a3a51 --- /dev/null +++ b/python-test/unit-tests/features/serialization/ChoiceJavaFormat.rosetta @@ -0,0 +1,82 @@ +namespace rosetta_dsl.test.serialization.choice_java_format : <"generate Python unit tests for reading and writing choice types in the Rune JSON format."> + +type A: + fieldA string (1..1) + +type B: + fieldB string (1..1) + +choice ChoiceData: + A + B + +choice ChoiceBasic: + string + int + +choice ChoiceNested: + ChoiceData + ChoiceBasic + +type PaymentDate: + [metadata key] + day string (1..1) + +type WithDate: + paymentDate PaymentDate (0..1) + +choice ChoiceDated: + WithDate + B + +enum CurrencyEnum: + USD + EUR + +// @type is the enum's Rune FQN for an enum option, and the option name for a basic option. +// (Java writes the option name for a plain enum option.) +choice ChoiceEnum: + CurrencyEnum + string + +// Metadata doesn't change @type, which is never a language-specific name. +// (Java writes the wrapped value's Java class name here, e.g. java.lang.String.) +choice ChoiceEnumMeta: + CurrencyEnum + [metadata scheme] + string + [metadata scheme] + +// The same fields as a choice of A and B, but not a choice: its JSON uses attribute names. +type ViaOneOf: + A A (0..1) + B B (0..1) + + condition Choice: + one-of + +type Root: + data ChoiceData (0..1) + basic ChoiceBasic (0..1) + nested ChoiceNested (0..1) + entries ChoiceData (0..*) + located ChoiceData (0..1) + [metadata location] + dated ChoiceDated (0..1) + dateRef PaymentDate (0..1) + [metadata reference] + viaOneOf ViaOneOf (0..1) + currency ChoiceEnum (0..1) + currencyMeta ChoiceEnumMeta (0..1) + +// LoopOption and LoopHolder refer to each other through LoopChoice, so all three are bundled. +type LoopOption: + back LoopHolder (0..1) + +choice LoopChoice: + LoopOption + B + +type LoopHolder: + option LoopChoice (0..1) + options LoopChoice (0..*) diff --git a/python-test/unit-tests/features/serialization/ScopedKeySerialization.rosetta b/python-test/unit-tests/features/serialization/ScopedKeySerialization.rosetta new file mode 100644 index 00000000..0cde72a8 --- /dev/null +++ b/python-test/unit-tests/features/serialization/ScopedKeySerialization.rosetta @@ -0,0 +1,23 @@ +namespace rosetta_dsl.test.serialization.scoped_key : <"generate Python unit tests for writing scoped keys."> + +// BundledHolder and Back refer to each other, so they are bundled. Holder is standalone and +// otherwise the same. + +type Measure: + value int (0..1) + +type Holder: + measure Measure (0..1) + [metadata location] + measureRef Measure (0..1) + [metadata address "pointsTo"=Holder->measure] + +type BundledHolder: + measure Measure (0..1) + [metadata location] + measureRef Measure (0..1) + [metadata address "pointsTo"=BundledHolder->measure] + back Back (0..1) + +type Back: + holder BundledHolder (0..1) diff --git a/python-test/unit-tests/features/serialization/TypeLevelKeySerialization.rosetta b/python-test/unit-tests/features/serialization/TypeLevelKeySerialization.rosetta new file mode 100644 index 00000000..0860c405 --- /dev/null +++ b/python-test/unit-tests/features/serialization/TypeLevelKeySerialization.rosetta @@ -0,0 +1,13 @@ +namespace rosetta_dsl.test.serialization.type_level_key : <"generate Python unit tests for writing type-level keys."> + +// Party has a key on the type, not on the attribute. SingleParty and PartyList are standalone. + +type Party: + [metadata key] + name string (0..1) + +type SingleParty: + party Party (0..1) + +type PartyList: + parties Party (0..*) diff --git a/python-test/unit-tests/features/serialization/test_choice_java_format.py b/python-test/unit-tests/features/serialization/test_choice_java_format.py new file mode 100644 index 00000000..9a98cc92 --- /dev/null +++ b/python-test/unit-tests/features/serialization/test_choice_java_format.py @@ -0,0 +1,135 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''choice types are read and written in the Rune JSON format used by Java''' +import json + +import pytest + +from rune.runtime.base_data_class import BaseDataClass +from rosetta_dsl.test.serialization.choice_java_format.A import A +from rosetta_dsl.test.serialization.choice_java_format.B import B +from rosetta_dsl.test.serialization.choice_java_format.ChoiceData import ChoiceData +from rosetta_dsl.test.serialization.choice_java_format.ChoiceEnum import ChoiceEnum +from rosetta_dsl.test.serialization.choice_java_format.CurrencyEnum import CurrencyEnum +from rosetta_dsl.test.serialization.choice_java_format.LoopHolder import LoopHolder +from rosetta_dsl.test.serialization.choice_java_format.Root import Root + +NS = 'rosetta_dsl.test.serialization.choice_java_format' +A_JSON = {'@type': f'{NS}.A', 'fieldA': 'foo'} +B_JSON = {'@type': f'{NS}.B', 'fieldB': 'bar'} + +# @type is the enum's Rune FQN for an enum option and the option name for a basic option, with or +# without metadata. It never holds a language-specific name such as a Java class (java.lang.String), +# which Java writes for a basic option with metadata. Java writes a plain enum option with its +# option name. +ENUM_JSON = {'@type': f'{NS}.CurrencyEnum', '@data': 'USD'} +ENUM_META_JSON = {'@type': f'{NS}.CurrencyEnum', '@data': 'USD', '@scheme': 'urn:ccy'} +STRING_META_JSON = {'@type': 'string', '@data': 'foo', '@scheme': 'urn:x'} + + +def _read(body: dict, root: str = 'Root', validate: bool = True): + '''reads a document whose root type is in the test namespace''' + data = {'@type': f'{NS}.{root}', '@model': 'rosetta_dsl', '@version': '0.0.0', **body} + return BaseDataClass.rune_deserialize(json.dumps(data), validate_model=validate) + + +@pytest.mark.parametrize('attribute, value', [ + pytest.param('data', A_JSON, id='data-option'), + pytest.param('basic', {'@type': 'string', '@data': 'foo'}, id='string-option'), + pytest.param('basic', {'@type': 'int', '@data': 10}, id='int-option'), + pytest.param('nested', A_JSON, id='nested-choice'), + pytest.param('entries', [A_JSON, B_JSON], id='list'), + pytest.param('located', {'@key:scoped': 'outer', '@data': A_JSON}, + id='choice-attribute-with-metadata'), + pytest.param('currency', ENUM_JSON, id='enum-option'), + pytest.param('currency', {'@type': 'string', '@data': 'foo'}, id='string-option-beside-enum'), + pytest.param('currencyMeta', ENUM_META_JSON, id='enum-option-with-metadata'), + pytest.param('currencyMeta', STRING_META_JSON, id='string-option-with-metadata'), +]) +def test_round_trip(attribute, value): + '''Java's JSON is read, and written back unchanged''' + obj = _read({attribute: value}, validate=False) + assert json.loads(obj.rune_serialize(validate_model=False))[attribute] == value + + +def test_read_selects_the_option(): + '''@type selects the option''' + assert _read({'data': A_JSON}).data.A.fieldA == 'foo' + assert _read({'nested': B_JSON}).nested.ChoiceData.B.fieldB == 'bar' + assert _read({'basic': {'@type': 'string', '@data': 'foo'}}).basic.string == 'foo' + + +def test_write_object_built_in_python(): + '''an object built in Python is written in the same format''' + written = json.loads(Root(data=ChoiceData(A=A(fieldA='foo'))).rune_serialize()) + assert written['data'] == A_JSON + + +def test_write_enum_option_built_in_python(): + '''a plain enum option built in Python is written with the enum's Rune FQN as @type''' + written = json.loads(Root(currency=ChoiceEnum(CurrencyEnum=CurrencyEnum.USD)).rune_serialize()) + assert written['currency'] == ENUM_JSON + + +def test_reference_to_key_inside_choice(): + '''a key inside the selected option can be referenced from elsewhere''' + obj = _read({ + 'dated': {'@type': f'{NS}.WithDate', + 'paymentDate': {'@key:external': 'k1', 'day': 'Monday'}}, + 'dateRef': {'@ref:external': 'k1'}, + }, validate=False) + assert obj.dateRef is obj.dated.WithDate.paymentDate + + +def test_type_with_one_of_condition_keeps_attribute_names(): + '''a type that isn't a choice is read and written by attribute name''' + value = {'A': {'fieldA': 'foo'}} + obj = _read({'viaOneOf': value}) + assert obj.viaOneOf.A.fieldA == 'foo' + assert json.loads(obj.rune_serialize())['viaOneOf'] == value + + +@pytest.mark.parametrize('validate', [True, False]) +@pytest.mark.parametrize('attribute, value', [ + pytest.param('data', {'A': {'fieldA': 'foo'}}, id='nested-form'), + pytest.param('basic', {'string': 'foo'}, id='nested-form-basic'), + pytest.param('data', {'@type': f'{NS}.PaymentDate', 'day': 'Monday'}, id='not-an-option'), + # An enum option is selected only by its Rune FQN, a basic option only by its option name + pytest.param('currency', {'@type': 'CurrencyEnum', '@data': 'USD'}, + id='enum-option-with-option-name'), + pytest.param('currencyMeta', {'@type': 'CurrencyEnum', '@data': 'USD', '@scheme': 'urn:ccy'}, + id='enum-option-with-metadata-with-option-name'), + pytest.param('currencyMeta', {'@type': 'java.lang.String', '@data': 'foo', '@scheme': 'urn:x'}, + id='string-option-with-metadata-with-java-class-name'), +]) +def test_invalid_choice_is_rejected(attribute, value, validate): + '''only a declared option in the Rune JSON format is accepted''' + with pytest.raises(Exception): + _read({attribute: value}, validate=validate) + + +def test_bundled_choice_round_trip(): + '''a choice in a bundled class is read and written the same way''' + value = {'@type': f'{NS}.B', 'fieldB': 'bar'} + obj = _read({'option': value}, root='LoopHolder', validate=False) + assert isinstance(obj, LoopHolder) + assert obj.option.B.fieldB == 'bar' + assert json.loads(obj.rune_serialize(validate_model=False))['option'] == value + + +def test_bundled_list_of_choices_round_trip(): + '''a list of choices in a bundled class is read and written the same way. + + Fails until the runtime can write a list whose serializer wraps the whole list (#42, bug 4).''' + value = [{'@type': f'{NS}.B', 'fieldB': 'bar'}] + obj = _read({'options': value}, root='LoopHolder', validate=False) + assert json.loads(obj.rune_serialize(validate_model=False))['options'] == value + + +def test_b_option_is_written_with_its_own_type(): + '''@type names the selected option, not the first one''' + written = json.loads(Root(data=ChoiceData(B=B(fieldB='bar'))).rune_serialize()) + assert written['data'] == B_JSON diff --git a/python-test/unit-tests/features/serialization/test_choice_serialization.py b/python-test/unit-tests/features/serialization/test_choice_serialization.py index 788d2216..6b4dc234 100644 --- a/python-test/unit-tests/features/serialization/test_choice_serialization.py +++ b/python-test/unit-tests/features/serialization/test_choice_serialization.py @@ -12,18 +12,18 @@ from rune.runtime.base_data_class import BaseDataClass def test_choice_serialization(): - '''test choice serialization''' + '''a choice at the root is written as its selected option, and read back as that option''' a = A(value=1) choice_from_a_in = ChoiceSerializatonTest(A=a) serialized_a = choice_from_a_in.rune_serialize() choice_from_a_out = BaseDataClass.rune_deserialize(serialized_a) - assert choice_from_a_out == choice_from_a_in + assert choice_from_a_out == a b = B(value=2) choice_from_b_in = ChoiceSerializatonTest(B=b) serialized_b = choice_from_b_in.rune_serialize() choice_from_b_out = BaseDataClass.rune_deserialize(serialized_b) - assert choice_from_b_out == choice_from_b_in + assert choice_from_b_out == b if __name__ == "__main__": test_choice_serialization() \ No newline at end of file diff --git a/python-test/unit-tests/features/serialization/test_scoped_key_serialization.py b/python-test/unit-tests/features/serialization/test_scoped_key_serialization.py new file mode 100644 index 00000000..b8228717 --- /dev/null +++ b/python-test/unit-tests/features/serialization/test_scoped_key_serialization.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''scoped keys are written for standalone and bundled classes''' +import json + +import pytest + +from rune.runtime.base_data_class import BaseDataClass +# Imported so that the types can be resolved from @type when reading +import rosetta_dsl.test.serialization.scoped_key.BundledHolder # noqa: F401 +import rosetta_dsl.test.serialization.scoped_key.Holder # noqa: F401 + +NAMESPACE = 'rosetta_dsl.test.serialization.scoped_key' + + +@pytest.mark.parametrize('holder', ['Holder', 'BundledHolder']) +def test_scoped_key_round_trip(holder): + '''a scoped key and the reference to it are both written back''' + data = { + '@type': f'{NAMESPACE}.{holder}', + '@model': 'rosetta_dsl', + '@version': '0.0.0', + 'measure': {'@key:scoped': 'm1', 'value': 5}, + 'measureRef': {'@ref:scoped': 'm1'}, + } + obj = BaseDataClass.rune_deserialize(json.dumps(data)) + written = json.loads(obj.rune_serialize()) + assert written['measure'] == data['measure'] + assert written['measureRef'] == data['measureRef'] diff --git a/python-test/unit-tests/features/serialization/test_type_level_key_serialization.py b/python-test/unit-tests/features/serialization/test_type_level_key_serialization.py new file mode 100644 index 00000000..ff2d46c8 --- /dev/null +++ b/python-test/unit-tests/features/serialization/test_type_level_key_serialization.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC +# SPDX-License-Identifier: Apache-2.0 +# + +'''keys on a type with [metadata key] are written back''' +import json + +import pytest + +from rune.runtime.base_data_class import BaseDataClass +# Imported so that the types can be resolved from @type when reading +import rosetta_dsl.test.serialization.type_level_key.PartyList # noqa: F401 +import rosetta_dsl.test.serialization.type_level_key.SingleParty # noqa: F401 + +NAMESPACE = 'rosetta_dsl.test.serialization.type_level_key' +PARTY = {'@key:external': 'p1', 'name': 'A'} + + +@pytest.mark.parametrize('holder, attr, value', [ + ('SingleParty', 'party', PARTY), + ('PartyList', 'parties', [PARTY]), +]) +def test_type_level_key_round_trip(holder, attr, value): + '''a key read with the object is written back with it''' + data = { + '@type': f'{NAMESPACE}.{holder}', + '@model': 'rosetta_dsl', + '@version': '0.0.0', + attr: value, + } + obj = BaseDataClass.rune_deserialize(json.dumps(data)) + written = json.loads(obj.rune_serialize()) + assert written[attr] == value diff --git a/python-test/unit-tests/pytest.ini b/python-test/unit-tests/pytest.ini new file mode 100644 index 00000000..1c818fe4 --- /dev/null +++ b/python-test/unit-tests/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +# A generated annotation that resolves to an object instead of a type leaves its field +# unvalidated (#331). Fail the test run instead of only warning. Matched by message, since +# pydantic.warnings.ArbitraryTypeWarning doesn't exist in every supported pydantic version. +filterwarnings = + error:.*is not a Python type \(it may be an instance of an object\):UserWarning diff --git a/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGenerator.java b/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGenerator.java index 4e9565e5..c35f9239 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGenerator.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGenerator.java @@ -6,14 +6,17 @@ import static com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorConstants.INIT; import static com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorConstants.PYPROJECT_TOML; +import static com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorConstants.README_MD; import static com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorConstants.PYTHON; import static com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorConstants.SRC; +import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -25,7 +28,9 @@ import org.eclipse.emf.ecore.resource.Resource; import org.eclipse.emf.ecore.resource.ResourceSet; import org.jgrapht.Graph; +import org.jgrapht.Graphs; import org.jgrapht.alg.connectivity.KosarajuStrongConnectivityInspector; +import org.jgrapht.graph.AsSubgraph; import org.jgrapht.graph.DefaultDirectedGraph; import org.jgrapht.graph.DefaultEdge; import org.jgrapht.traverse.TopologicalOrderIterator; @@ -123,6 +128,9 @@ public final class PythonCodeGenerator extends AbstractExternalGenerator { */ private static final Logger LOGGER = LoggerFactory.getLogger(PythonCodeGenerator.class); + /** The start of a bundle annotation update to a field, up to the field's name. */ + private static final String MODEL_FIELDS_PREFIX = ".model_fields[\""; + /** * The contexts. */ @@ -294,6 +302,8 @@ public PythonCodeGenerator() { } } result.put(PYPROJECT_TOML, PythonCodeGeneratorUtil.createPYProjectTomlFile(null, cleanVersion, resolvedProjectName)); + // pyproject.toml names this file as the package description + result.put(README_MD, PythonCodeGeneratorUtil.createReadmeFile(resolvedProjectName, cleanVersion)); return result; } @@ -304,6 +314,16 @@ private void partitionClasses(PythonCodeGeneratorContext context) { List> sccs = inspector.stronglyConnectedSets(); context.setSccs(sccs); + // Cycles made only of function calls. Classes take part in other cycles, but conditions + // import the functions they call inside themselves, so only function-to-function calls + // can form a cycle of module-level imports. + Graph functionCalls = new AsSubgraph<>(dependencyDAG, + context.getFunctionNames().stream().filter(dependencyDAG::containsVertex) + .collect(Collectors.toSet())); + new KosarajuStrongConnectivityInspector<>(functionCalls).stronglyConnectedSets().stream() + .filter(cycle -> cycle.size() > 1) + .forEach(context::addFunctionCycle); + Set standaloneClasses = context.getStandaloneClasses(); for (Set scc : sccs) { if (scc.size() == 1) { @@ -327,11 +347,75 @@ private void partitionClasses(PythonCodeGeneratorContext context) { } } + // Each kind of promotion can create new bundled types for the other, so repeat until + // nothing changes. + Map superTypes = context.getSuperTypes(); + do { + promoteInheritanceRelatives(standaloneClasses, ownTypes, superTypes); + } while (promoteTypesOnPathsBetweenBundledTypes(dependencyDAG, standaloneClasses, ownTypes)); + } + + /** + * Promotes every own type that a bundled type depends on, and that itself depends on a bundled + * type, directly or through other types. + * + *

Cycle groups are found separately, so a path such as {@code GroupA1 -> Outer -> Inner -> + * GroupB1} is not a cycle in the dependency graph when {@code GroupA1} and {@code GroupB1} are in + * different groups. All bundled types share one {@code _bundle.py}, though, so at file level the + * bundle imports {@code Outer}, which imports {@code Inner}, which imports the bundle. If those + * types stayed standalone, importing one of them first would raise a circular ImportError, and + * importing the bundle first would build their schemas from the Phase 1 placeholders. + * + *

Paths may pass through functions and external types, but only own types can be promoted. + * + * @param dependencyDAG The type dependency graph, with edges from dependency to dependent. + * @param standaloneClasses The standalone classes (mutated). + * @param ownTypes The types belonging to this namespace. + * @return true if any type was promoted. + */ + private boolean promoteTypesOnPathsBetweenBundledTypes( + Graph dependencyDAG, + Set standaloneClasses, + Set ownTypes + ) { + Set bundledTypes = new HashSet<>(dependencyDAG.vertexSet()); + bundledTypes.retainAll(ownTypes); + bundledTypes.removeAll(standaloneClasses); + + Set toPromote = collectReachable(bundledTypes, node -> Graphs.predecessorListOf(dependencyDAG, node)); + toPromote.retainAll(collectReachable(bundledTypes, node -> Graphs.successorListOf(dependencyDAG, node))); + toPromote.retainAll(ownTypes); + toPromote.retainAll(standaloneClasses); + + toPromote.forEach(cls -> LOGGER.debug("Promoted {} to bundled (on a path between bundled types)", cls)); + return standaloneClasses.removeAll(toPromote); + } + + private Set collectReachable( + Set startNodes, + java.util.function.Function> neighbours + ) { + Set reachable = new HashSet<>(); + Deque pending = new ArrayDeque<>(startNodes); + while (!pending.isEmpty()) { + for (String neighbour : neighbours.apply(pending.pop())) { + if (reachable.add(neighbour)) { + pending.push(neighbour); + } + } + } + return reachable; + } + + private void promoteInheritanceRelatives( + Set standaloneClasses, + Set ownTypes, + Map superTypes + ) { // Promote to bundled: any own-type class whose bundled ancestor is also an own type. // Pydantic's model_rebuild(force=True) does not propagate parent annotation changes to // a child's model_fields — each subclass must be in the bundle to receive explicit // Phase 2 and Phase 3 treatment for inherited deferred fields. - Map superTypes = context.getSuperTypes(); boolean anyPromotion = true; while (anyPromotion) { anyPromotion = false; @@ -438,20 +522,17 @@ private Map processDAG( // defined by the time Phase 3 runs, and Pydantic's schema builder can resolve // cross-type references in a single pass (~4× faster than without defer_build). // - // Standalone classes with deferred imports are integrated into the rebuild graph - // so they are ordered correctly relative to bundled classes. - // Function imports from conditions are not type classes — do NOT pass to - // integrateStandaloneRebuilds, which would add them to pendingRebuilds and - // generate model_rebuild() calls on Python function objects. - integrateStandaloneRebuilds(context, headerResult.deferredStandaloneImports(), pendingRebuilds); + // Standalone classes are not rebuilt. promoteTypesOnPathsBetweenBundledTypes guarantees + // that a standalone class the bundle imports never depends on a bundled class, so it + // cannot capture a Phase 1 placeholder, and defer_build=True postpones its schema until + // first use. Rebuilding it here would force the build that defer_build avoids. String rebuildContent = emitRebuildCallsInOrder(pendingRebuilds, context); - // Combine type and function imports for the deferred-import section of the bundle. - List allDeferredImports = new ArrayList<>(headerResult.deferredStandaloneImports()); - allDeferredImports.addAll(headerResult.deferredFunctionImports()); + // Functions called by conditions are imported inside each condition, not here: a + // module-level import would make the bundle import a function that imports the bundle. assembleBundleFile(nameSpace, context, bundleWriter, dataObjectsWriter, functionsWriter, - annotationUpdateWriter, rebuildContent, allDeferredImports, result); + annotationUpdateWriter, rebuildContent, headerResult.deferredStandaloneImports(), result); return result; } @@ -519,9 +600,7 @@ private BundleHeaderResult buildBundleHeader(PythonCodeGeneratorContext context, } } - List deferredFunctionImports = new ArrayList<>(context.getBundleConditionFunctionImports()); - - return new BundleHeaderResult(deferredStandaloneImports, deferredFunctionImports, standaloneSupertypesOfBundled); + return new BundleHeaderResult(deferredStandaloneImports, standaloneSupertypesOfBundled); } /** @@ -783,12 +862,13 @@ private void propagateInheritedPhase2Updates(PythonCodeGeneratorContext context) } private String extractModelFieldsName(String updateLine) { - int start = updateLine.indexOf(".model_fields[\""); + int start = updateLine.indexOf(MODEL_FIELDS_PREFIX); if (start < 0) { return null; } - int end = updateLine.indexOf("\"", start + 15); - return end >= 0 ? updateLine.substring(start + 15, end) : null; + int nameStart = start + MODEL_FIELDS_PREFIX.length(); + int end = updateLine.indexOf("\"", nameStart); + return end >= 0 ? updateLine.substring(nameStart, end) : null; } private Set extractFieldNamesFromUpdates(List updates) { @@ -804,7 +884,7 @@ private Set extractFieldNamesFromUpdates(List updates) { private boolean isUpdateLineForFields(String line, Set fields, String bundleName) { for (String field : fields) { - if (line.startsWith(bundleName + ".model_fields[\"" + field + "\"]") + if (line.startsWith(bundleName + MODEL_FIELDS_PREFIX + field + "\"]") || line.startsWith(bundleName + ".__annotations__[\"" + field + "\"]")) { return true; } @@ -812,77 +892,6 @@ private boolean isUpdateLineForFields(String line, Set fields, String bu return false; } - /** - * Adds deferred standalone imports into the rebuild graph so they are ordered correctly - * inside Phase 3 rather than being relegated to an unordered Phase 4 after all bundled - * rebuilds. - * - *

Two sets of edges are added: - *

    - *
  1. S → bundled dep: standalone class S must rebuild after every bundled - * class it references (derived from the dependency DAG).
  2. - *
  3. bundled B → S: bundled class B must rebuild after standalone S - * whenever B's Phase 2 annotation strings mention S's class name (meaning B inlines - * S's schema at rebuild time).
  4. - *
- * - * @param context The generator context (DAG, post-definition updates, standalone set). - * @param deferredImports List of {@code "from import "} import strings. - * @param pendingRebuilds The rebuild list (mutated: standalone class names are appended). - */ - private void integrateStandaloneRebuilds( - PythonCodeGeneratorContext context, - List deferredImports, - List pendingRebuilds - ) { - if (deferredImports.isEmpty()) { - return; - } - Set standaloneClasses = context.getStandaloneClasses(); - Map> updates = context.getPostDefinitionUpdates(); - Graph dag = context.getDependencyDAG(); - - Set pendingSet = new HashSet<>(pendingRebuilds); - - for (String imp : deferredImports) { - int importIdx = imp.lastIndexOf(" import "); - if (importIdx < 0) continue; - String className = imp.substring(importIdx + " import ".length()).trim(); - String fqn = imp.substring("from ".length(), importIdx).trim(); - - // Add standalone class to pendingRebuilds if not already present - if (pendingSet.add(className)) { - pendingRebuilds.add(className); - } - - // Dep 1: standalone S must rebuild AFTER bundled types it directly references. - // The DAG convention is addEdge(dependency, dependent), so INCOMING edges to S's - // vertex are the types S depends on (S's field types), while OUTGOING edges are - // classes that depend on S. - if (dag.containsVertex(fqn)) { - for (DefaultEdge edge : dag.incomingEdgesOf(fqn)) { - String depFqn = dag.getEdgeSource(edge); - if (!standaloneClasses.contains(depFqn)) { - // depFqn is a bundled type that S references — S must rebuild after it - context.addRebuildDep(className, getBundleClassName(depFqn)); - } - } - } - - // Dep 2: bundled class B must rebuild AFTER standalone S if B's Phase 2 - // annotation strings reference S by its imported name. - for (Map.Entry> entry : updates.entrySet()) { - String bundleClass = entry.getKey(); - for (String updateLine : entry.getValue()) { - if (updateLine.contains(className)) { - context.addRebuildDep(bundleClass, className); - break; - } - } - } - } - } - /** * Sorts and emits the Phase 3 {@code model_rebuild(force=True)} calls in dependency order. * @@ -1096,6 +1105,10 @@ private void emitStandaloneFile( String depName = dependencyDAG.getEdgeSource(edge); boolean isClassDep = context.getClassObjects().containsKey(depName); boolean isFuncDep = context.getFunctionObjects().containsKey(depName); + if (isFuncDep && context.isInFunctionCycle(name, depName)) { + // Imported inside the calling function instead + continue; + } if (isClassDep || (context.getStandaloneClasses().contains(depName) && isFuncDep)) { // Data class dep (standalone or bundled) or standalone function dep: // standalone function signatures use the simple class name, so import @@ -1256,7 +1269,6 @@ private void assembleBundleFile( */ private record BundleHeaderResult( List deferredStandaloneImports, - List deferredFunctionImports, Set standaloneSupertypesOfBundled) { } diff --git a/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorContext.java b/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorContext.java index 5d8cfb13..01b05f99 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorContext.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/PythonCodeGeneratorContext.java @@ -105,18 +105,6 @@ public final class PythonCodeGeneratorContext { * calls in dependency order. */ private Map> rebuildDeps = null; - /** - * Deferred imports for standalone functions referenced from bundle condition methods - * (insertion-ordered for deterministic output). Each entry is a complete import statement, - * e.g. "from com.rosetta.model.Func import Func". - */ - private LinkedHashSet bundleConditionFunctionImports = null; - /** - * Set to true while generating conditions for a bundled (non-standalone) type. - * Guards {@link #addBundleConditionFunctionImport} so that function calls inside - * standalone function bodies do not pollute the bundle's deferred import list. - */ - private boolean generatingBundledTypeCondition = false; public PythonCodeGeneratorContext() { this.subfolders = new LinkedHashSet<>(); @@ -136,7 +124,6 @@ public PythonCodeGeneratorContext() { this.superTypes = new HashMap<>(); this.functionEnumImports = new HashMap<>(); this.rebuildDeps = new HashMap<>(); - this.bundleConditionFunctionImports = new LinkedHashSet<>(); } public List getSubfolders() { @@ -246,6 +233,27 @@ public Map getSuperTypes() { return superTypes; } + /** + * For each function in a cycle of function calls, the functions in that cycle. A call + * between them is imported inside the calling function, because importing at module level + * would make each module import the other while it is still loading. + */ + private final Map> functionCycles = new HashMap<>(); + + public void addFunctionCycle(Set cycle) { + cycle.forEach(function -> functionCycles.put(function, cycle)); + } + + /** + * @param caller the calling function's name + * @param callee the called function's name + * @return true if {@code caller} and {@code callee} are different functions in the same + * cycle of function calls + */ + public boolean isInFunctionCycle(String caller, String callee) { + return !caller.equals(callee) && functionCycles.getOrDefault(caller, Set.of()).contains(callee); + } + public List> getSccs() { return sccs; } @@ -264,20 +272,6 @@ public Map> getRebuildDeps() { return rebuildDeps; } - public void setGeneratingBundledTypeCondition(boolean value) { - this.generatingBundledTypeCondition = value; - } - - public void addBundleConditionFunctionImport(String fqn, String name) { - if (generatingBundledTypeCondition) { - bundleConditionFunctionImports.add("from " + fqn + " import " + name); - } - } - - public Set getBundleConditionFunctionImports() { - return bundleConditionFunctionImports; - } - public String getNamespacePrefix() { return namespacePrefix; } diff --git a/src/main/java/com/regnosys/rosetta/generator/python/enums/PythonEnumGenerator.java b/src/main/java/com/regnosys/rosetta/generator/python/enums/PythonEnumGenerator.java index 2fb0070e..5c0e3979 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/enums/PythonEnumGenerator.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/enums/PythonEnumGenerator.java @@ -30,7 +30,7 @@ public class PythonEnumGenerator { public Map generate(Iterable rosettaEnums, PythonCodeGeneratorContext context) { Map result = new HashMap<>(); for (RosettaEnumeration enumeration : rosettaEnums) { - RosettaModel tr = (RosettaModel) enumeration.eContainer(); + RosettaModel tr = enumeration.getModel(); String namespace = context.applyPrefix(tr.getName()); PythonCodeWriter writer = new PythonCodeWriter(); diff --git a/src/main/java/com/regnosys/rosetta/generator/python/expressions/PythonExpressionGenerator.java b/src/main/java/com/regnosys/rosetta/generator/python/expressions/PythonExpressionGenerator.java index bb70627e..ccfc02e6 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/expressions/PythonExpressionGenerator.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/expressions/PythonExpressionGenerator.java @@ -5,13 +5,17 @@ package com.regnosys.rosetta.generator.python.expressions; import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import com.regnosys.rosetta.generator.java.enums.EnumHelper; import com.regnosys.rosetta.generator.python.util.PythonCodeWriter; import com.regnosys.rosetta.generator.python.util.RuneToPythonMapper; import com.regnosys.rosetta.rosetta.RosettaCallableWithArgs; +import com.regnosys.rosetta.rosetta.RosettaNamed; import com.regnosys.rosetta.rosetta.RosettaEnumValue; import com.regnosys.rosetta.rosetta.RosettaEnumValueReference; import com.regnosys.rosetta.rosetta.RosettaEnumeration; @@ -75,7 +79,6 @@ import com.regnosys.rosetta.rosetta.simple.Condition; import com.regnosys.rosetta.rosetta.simple.Data; import com.regnosys.rosetta.rosetta.simple.ShortcutDeclaration; -import com.regnosys.rosetta.rosetta.simple.impl.FunctionImpl; import com.regnosys.rosetta.types.CardinalityProvider; import com.regnosys.rosetta.types.RChoiceType; import com.regnosys.rosetta.types.RDataType; @@ -83,6 +86,7 @@ import com.regnosys.rosetta.types.RMetaAnnotatedType; import com.regnosys.rosetta.types.RType; import com.regnosys.rosetta.types.RosettaTypeProvider; +import com.regnosys.rosetta.types.TypeSystem; import com.regnosys.rosetta.generator.python.PythonCodeGeneratorContext; import com.regnosys.rosetta.generator.python.util.PythonCodeGeneratorUtil; @@ -114,6 +118,12 @@ public record ExpressionResult(String expression, List companionBlocks) @Inject private RosettaTypeProvider typeProvider; + /** + * Strips type aliases, to tell an enum argument of to-enum from a string one. + */ + @Inject + private TypeSystem typeSystem; + /** * The cardinality provider. */ @@ -125,9 +135,56 @@ public record ExpressionResult(String expression, List companionBlocks) */ private int generatedFunctionCounter = 0; + /** + * Imports for the functions called and types built by the class condition being generated, or + * null when no class condition is being generated. Each is emitted inside that condition, so it + * only runs once all modules have loaded, whichever module was imported first. + */ + private Set conditionImports = null; + + /** + * True when generating code for a standalone file rather than for _bundle.py. A standalone + * file refers to a bundled type by its stub name; only _bundle.py defines the bundle name. + */ + private boolean writingStandaloneFile = false; + + /** + * The inputs of the function being generated, which are read from the function's frame rather + * than from the current item. Empty when generating a class. + */ + private Set functionInputs = Set.of(); + + /** The function being generated, or null when generating a class. */ + private String currentFunction = null; + + /** + * Imports for calls to functions in the same cycle of function calls as the function being + * generated. They are written inside that function, rather than at the top of its module. + */ + private final Set functionLocalImports = new LinkedHashSet<>(); + /** The Python code generator context. */ private PythonCodeGeneratorContext context; + public void setWritingStandaloneFile(boolean writingStandaloneFile) { + this.writingStandaloneFile = writingStandaloneFile; + } + + public void setCurrentFunction(String functionFqn) { + this.currentFunction = functionFqn; + } + + /** + * @return the imports to write inside the function being generated + */ + public Set getFunctionLocalImports() { + return functionLocalImports; + } + + public void setFunctionInputs(Collection functionInputs) { + this.functionInputs = Set.copyOf(functionInputs); + } + public void setContext(PythonCodeGeneratorContext context) { this.context = context; } @@ -238,8 +295,7 @@ private String generateExpression(RosettaExpression expr, return generateSwitchOperation(switchOp, scope); } case ToEnumOperation toEnum -> { - return toEnum.getEnumeration().getName() + "(" - + generateExpression(toEnum.getArgument(), scope) + ")"; + return generateToEnumOperation(toEnum, scope); } case ToStringOperation toString -> { return "rune_str(" + generateExpression(toString.getArgument(), scope) + ")"; @@ -374,7 +430,7 @@ private String generateAsOperation(AsOperation expr, PythonExpressionScope scope // Narrowed values may still be wrapped in a copy-on-write proxy (rune_cow), which is // not a subclass of the wrapped type, so the isinstance check must unwrap first. - String targetTypeName = ((Data) expr.getType()).getName(); + String targetTypeName = typeReference((Data) expr.getType()); if (isMulti) { return "[_x for _x in (" + arg + " or []) if isinstance(rune_unwrap(_x), " + targetTypeName + ")]"; } @@ -431,7 +487,13 @@ private String generateConditionalExpression(RosettaConditionalExpression expr, private String generateFeatureCall(RosettaFeatureCall expr, PythonExpressionScope scope) { if (expr.getFeature() instanceof RosettaEnumValue evalue) { - return generateEnumString(evalue); + // Name the enum written in the model: for an extended enum, the declaring parent + // is a different Python Enum, whose members don't compare equal. + if (expr.getReceiver() instanceof RosettaSymbolReference receiverRef + && receiverRef.getSymbol() instanceof RosettaEnumeration receiverEnum) { + return generateEnumString(evalue, receiverEnum.getModel().getName(), receiverEnum.getName()); + } + return generateEnumString(evalue, typeProvider.getRMetaAnnotatedType(expr).getRType()); } if (expr.getFeature() instanceof RosettaMetaType metaType) { String metaDataName = PythonCodeGeneratorUtil.mapMetaTypeToMetaDataName(metaType.getName()); @@ -517,7 +579,9 @@ private String tryGenerateReferenceFilterChain( String param = thenFunc.getParameters().isEmpty() ? "item" : thenFunc.getParameters().get(0).getName(); String rawCollection = generateExpression(mapOp.getArgument(), scope); - String rawArg = "[x for x in " + rawCollection + " or [] if x is not None]"; + String rawItems = cardinalityProvider.isMulti(mapOp.getArgument()) + ? rawCollection + " or []" : "[" + rawCollection + "]"; + String rawArg = "[x for x in " + rawItems + " if x is not None]"; String fieldName = refAttr.getName(); String resolveCall = param + ".resolve_ref_key(\"" + fieldName + "\")"; @@ -580,6 +644,11 @@ private String generateFilterOperation(FilterOperation expr, PythonExpressionSco } String filterExpression = generateExpression(expr.getFunction().getBody(), subScope); + if (!cardinalityProvider.isMulti(expr.getArgument())) { + // A single value isn't iterable, and the result stays a single value + return "(lambda " + param + ": " + param + " if " + param + " is not None and (" + filterExpression + + ") else None)(" + argument + ")"; + } return "rune_filter(" + argument + ", lambda " + param + ": " + filterExpression + ")"; } @@ -594,6 +663,10 @@ private String generateMapOperation(MapOperation expr, PythonExpressionScope sco } String funcBody = generateExpression(inlineFunc.getBody(), subScope); + if (!cardinalityProvider.isMulti(expr.getArgument())) { + // A single value isn't iterable, and the result stays a single value + return "(lambda " + param + ": None if " + param + " is None else " + funcBody + ")(" + argument + ")"; + } String lambdaFunction = "lambda " + param + ": " + funcBody; return "[x for x in map(" + lambdaFunction + ", " + argument + " or []) if x is not None]"; } @@ -602,8 +675,7 @@ private String generateConstructorExpression(RosettaConstructorExpression expr, String type = null; if (expr.getTypeCall() != null && expr.getTypeCall().getType() != null) { com.regnosys.rosetta.rosetta.RosettaNamed typeNamed = expr.getTypeCall().getType(); - String fqn = context.getFullyQualifiedName(typeNamed); - type = context.getStandaloneClasses().contains(fqn) ? typeNamed.getName() : context.getBundleObjectName(typeNamed); + type = typeReference(typeNamed); } if (type != null) { return type + "(" + expr.getValues().stream() @@ -628,7 +700,8 @@ private String generateConstructorExpression(RosettaConstructorExpression expr, } } - private String getGuardExpression(SwitchCaseGuard caseGuard, PythonExpressionScope scope) { + private String getGuardExpression(SwitchCaseGuard caseGuard, RosettaExpression switchArgument, + PythonExpressionScope scope) { if (caseGuard == null) { throw new UnsupportedOperationException("Null SwitchCaseGuard"); } @@ -638,8 +711,8 @@ private String getGuardExpression(SwitchCaseGuard caseGuard, PythonExpressionSco } RosettaEnumValue enumGuard = caseGuard.getEnumGuard(); if (enumGuard != null) { - return "switchAttribute == rune_resolve_attr(" + generateEnumString(enumGuard) + ",\"" - + enumGuard.getName() + "\")"; + RType argumentType = typeProvider.getRMetaAnnotatedType(switchArgument).getRType(); + return "switchAttribute == " + generateEnumString(enumGuard, argumentType); } RosettaFeature optionGuard = caseGuard.getChoiceOptionGuard(); if (optionGuard != null) { @@ -660,7 +733,7 @@ private String generateSymbolReference(RosettaSymbolReference expr, PythonExpres } else if (symbol instanceof Attribute attr) { return generateAttributeReference(attr, scope); } else if (symbol instanceof RosettaEnumValue evalue) { - return generateEnumString(evalue); + return generateEnumString(evalue, typeProvider.getRMetaAnnotatedType(expr).getRType()); } else if (symbol instanceof RosettaCallableWithArgs callable) { return generateCallableWithArgsCall(callable, expr, scope); } else if (symbol instanceof ClosureParameter) { @@ -681,18 +754,48 @@ private String generateAttributeReference(Attribute s, PythonExpressionScope sco if (scope.shadowedSymbols().containsKey(s)) { return scope.shadowedSymbols().get(s); } - boolean isInput = s.eContainer() instanceof FunctionImpl c - && c.getInputs().stream().anyMatch(attr -> attr.getName().equals(s.getName())); - String receiver = isInput ? "self" : scope.receiver(); + String receiver = functionInputs.contains(s) ? "self" : scope.receiver(); return "rune_resolve_attr(" + receiver + ", \"" + s.getName() + "\")"; } + /** + * A conversion that gives nothing when the input matches no value, as Rune specifies and the + * Java generator does. A string matches by display name, which the generated Python enums + * store as each member's value. Another enum matches by value name, as Java's + * {@code valueOf(e.name())} does; {@code _EnumWrapper.name} passes through to the member. + */ + private String generateToEnumOperation(ToEnumOperation toEnum, PythonExpressionScope scope) { + RosettaEnumeration enumeration = toEnum.getEnumeration(); + String enumName = context.applyPrefix(enumeration.getModel().getName()) + "." + + enumeration.getName() + "." + enumeration.getName(); + String argument = generateExpression(toEnum.getArgument(), scope); + boolean argumentIsEnum = typeSystem.stripFromTypeAliases( + typeProvider.getRMetaAnnotatedType(toEnum.getArgument()).getRType()) instanceof REnumType; + String lookup = argumentIsEnum + ? enumName + ".__members__.get(rune_unwrap(v).name)" + : "next((m for m in " + enumName + " if m.value == v), None)"; + return "(lambda v: None if v is None else " + lookup + ")(" + argument + ")"; + } + private String generateEnumString(RosettaEnumValue rev) { - String value = EnumHelper.convertValue(rev); RosettaEnumeration parent = rev.getEnumeration(); - String parentName = parent.getName(); - String modelName = context.applyPrefix(parent.getModel().getName()); - return modelName + "." + parentName + "." + parentName + "." + value; + return generateEnumString(rev, parent.getModel().getName(), parent.getName()); + } + + /** + * Names an enum value through the enum type of the expression, if known. For a value that an + * extended enum inherits, that's the extended enum, not the parent that declares the value. + */ + private String generateEnumString(RosettaEnumValue rev, RType referencedType) { + if (referencedType instanceof REnumType enumType) { + return generateEnumString(rev, enumType.getNamespace().toString(), enumType.getName()); + } + return generateEnumString(rev); + } + + private String generateEnumString(RosettaEnumValue rev, String namespace, String enumName) { + String value = EnumHelper.convertValue(rev); + return context.applyPrefix(namespace) + "." + enumName + "." + enumName + "." + value; } private String generateCallableWithArgsCall(RosettaCallableWithArgs s, RosettaSymbolReference expr, @@ -707,7 +810,12 @@ private String generateCallableWithArgsCall(RosettaCallableWithArgs s, RosettaSy case null, default -> { String fqn = context.getFullyQualifiedName(s); if (context.getStandaloneClasses().contains(fqn)) { - context.addBundleConditionFunctionImport(fqn, s.getName()); + if (conditionImports != null) { + conditionImports.add("from " + fqn + " import " + s.getName()); + } + if (currentFunction != null && context.isInFunctionCycle(currentFunction, fqn)) { + functionLocalImports.add("from " + fqn + " import " + s.getName()); + } yield s.getName(); } else { yield context.getBundleObjectName(s); @@ -768,12 +876,13 @@ public String generateTypeConditions(Data cls) { int nConditions = 0; StringBuilder result = new StringBuilder(); for (Condition cond : cls.getConditions()) { - result.append(generateConditionBoilerPlate(cond, nConditions)); - if (isConstraintCondition(cond)) { - result.append(generateConstraintCondition(cls, cond)); - } else { - result.append(generateIfThenElseOrSwitch(cond)); - } + conditionImports = new LinkedHashSet<>(); + String body = isConstraintCondition(cond) + ? generateConstraintCondition(cls, cond) + : generateIfThenElseOrSwitch(cond); + result.append(generateConditionBoilerPlate(cond, nConditions, conditionImports)); + result.append(body); + conditionImports = null; nConditions++; } return result.toString(); @@ -804,7 +913,7 @@ private boolean isChoice(Condition cond) { return cond.getExpression() instanceof ChoiceOperation; } - private String generateConditionBoilerPlate(Condition cond, int nConditions) { + private String generateConditionBoilerPlate(Condition cond, int nConditions, Set imports) { PythonCodeWriter writer = new PythonCodeWriter(); writer.newLine(); writer.appendLine("@rune_condition"); @@ -816,6 +925,7 @@ private String generateConditionBoilerPlate(Condition cond, int nConditions) { writer.appendLine(cond.getDefinition()); writer.appendLine("\"\"\""); } + imports.forEach(writer::appendLine); writer.appendLine("item = self"); return writer.toString(); } @@ -868,8 +978,16 @@ private String generateIfThenElseOrSwitch(Condition c) { // ... (helper methods like isConstraintCondition, etc. remain unchanged) ... + /** + * A switch tries its cases in order, as Rune's Java generator does. An absent value gives + * nothing. A case on a type or on a choice option passes the matched value to its body as + * {@code item}, so the body can use that value's attributes directly. + */ private String generateSwitchOperation(SwitchOperation expr, PythonExpressionScope scope) { String attr = generateExpression(expr.getArgument(), scope); + RType argumentType = typeSystem.stripFromTypeAliases( + typeProvider.getRMetaAnnotatedType(expr.getArgument()).getRType()); + boolean narrows = argumentType instanceof RChoiceType || argumentType instanceof RDataType; PythonCodeWriter writer = new PythonCodeWriter(); String switchFuncName = "_switch_fn_" + generatedFunctionCounter++; @@ -880,30 +998,66 @@ private String generateSwitchOperation(SwitchOperation expr, PythonExpressionSco var cases = expr.getCases(); for (int i = 0; i < cases.size(); i++) { var currentCase = cases.get(i); - String funcName = currentCase.isDefault() ? "_then_default" : "_then_" + (i + 1); - String thenExprDef = currentCase.isDefault() ? generateExpression(expr.getDefault(), scope) - : generateExpression(currentCase.getExpression(), scope); - - writer.appendLine("def " + funcName + "():"); + boolean isDefault = currentCase.isDefault(); + String funcName = isDefault ? "_then_default" : "_then_" + (i + 1); + String thenExprDef; + if (isDefault) { + thenExprDef = generateExpression(expr.getDefault(), scope); + writer.appendLine("def " + funcName + "():"); + } else if (narrows) { + thenExprDef = generateExpression(currentCase.getExpression(), scope.withReceiver("item")); + writer.appendLine("def " + funcName + "(item):"); + } else { + thenExprDef = generateExpression(currentCase.getExpression(), scope); + writer.appendLine("def " + funcName + "():"); + } writer.indent(); writer.appendLine("return " + thenExprDef); writer.unindent(); } writer.appendLine("switchAttribute = " + attr); + writer.appendLine("if switchAttribute is None:"); + writer.indent(); + writer.appendLine("return None"); + writer.unindent(); for (int i = 0; i < cases.size(); i++) { var currentCase = cases.get(i); - String funcName = currentCase.isDefault() ? "_then_default" : "_then_" + (i + 1); if (currentCase.isDefault()) { writer.appendLine("else:"); + writer.indent(); + writer.appendLine("return _then_default()"); + writer.unindent(); + continue; + } + String funcName = "_then_" + (i + 1); + String prefix = (i == 0) ? "if " : "elif "; + SwitchCaseGuard guard = currentCase.getGuard(); + String call; + if (argumentType instanceof RChoiceType choiceType && guard.getChoiceOptionGuard() != null) { + // The option may be in a nested choice, so follow the path of options to it + List path = findChoiceOptionPath(choiceType, guard.getChoiceOptionGuard()); + if (path == null) { + path = List.of(guard.getChoiceOptionGuard().getName()); + } + String option = "switchAttribute"; + for (String step : path) { + option = "rune_resolve_attr(" + option + ", \"" + step + "\")"; + } + writer.appendLine(prefix + "(caseValue := " + option + ") is not None:"); + call = funcName + "(caseValue)"; + } else if (guard.getDataGuard() != null) { + // Copy-on-write proxies aren't subclasses of the type they wrap, so unwrap first + writer.appendLine(prefix + "isinstance(rune_unwrap(switchAttribute), " + + typeReference(guard.getDataGuard()) + "):"); + call = funcName + "(switchAttribute)"; } else { - SwitchCaseGuard guard = currentCase.getGuard(); - String prefix = (i == 0) ? "if " : "elif "; - writer.appendLine(prefix + getGuardExpression(guard, scope) + ":"); + writer.appendLine(prefix + getGuardExpression(guard, expr.getArgument(), scope) + ":"); + call = narrows ? funcName + "(switchAttribute)" : funcName + "()"; } writer.indent(); - writer.appendLine("return " + funcName + "()"); + writer.appendLine("return " + call); writer.unindent(); } @@ -913,6 +1067,22 @@ private String generateSwitchOperation(SwitchOperation expr, PythonExpressionSco return switchFuncName + "()"; } + /** + * The name to use for a type in generated code: the stub name for a bundled type in a + * standalone file, the bundle name inside _bundle.py, or the type's own name if it is + * standalone. Inside a class condition, the name is also imported there. + */ + private String typeReference(RosettaNamed type) { + String fqn = context.getFullyQualifiedName(type); + if (writingStandaloneFile || context.getStandaloneClasses().contains(fqn)) { + if (conditionImports != null) { + conditionImports.add("from " + fqn + " import " + type.getName()); + } + return type.getName(); + } + return context.getBundleObjectName(type); + } + private String generateWithMetaOperation(WithMetaOperation expr, PythonExpressionScope scope) { RType argType = typeProvider.getRMetaAnnotatedType(expr.getArgument()).getRType(); String arg = generateExpression(expr.getArgument(), scope); @@ -936,7 +1106,7 @@ private String generateWithMetaOperation(WithMetaOperation expr, PythonExpressio // are referenced elsewhere in function bodies (namespace.EnumName.EnumName). if (argType instanceof REnumType) { String enumName = argType.getName(); - String enumType = argType.getNamespace().toString() + "." + enumName + "." + enumName; + String enumType = context.applyPrefix(argType.getNamespace().toString()) + "." + enumName + "." + enumName; List mappedEntries = expr.getEntries().stream() .map(entry -> { String mappedKey = switch (entry.getKey().getName()) { @@ -1034,7 +1204,9 @@ private String generateReverseOperation(ReverseOperation reverse, PythonExpressi private String generateCountOperation(RosettaCountOperation count, PythonExpressionScope scope) { String arg = generateExpression(count.getArgument(), scope); - return "(lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))(" + arg + ")"; + // Models define __iter__ over their fields, and function inputs are proxies that + // appear to have it, so a single model object must not be treated as a collection. + return "(lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))(" + arg + ")"; } private String generateExistsOperation(RosettaExistsExpression exists, PythonExpressionScope scope) { diff --git a/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionDependencyProvider.java b/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionDependencyProvider.java index 9e6f9662..7173eed1 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionDependencyProvider.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionDependencyProvider.java @@ -4,44 +4,24 @@ */ package com.regnosys.rosetta.generator.python.functions; -import java.util.Collections; -import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import com.regnosys.rosetta.rosetta.RosettaEnumValueReference; +import com.regnosys.rosetta.rosetta.RosettaEnumValue; import com.regnosys.rosetta.rosetta.RosettaEnumeration; import com.regnosys.rosetta.rosetta.RosettaModel; import com.regnosys.rosetta.rosetta.RosettaNamed; -import com.regnosys.rosetta.rosetta.RosettaRule; -import com.regnosys.rosetta.rosetta.RosettaSymbol; -import com.regnosys.rosetta.rosetta.expression.FilterOperation; import com.regnosys.rosetta.rosetta.expression.InlineFunction; -import com.regnosys.rosetta.rosetta.expression.ListLiteral; -import com.regnosys.rosetta.rosetta.expression.MapOperation; -import com.regnosys.rosetta.rosetta.expression.RosettaBinaryOperation; -import com.regnosys.rosetta.rosetta.expression.RosettaConditionalExpression; -import com.regnosys.rosetta.rosetta.expression.RosettaConstructorExpression; import com.regnosys.rosetta.rosetta.expression.RosettaExpression; -import com.regnosys.rosetta.rosetta.expression.RosettaFeatureCall; -import com.regnosys.rosetta.rosetta.expression.RosettaFunctionalOperation; -import com.regnosys.rosetta.rosetta.expression.RosettaOnlyExistsExpression; -import com.regnosys.rosetta.rosetta.expression.RosettaSymbolReference; -import com.regnosys.rosetta.rosetta.expression.RosettaUnaryOperation; -import com.regnosys.rosetta.rosetta.expression.WithMetaOperation; -import com.regnosys.rosetta.rosetta.simple.Function; import com.regnosys.rosetta.types.RAliasType; import com.regnosys.rosetta.types.RDataType; import com.regnosys.rosetta.types.REnumType; -import com.regnosys.rosetta.types.RFunction; -import com.regnosys.rosetta.types.RObjectFactory; import com.regnosys.rosetta.types.RType; +import com.regnosys.rosetta.types.RosettaTypeProvider; import com.regnosys.rosetta.types.TypeSystem; import com.regnosys.rosetta.generator.python.PythonCodeGeneratorContext; +import com.regnosys.rosetta.generator.python.util.PythonDependencyCollector; import jakarta.inject.Inject; /** @@ -49,25 +29,32 @@ */ public final class PythonFunctionDependencyProvider { /** - * RObjectFactory is used to build RType objects from Rosetta objects. + * Collects the functions, types and enums an expression refers to. */ @Inject - private RObjectFactory rTypeBuilderFactory; + private PythonDependencyCollector dependencyCollector; /** The Rosetta type system. */ @Inject private TypeSystem typeSystem; + /** Gives the enum type an expression refers to, which may extend the enum declaring a value. */ + @Inject + private RosettaTypeProvider typeProvider; + public void addDependencies(RosettaNamed named, Set enumImports, String sourceNamespace, PythonCodeGeneratorContext context) { if (named instanceof RosettaEnumeration enumeration) { String name = enumeration.getName(); - RosettaModel model = (RosettaModel) enumeration.eContainer(); + RosettaModel model = enumeration.getModel(); String prefix = context.applyPrefix(model.getName()); enumImports.add("import " + prefix + "." + name); + } else if (named instanceof RosettaEnumValue enumValue) { + // A value written without its type, such as `AssetPrice`, still needs its enum + addDependencies(enumValue.getEnumeration(), enumImports, sourceNamespace, context); } else if (named instanceof com.regnosys.rosetta.rosetta.simple.Data data) { String name = data.getName(); - RosettaModel model = (RosettaModel) data.eContainer(); + RosettaModel model = data.getModel(); String fullNamespace = context.applyPrefix(model.getName()); String rootNamespace = fullNamespace.split("\\.")[0]; String sourceRoot = sourceNamespace.split("\\.")[0]; @@ -92,77 +79,12 @@ public void addDependencies(InlineFunction inline, Set enumImports, Stri public void addDependencies(RosettaExpression expression, Set enumImports, String sourceNamespace, PythonCodeGeneratorContext context) { - if (expression == null) { - return; - } - switch (expression) { - case RosettaBinaryOperation binary -> { - addDependencies(binary.getLeft(), enumImports, sourceNamespace, context); - addDependencies(binary.getRight(), enumImports, sourceNamespace, context); - } - case RosettaConditionalExpression cond -> { - addDependencies(cond.getIf(), enumImports, sourceNamespace, context); - addDependencies(cond.getIfthen(), enumImports, sourceNamespace, context); - addDependencies(cond.getElsethen(), enumImports, sourceNamespace, context); - } - case RosettaOnlyExistsExpression onlyExists -> { - onlyExists.getArgs().forEach(arg -> addDependencies(arg, enumImports, sourceNamespace, context)); - } - case RosettaFunctionalOperation functional -> { - if (functional.getArgument() != null) { - addDependencies(functional.getArgument(), enumImports, sourceNamespace, context); - } - if (functional instanceof FilterOperation filter) { - addDependencies(filter.getFunction(), enumImports, sourceNamespace, context); - } else if (functional instanceof MapOperation map) { - addDependencies(map.getFunction(), enumImports, sourceNamespace, context); - } - } - case WithMetaOperation withMeta -> { - addDependencies(withMeta.getArgument(), enumImports, sourceNamespace, context); - withMeta.getEntries().forEach(entry -> addDependencies(entry.getValue(), enumImports, sourceNamespace, context)); - } - case RosettaUnaryOperation unary -> { - addDependencies(unary.getArgument(), enumImports, sourceNamespace, context); - } - case RosettaFeatureCall featureCall -> { - addDependencies(featureCall.getReceiver(), enumImports, sourceNamespace, context); - } - case RosettaSymbolReference symbolRef -> { - if (symbolRef.getSymbol() instanceof RosettaNamed named) { - addDependencies(named, enumImports, sourceNamespace, context); - } - symbolRef.getArgs().forEach(arg -> addDependencies(arg, enumImports, sourceNamespace, context)); - } - case RosettaEnumValueReference ref -> { - addDependencies(ref.getEnumeration(), enumImports, sourceNamespace, context); - } - case InlineFunction inline -> { - addDependencies(inline, enumImports, sourceNamespace, context); - } - case ListLiteral listLiteral -> { - listLiteral.getElements().forEach(el -> addDependencies(el, enumImports, sourceNamespace, context)); - } - case RosettaConstructorExpression constructor -> { - if (constructor.getTypeCall() != null && constructor.getTypeCall().getType() != null) { - addDependencies(constructor.getTypeCall().getType(), enumImports, sourceNamespace, context); - } - constructor.getValues() - .forEach(valuePair -> addDependencies(valuePair.getValue(), enumImports, sourceNamespace, context)); - } - default -> { - // For unknown expressions, we can still use eContents() - expression.eContents().forEach(child -> { - if (child instanceof RosettaExpression expr) { - addDependencies(expr, enumImports, sourceNamespace, context); - } else if (child instanceof RosettaNamed named) { - addDependencies(named, enumImports, sourceNamespace, context); - } else if (child instanceof InlineFunction inline) { - addDependencies(inline, enumImports, sourceNamespace, context); - } - }); - } - } + PythonDependencyCollector.Dependencies found = dependencyCollector.collect(expression); + found.symbols().forEach(named -> addDependencies(named, enumImports, sourceNamespace, context)); + found.constructedTypes().forEach(type -> addDependencies(type, enumImports, sourceNamespace, context)); + found.toEnumTargets().forEach(target -> addDependencies(target, enumImports, sourceNamespace, context)); + // Written through the expression's enum type, as in PythonExpressionGenerator + found.enumValueTypes().forEach(type -> addDependencies(type, enumImports, sourceNamespace, context)); } public void addDependencies(RType type, Set enumImports, String sourceNamespace, @@ -185,39 +107,4 @@ public void addDependencies(RType type, Set enumImports, String sourceNa } } } - - public Set rFunctionDependencies(RosettaExpression expression) { - Set result = new HashSet<>(); - collectSymbolReferences(expression, result); - return result; - } - - private void collectSymbolReferences(RosettaExpression expression, Set result) { - if (expression == null) { - return; - } - if (expression instanceof RosettaSymbolReference ref) { - RosettaSymbol symbol = ref.getSymbol(); - if (symbol instanceof Function f) { - result.add(rTypeBuilderFactory.buildRFunction(f)); - } else if (symbol instanceof RosettaRule r) { - result.add(rTypeBuilderFactory.buildRFunction(r)); - } - ref.getArgs().forEach(arg -> collectSymbolReferences(arg, result)); - } - expression.eContents().forEach(child -> { - if (child instanceof RosettaExpression expr) { - collectSymbolReferences(expr, result); - } - }); - } - - public Set rFunctionDependencies(Iterable expressions) { - if (expressions == null) { - return Collections.emptySet(); - } - return StreamSupport.stream(expressions.spliterator(), false) - .flatMap(expr -> rFunctionDependencies(expr).stream()) - .collect(Collectors.toSet()); - } } diff --git a/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionGenerator.java b/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionGenerator.java index def9adf9..f77b579b 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionGenerator.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionGenerator.java @@ -4,6 +4,7 @@ */ package com.regnosys.rosetta.generator.python.functions; +import com.regnosys.rosetta.types.CardinalityProvider; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -29,11 +30,7 @@ import com.regnosys.rosetta.rosetta.RosettaModel; import com.regnosys.rosetta.rosetta.RosettaNamed; import com.regnosys.rosetta.rosetta.RosettaType; -import com.regnosys.rosetta.rosetta.expression.AsOperation; -import com.regnosys.rosetta.rosetta.expression.InlineFunction; -import com.regnosys.rosetta.rosetta.expression.RosettaConstructorExpression; import com.regnosys.rosetta.rosetta.expression.RosettaExpression; -import com.regnosys.rosetta.rosetta.expression.RosettaSymbolReference; import com.regnosys.rosetta.rosetta.simple.Annotated; import com.regnosys.rosetta.rosetta.simple.AssignPathRoot; import com.regnosys.rosetta.rosetta.simple.Attribute; @@ -44,6 +41,7 @@ import com.regnosys.rosetta.rosetta.simple.ShortcutDeclaration; import com.regnosys.rosetta.types.RObjectFactory; +import com.regnosys.rosetta.generator.python.util.PythonDependencyCollector; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -54,6 +52,18 @@ public final class PythonFunctionGenerator { */ private static final Logger LOGGER = LoggerFactory.getLogger(PythonFunctionGenerator.class); + /** + * Collects the functions, types and enums an expression refers to. + */ + @Inject + private PythonDependencyCollector dependencyCollector; + + /** + * Determines whether an expression is multi-valued. + */ + @Inject + private CardinalityProvider cardinalityProvider; + /** * The function dependency provider. */ @@ -111,7 +121,7 @@ public Map generate(Iterable functions, PythonCodeGene private void processFunction(Function function, PythonCodeGeneratorContext context, Map result) { - RosettaModel model = (RosettaModel) function.eContainer(); + RosettaModel model = function.getModel(); if (model == null) { LOGGER.warn("Function {} has no container, skipping", function.getName()); return; @@ -130,42 +140,14 @@ private void processFunction(Function function, PythonCodeGeneratorContext conte } private void collectNamedDependencies(RosettaExpression expression, Set dependencies) { - if (expression == null) { - return; - } - switch (expression) { - case RosettaSymbolReference ref -> { - if (ref.getSymbol() instanceof Function || ref.getSymbol() instanceof Data - || ref.getSymbol() instanceof RosettaEnumeration) { - dependencies.add((RosettaNamed) ref.getSymbol()); - } - ref.getArgs().forEach(arg -> collectNamedDependencies(arg, dependencies)); - } - case RosettaConstructorExpression cons -> { - if (cons.getTypeCall() != null && cons.getTypeCall().getType() != null) { - dependencies.add(cons.getTypeCall().getType()); - } - cons.getValues().forEach(val -> collectNamedDependencies(val.getValue(), dependencies)); - } - case AsOperation asOp -> { - // The target of `as` is a cross-reference (not a containment child), so it - // is invisible to the eContents() walk below and must be added explicitly. - // Only data type targets need an import; choice option targets are just an - // attribute name on the choice's own (already-imported) class. - if (asOp.getType() instanceof Data dataTarget) { - dependencies.add(dataTarget); - } - } - default -> { - } - } - expression.eContents().forEach(child -> { - if (child instanceof RosettaExpression childExpr) { - collectNamedDependencies(childExpr, dependencies); - } else if (child instanceof InlineFunction inlineFn && inlineFn.getBody() != null) { - collectNamedDependencies(inlineFn.getBody(), dependencies); - } - }); + PythonDependencyCollector.Dependencies found = dependencyCollector.collect(expression); + found.symbols().stream() + .filter(symbol -> symbol instanceof Function || symbol instanceof Data + || symbol instanceof RosettaEnumeration) + .forEach(dependencies::add); + dependencies.addAll(found.constructedTypes()); + dependencies.addAll(found.asTargets()); + dependencies.addAll(found.switchCaseTypes()); } private void addFunctionDependencies( @@ -282,10 +264,31 @@ private String generateFunctionBody(Function function, String pythonName, boolea Set funcDeps = collectFunctionDependencies(function, context); context.getEnumImports().addAll(funcDeps); String functionFQN = context.getFullyQualifiedName(function); - context.getFunctionEnumImports().put(functionFQN, funcDeps); + // Dispatch implementations are generated under the base function's name, so add to the + // imports already recorded rather than replace them. + context.getFunctionEnumImports().computeIfAbsent(functionFQN, k -> new HashSet<>()).addAll(funcDeps); PythonExpressionGenerator expressionGenerator = expressionGeneratorProvider.get(); expressionGenerator.setContext(context); + expressionGenerator.setWritingStandaloneFile(isStandalone); + expressionGenerator.setFunctionInputs(rosettaFunctionExtensions.getInputs(function)); + expressionGenerator.setCurrentFunction(functionFQN); + + // Generate the body first, so that the imports it needs are known before it is written + String conditions = generateConditions(function, expressionGenerator); + String aliases = generateAlias(function, expressionGenerator); + boolean isCodeImplementation = isCodeImplementation(function); + boolean isDispatcher = rosettaFunctionExtensions.handleAsEnumFunction(function); + String operations; + if (isDispatcher) { + operations = generateDispatchLogic(function, pythonName, isCodeImplementation, context); + } else if (isCodeImplementation) { + operations = generateNativeFunctionCall(function, context); + } else { + operations = generateOperations(function, context, expressionGenerator); + } + String output = generateOutput(function, isCodeImplementation, expressionGenerator); + PythonCodeWriter writer = new PythonCodeWriter(); writer.appendLine(""); @@ -298,6 +301,9 @@ private String generateFunctionBody(Function function, String pythonName, boolea writer.appendBlock(generateDescription(function, context)); + // Functions in the same cycle of calls are imported here, once both modules have loaded + expressionGenerator.getFunctionLocalImports().forEach(writer::appendLine); + writer.appendLine("self = inspect.currentframe()"); writer.appendLine(""); Iterable inputs = rosettaFunctionExtensions.getInputs(function); @@ -308,21 +314,14 @@ private String generateFunctionBody(Function function, String pythonName, boolea writer.appendLine(""); } - writer.appendBlock(generateConditions(function, expressionGenerator)); - writer.appendBlock(generateAlias(function, expressionGenerator)); - - boolean isCodeImplementation = isCodeImplementation(function); - boolean isDispatcher = rosettaFunctionExtensions.handleAsEnumFunction(function); - - if (isDispatcher) { - writer.appendBlock(generateDispatchLogic(function, pythonName, isCodeImplementation, context)); - } else if (isCodeImplementation) { - writer.appendLine(generateNativeFunctionCall(function, context)); + writer.appendBlock(conditions); + writer.appendBlock(aliases); + if (isCodeImplementation && !isDispatcher) { + writer.appendLine(operations); } else { - writer.appendBlock(generateOperations(function, context, expressionGenerator)); + writer.appendBlock(operations); } - - writer.appendBlock(generateOutput(function, isCodeImplementation, expressionGenerator)); + writer.appendBlock(output); writer.unindent(); writer.newLine(); @@ -453,9 +452,21 @@ private String generateInputs(Function function, PythonCodeGeneratorContext cont } private String getTypeNameForContext(RosettaNamed type, PythonCodeGeneratorContext context) { + return getTypeNameForContext(type, context, false); + } + + /** + * The name to use for a type in generated code. A standalone file refers to a bundled type by + * its stub name, which it imports; only _bundle.py defines the bundle name. + */ + private String getTypeNameForContext(RosettaNamed type, PythonCodeGeneratorContext context, + boolean writingStandaloneFile) { if (type instanceof RosettaEnumeration || RuneToPythonMapper.isRosettaBasicType(type.getName())) { return context.getBundleObjectName(type); } + if (writingStandaloneFile && type instanceof Data) { + return type.getName(); + } String fqn = context.getFullyQualifiedName(type); if (context.getStandaloneClasses().contains(fqn)) { return type.getName(); @@ -660,7 +671,8 @@ private String generateOperations(Function function, PythonExpressionGenerator expressionGenerator) { if (rosettaFunctionExtensions.getOutput(function) != null) { PythonCodeWriter writer = new PythonCodeWriter(); - PythonFunctionGenerationScope scope = new PythonFunctionGenerationScope(); + boolean isStandalone = context.getStandaloneClasses().contains(context.getFullyQualifiedName(function)); + PythonFunctionGenerationScope scope = new PythonFunctionGenerationScope(isStandalone); // 1. Preamble: Proactive initialization Set seenRoots = new HashSet<>(); @@ -681,7 +693,7 @@ private String generateOperations(Function function, scope.markInitialized(rootName); } else if (operation.getPath() != null) { String bundleName = (type instanceof RosettaNamed named) - ? getTypeNameForContext(named, context) + ? getTypeNameForContext(named, context, scope.isStandalone()) : context.getBundleObjectName((RosettaNamed) type); scope.markAsObjectBuilder(rootName); writer.appendLine(rootName + " = ObjectBuilder(" + bundleName + ")"); @@ -741,13 +753,20 @@ private String generateSetOperation( RosettaType rootType = (root instanceof Attribute attr) ? attr.getTypeCall().getType() : null; if (rootType instanceof RosettaEnumeration || operation.getPath() == null) { + if (operation.getPath() == null && root instanceof Attribute attr && attr.getCard().isIsMany() + && !cardinalityProvider.isMulti(operation.getExpression())) { + // A multi-valued output must be a list even when set from a single value. The value + // is still checked at runtime, because some single-valued expressions give a list. + expression = "(lambda v: [] if v is None else (v if isinstance(rune_unwrap(v), list) else [v]))(" + + expression + ")"; + } writer.appendLine(rootName + " = " + expression); scope.markInitialized(rootName); } else { if (!scope.isInitialized(rootName)) { if (rootType != null) { String bundleName = (rootType instanceof RosettaNamed named) - ? getTypeNameForContext(named, context) + ? getTypeNameForContext(named, context, scope.isStandalone()) : context.getBundleObjectName((RosettaNamed) rootType); scope.markAsObjectBuilder(rootName); writer.appendLine(rootName + " = ObjectBuilder(" + bundleName + ")"); @@ -790,7 +809,7 @@ private String generateAddOperation( if (!scope.isInitialized(rootName)) { if (operation.getPath() != null) { String bundleName = (attributeType instanceof RosettaNamed named) - ? getTypeNameForContext(named, context) + ? getTypeNameForContext(named, context, scope.isStandalone()) : context.getBundleObjectName((RosettaNamed) attributeType); scope.markAsObjectBuilder(rootName); writer.appendLine(rootName + " = ObjectBuilder(" + bundleName + ")"); @@ -853,7 +872,7 @@ private void initializePathInPreamble( } else if (current.getNext() != null) { RosettaType attrType = attribute.getTypeCall().getType(); String bundleName = (attrType instanceof RosettaNamed named) - ? getTypeNameForContext(named, context) + ? getTypeNameForContext(named, context, scope.isStandalone()) : context.getBundleObjectName((RosettaNamed) attrType); writer.appendLine(currentPath + " = ObjectBuilder(" + bundleName + ")"); scope.markInitialized(currentPath); @@ -895,11 +914,26 @@ private List getSegmentNames(Segment path) { * a function. */ private static class PythonFunctionGenerationScope { + /** Whether the function is written to its own file rather than to _bundle.py. */ + private final boolean standalone; /** The set of variable names that have been initialized in this scope. */ private final HashSet initializedNames = new HashSet<>(); /** The set of variable names that are marked as ObjectBuilders. */ private final HashSet objectBuilderNames = new HashSet<>(); + PythonFunctionGenerationScope(boolean standalone) { + this.standalone = standalone; + } + + /** + * Whether the function is written to its own file rather than to _bundle.py. + * + * @return true for a standalone function + */ + public boolean isStandalone() { + return standalone; + } + /** * Checks if a variable name has been initialized in this scope. * diff --git a/src/main/java/com/regnosys/rosetta/generator/python/object/PythonAttributeProcessor.java b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonAttributeProcessor.java index 4029a99e..d2f73d56 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/object/PythonAttributeProcessor.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonAttributeProcessor.java @@ -91,7 +91,7 @@ private AttributeResult generateAttribute( } private AttributeResult createAttributeResult(Data rc, RAttribute ra, MetaDataResult metaDataResult, PythonCodeGeneratorContext context) { - RosettaModel model = (RosettaModel) rc.eContainer(); + RosettaModel model = rc.getModel(); String className = context.applyPrefix(model.getName()) + "." + rc.getName(); boolean isStandalone = context.getStandaloneClasses().contains(className); String attrName = RuneToPythonMapper.mangleName(ra.getName()); @@ -114,9 +114,14 @@ private AttributeResult createAttributeResult(Data rc, RAttribute ra, MetaDataRe // Phase 2 updates both __annotations__ (full Annotated type for pydantic schema) and // model_fields[x].annotation (bare type, required so model_rebuild(force=True) picks up // the real type rather than the NoneType placeholder stored in model_fields). + // model_rebuild builds the field from model_fields, so the field's serializer and + // validator are also added to its metadata; otherwise the field would be built without + // them and its metadata (e.g. @key:scoped) would not be written. pythonType = "None"; annotationUpdates.add(String.format("model_fields[\"%s\"].annotation = %s", attrName, hint.build(false))); annotationUpdates.add(String.format("__annotations__[\"%s\"] = %s", attrName, hint.build(true))); + hint.runeAnnotations().ifPresent(runeAnnotations -> annotationUpdates.add( + String.format("model_fields[\"%s\"].metadata += [%s]", attrName, runeAnnotations))); // Record rebuild dependency: className's model_rebuild must run after the referenced // type's model_rebuild. Only track types in the same bundle (same context, not standalone). @@ -145,7 +150,7 @@ private PythonTypeHint deriveTypeHint( MetaDataResult metaDataResult, PythonCodeGeneratorContext context ) { - RosettaModel model = (RosettaModel) rc.eContainer(); + RosettaModel model = rc.getModel(); String className = context.applyPrefix(model.getName()) + "." + rc.getName(); boolean isSourceStandalone = context.getStandaloneClasses().contains(className); @@ -190,9 +195,16 @@ private PythonTypeHint deriveTypeHint( int max = cardinality.getMax().orElse(-1); boolean isList = (cardinality.isMulti() || max > 1); - String baseTypeName = (!validators.isEmpty()) - ? RuneToPythonMapper.getAttributeTypeWithMeta(attrTypeNameIn) - : attrTypeNameIn; + String baseTypeName; + if (!validators.isEmpty()) { + baseTypeName = RuneToPythonMapper.getAttributeTypeWithMeta(attrTypeNameIn); + } else if (RuneToPythonMapper.isRosettaBasicType(rt)) { + // An attribute named after the type it holds (int, str, ..., datetime) would shadow it (#331) + baseTypeName = RuneToPythonMapper.qualifyShadowedType(attrTypeNameIn, + RuneToPythonMapper.mangleName(ra.getName())); + } else { + baseTypeName = attrTypeNameIn; + } String propString = validationProps.toFieldArgs(); Optional fieldProps = (isList && !propString.isEmpty()) ? Optional.of(propString) : Optional.empty(); @@ -295,6 +307,17 @@ private MetaDataResult processMetaDataAttributes( if (!validators.isEmpty()) { keyRefConstraints.put(ra.getName(), new ArrayList<>(validators)); } + // A type with [metadata key] can carry @key and @key:external wherever it is used. The + // runtime writes them only through the field's serializer, so the field needs the hooks + // and a validator that allows the keys, even without metadata on the attribute itself. + if (resolveType(ra) instanceof RDataType dataType + && dataType.getMetaAttributes().stream().anyMatch(ma -> "key".equals(ma.getName()))) { + for (String key : List.of("@key", "@key:external")) { + if (!validators.contains(key)) { + validators.add(key); + } + } + } validators.addAll(otherMeta); return new MetaDataResult(validators, hasReference); } @@ -407,7 +430,7 @@ public void getImportsFromAttributes( Set standaloneClasses = context.getStandaloneClasses(); RDataType buildRDataType = rObjectFactory.buildRDataType(rc); Collection allAttributes = buildRDataType.getOwnAttributes(); - RosettaModel rcModel = (RosettaModel) rc.eContainer(); + RosettaModel rcModel = rc.getModel(); String rcFqn = context.applyPrefix(rcModel.getName()) + "." + rc.getName(); for (RAttribute attr : allAttributes) { diff --git a/src/main/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchProcessor.java b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchProcessor.java new file mode 100644 index 00000000..22fe2f4f --- /dev/null +++ b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchProcessor.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.object; + +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +import com.regnosys.rosetta.generator.python.util.PythonCodeWriter; +import com.regnosys.rosetta.generator.python.util.RuneToPythonMapper; +import com.regnosys.rosetta.rosetta.simple.Choice; +import com.regnosys.rosetta.types.RChoiceOption; +import com.regnosys.rosetta.types.RChoiceType; +import com.regnosys.rosetta.types.RDataType; +import com.regnosys.rosetta.types.REnumType; +import com.regnosys.rosetta.types.RObjectFactory; +import com.regnosys.rosetta.types.RType; +import com.regnosys.rosetta.types.TypeSystem; +import com.regnosys.rosetta.types.builtin.RBuiltinTypeService; + +import jakarta.inject.Inject; + +/** + * Generates the {@code _CHOICE_DISPATCH} table of a choice class (#333). + * + *

The runtime reads a choice in the Rune JSON format, where {@code @type} names the selected + * leaf, not the option of the choice that holds it. The table maps each {@code @type} the choice + * can be read with to the Python attribute name of that option. Keys are Rune names only, never + * language-specific ones: + *

    + *
  • a data type or enum is keyed by its Rune fully qualified name, as {@code _fqrtn()} returns + * it, with no special treatment for namespace segments that are keywords in some language;
  • + *
  • a basic type is keyed by the option name;
  • + *
  • a type alias is keyed as the type it names, keeping the option name for a basic type;
  • + *
  • metadata doesn't change the key. (Java writes a Java class name, such as + * {@code java.lang.String}, for a basic option with metadata, and the option name for a plain + * enum option.)
  • + *
  • a nested choice contributes its leaves; its own FQN is included only when the option has + * metadata, because the option's metadata/reference wrapper is written with its own type.
  • + *
+ */ +public class PythonChoiceDispatchProcessor { + + /** Builds the choice type, with its own and nested options. */ + @Inject + private RObjectFactory rObjectFactory; + + /** Strips type aliases from option types. */ + @Inject + private TypeSystem typeSystem; + + /** Provides the {@code nothing} type, which options may have and the table skips. */ + @Inject + private RBuiltinTypeService builtins; + + /** + * Writes the {@code _CHOICE_DISPATCH} class attribute of a choice. + * + * @param writer the writer for the class body + * @param choice the choice the class is generated from + * @throws IllegalStateException if two options of the choice lead to the same {@code @type} + */ + public void generateChoiceDispatch(PythonCodeWriter writer, Choice choice) { + Map dispatch = buildDispatch(choice); + writer.appendLine("_CHOICE_DISPATCH: ClassVar[dict[str, str]] = {"); + writer.indent(); + dispatch.forEach((key, attribute) -> writer.appendLine(String.format("\"%s\": \"%s\",", key, attribute))); + writer.unindent(); + writer.appendLine("}"); + } + + private Map buildDispatch(Choice choice) { + RChoiceType choiceType = rObjectFactory.buildRChoiceType(choice); + // Sorted, so the generated table is stable + Map attributeByKey = new TreeMap<>(); + Map optionByKey = new TreeMap<>(); + for (RChoiceOption option : choiceType.getOwnOptions()) { + String optionName = option.getEObject().getName(); + String attribute = RuneToPythonMapper.mangleName(optionName); + Set keys = new TreeSet<>(); + collectOptionKeys(option, keys); + for (String key : keys) { + String owner = optionByKey.putIfAbsent(key, optionName); + if (owner != null && !owner.equals(optionName)) { + throw new IllegalStateException(String.format( + "Choice %s: '%s' is reached through both '%s' and '%s', so its option can't be " + + "told from @type", + fullyQualifiedName(choiceType), key, owner, optionName)); + } + attributeByKey.put(key, attribute); + } + } + return attributeByKey; + } + + private void collectOptionKeys(RChoiceOption option, Set keys) { + RType optionType = typeSystem.stripFromTypeAliases(option.getType().getRType()); + if (builtins.NOTHING.equals(optionType)) { + return; + } + if (optionType instanceof RChoiceType nested) { + // A choice cannot own type-level metadata. Attribute metadata belongs to + // the option's wrapper, whose @type still identifies the nested choice. + if (option.getType().hasAttributeMeta()) { + keys.add(fullyQualifiedName(nested)); + } + for (RChoiceOption nestedOption : nested.getOwnOptions()) { + collectOptionKeys(nestedOption, keys); + } + } else { + keys.add(key(optionType, option.getEObject().getName())); + } + } + + private static String key(RType optionType, String optionName) { + return (optionType instanceof RDataType || optionType instanceof REnumType) + ? fullyQualifiedName(optionType) + : optionName; + } + + private static String fullyQualifiedName(RType type) { + return type.getNamespace().toString() + "." + type.getName(); + } +} diff --git a/src/main/java/com/regnosys/rosetta/generator/python/object/PythonModelObjectGenerator.java b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonModelObjectGenerator.java index e43f6a51..5a5b8fef 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/object/PythonModelObjectGenerator.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/object/PythonModelObjectGenerator.java @@ -21,6 +21,7 @@ import com.regnosys.rosetta.generator.python.util.PythonCodeWriter; import com.regnosys.rosetta.generator.python.util.RuneToPythonMapper; import com.regnosys.rosetta.rosetta.RosettaModel; +import com.regnosys.rosetta.rosetta.simple.Choice; import com.regnosys.rosetta.rosetta.simple.Data; import com.regnosys.rosetta.types.RAliasType; import com.regnosys.rosetta.types.RAttribute; @@ -31,6 +32,11 @@ import com.regnosys.rosetta.types.TypeSystem; import com.regnosys.rosetta.rosetta.expression.RosettaExpression; +import com.regnosys.rosetta.generator.python.functions.PythonFunctionDependencyProvider; + +import com.regnosys.rosetta.generator.python.util.PythonDependencyCollector; +import com.regnosys.rosetta.rosetta.RosettaEnumeration; +import com.regnosys.rosetta.rosetta.simple.Function; import jakarta.inject.Inject; import jakarta.inject.Provider; @@ -70,12 +76,30 @@ public class PythonModelObjectGenerator { @Inject private PythonChoiceAliasProcessor pythonChoiceAliasProcessor; + /** + * The Python choice dispatch processor. + */ + @Inject + private PythonChoiceDispatchProcessor pythonChoiceDispatchProcessor; + /** * The Python expression generator provider. */ @Inject private Provider expressionGeneratorProvider; + /** + * Collects the functions, types and enums an expression refers to. + */ + @Inject + private PythonDependencyCollector dependencyCollector; + + /** + * Collects the imports that condition expressions need. + */ + @Inject + private PythonFunctionDependencyProvider functionDependencyProvider; + /** * Scan Rosetta classes to build dependency graph and record super types. * @@ -129,31 +153,13 @@ private void addExpressionDependencies( String className, RosettaExpression expression, PythonCodeGeneratorContext context) { - if (expression == null) { - return; - } - if (expression instanceof com.regnosys.rosetta.rosetta.expression.RosettaSymbolReference ref) { - if (ref.getSymbol() instanceof com.regnosys.rosetta.rosetta.simple.Data - || ref.getSymbol() instanceof com.regnosys.rosetta.rosetta.simple.Function - || ref.getSymbol() instanceof com.regnosys.rosetta.rosetta.RosettaEnumeration) { - - String depName = context.getFullyQualifiedName((com.regnosys.rosetta.rosetta.RosettaNamed) ref.getSymbol()); - addDependency(dependencyDAG, className, depName); - } - ref.getArgs().forEach(arg -> addExpressionDependencies(dependencyDAG, className, arg, context)); - } else if (expression instanceof com.regnosys.rosetta.rosetta.expression.RosettaConstructorExpression cons) { - if (cons.getTypeCall() != null && cons.getTypeCall().getType() != null) { - String depName = context.getFullyQualifiedName(cons.getTypeCall().getType()); - addDependency(dependencyDAG, className, depName); - } - cons.getValues().forEach(val -> addExpressionDependencies(dependencyDAG, className, val.getValue(), context)); - } - - expression.eContents().forEach(child -> { - if (child instanceof com.regnosys.rosetta.rosetta.expression.RosettaExpression childExpr) { - addExpressionDependencies(dependencyDAG, className, childExpr, context); - } - }); + PythonDependencyCollector.Dependencies found = dependencyCollector.collect(expression); + found.symbols().stream() + .filter(symbol -> symbol instanceof Data || symbol instanceof Function + || symbol instanceof RosettaEnumeration) + .forEach(symbol -> addDependency(dependencyDAG, className, context.getFullyQualifiedName(symbol))); + found.constructedTypes() + .forEach(type -> addDependency(dependencyDAG, className, context.getFullyQualifiedName(type))); } /** @@ -191,7 +197,7 @@ private void addDependency( } private String generateClass(Data rc, Set enumImports, PythonCodeGeneratorContext context) { - RosettaModel model = (RosettaModel) rc.eContainer(); + RosettaModel model = rc.getModel(); String className = context.applyPrefix(model.getName()) + "." + rc.getName(); boolean isStandalone = context.getStandaloneClasses().contains(className); if (rc.getSuperType() != null && rc.getSuperType().getName() == null) { @@ -204,11 +210,22 @@ private String generateClass(Data rc, Set enumImports, PythonCodeGenerat if (!isStandalone) { pythonAttributeProcessor.getImportsFromAttributes(rc, enumImports, context, false); + addConditionImports(rc, enumImports, context); } return generateBody(rc, context, isStandalone); } + /** + * Adds the imports used by the type's conditions, such as an enum that only they refer to. + */ + private void addConditionImports(Data rc, Set imports, PythonCodeGeneratorContext context) { + String sourceNamespace = context.applyPrefix(rc.getModel().getName()); + for (com.regnosys.rosetta.rosetta.simple.Condition condition : rc.getConditions()) { + functionDependencyProvider.addDependencies(condition.getExpression(), imports, sourceNamespace, context); + } + } + /** * Generates the _ALLOWED_METADATA string for the type. * @@ -272,6 +289,7 @@ private String keyRefConstraintsToString(Map> keyRefConstra private String generateBody(Data rc, PythonCodeGeneratorContext context, boolean isStandalone) { PythonExpressionGenerator expressionGenerator = expressionGeneratorProvider.get(); expressionGenerator.setContext(context); + expressionGenerator.setWritingStandaloneFile(isStandalone); RDataType rosettaDataType = rObjectFactory.buildRDataType(rc); Map> keyRefConstraints = new HashMap<>(); @@ -301,6 +319,7 @@ private String generateBody(Data rc, PythonCodeGeneratorContext context, boolean // Add attribute-level imports (all Data types safe for standalone files) Set imports = new java.util.HashSet<>(); pythonAttributeProcessor.getImportsFromAttributes(rc, imports, context, true); + addConditionImports(rc, imports, context); for (String imp : imports) { writer.appendLine(imp); } @@ -311,14 +330,21 @@ private String generateBody(Data rc, PythonCodeGeneratorContext context, boolean writer.appendLine("class " + classNameDefinition + "(" + superClassName + "):"); writer.indent(); - // Bundled classes defer initial schema compilation until after all class bodies are - // defined. By the time Phase 3 model_rebuild(force=True) runs, all cyclic types are - // fully available, so Pydantic can build schemas ~4× faster (~1.8 GB / ~5s for CDM - // vs ~7.9 GB / ~17s without defer_build). Phase 3 is still required: Pydantic - // builds None-typed placeholder schemas eagerly even with defer_build=True, so an - // explicit model_rebuild is needed to pick up Phase 2 annotation updates. - if (!isStandalone) { - writer.appendLine("model_config = ConfigDict(defer_build=True)"); + // Every class defers schema compilation. Standalone classes build on first use, which + // avoids compiling schemas that are never used. Bundled classes wait until all class + // bodies are defined: by the time Phase 3 model_rebuild(force=True) runs, all cyclic + // types are fully available, so Pydantic can build schemas ~4× faster (~1.8 GB / ~5s for + // CDM vs ~7.9 GB / ~17s without defer_build). Phase 3 is still required for bundled + // classes: Pydantic builds None-typed placeholder schemas eagerly even with + // defer_build=True, so an explicit model_rebuild is needed to pick up Phase 2 annotation + // updates. + writer.appendLine("model_config = ConfigDict(defer_build=True)"); + + // A generated choice looks the same as a type with optional attributes and a one-of + // condition, but its JSON format differs, so the runtime has to be told which it is. + if (rc instanceof Choice choice) { + writer.appendLine("_RUNE_CHOICE: ClassVar[bool] = True"); + pythonChoiceDispatchProcessor.generateChoiceDispatch(writer, choice); } String metaData = getClassMetaDataString(rc); @@ -355,9 +381,7 @@ private String generateBody(Data rc, PythonCodeGeneratorContext context, boolean writer.appendBlock(constraints); } - context.setGeneratingBundledTypeCondition(!isStandalone); writer.appendBlock(expressionGenerator.generateTypeConditions(rc)); - context.setGeneratingBundledTypeCondition(false); return writer.toString(); } diff --git a/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorConstants.java b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorConstants.java index 2ce14417..c68c5a4e 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorConstants.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorConstants.java @@ -21,6 +21,10 @@ private PythonCodeGeneratorConstants() { * The pyproject.toml file name. */ public static final String PYPROJECT_TOML = "pyproject.toml"; + /** + * The README file name, which pyproject.toml names as the package description. + */ + public static final String README_MD = "README.md"; /** * The __init__ file name. */ diff --git a/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorUtil.java b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorUtil.java index 74ec94b4..42d3eca9 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorUtil.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonCodeGeneratorUtil.java @@ -14,9 +14,17 @@ public final class PythonCodeGeneratorUtil { + /** A release version, {@code #.#.#} (e.g. {@code 1.2.3}). */ public static final Pattern VALID_VERSION_PATTERN = Pattern.compile("\\d+\\.\\d+\\.\\d+"); + /** + * A development version, {@code #.#.#-dev.#} (e.g. {@code 1.2.3-dev.4}); groups: the release, then the counter. + */ public static final Pattern DEV_VERSION_PATTERN = Pattern.compile("(\\d+\\.\\d+\\.\\d+)-dev\\.(\\d+)"); + /** A Maven snapshot version (e.g. {@code 0.0.0.main-SNAPSHOT}); group: the release. */ public static final Pattern SNAPSHOT_VERSION_PATTERN = Pattern.compile("(\\d+\\.\\d+\\.\\d+).*-SNAPSHOT"); + /** + * A version already in PEP 440 development form, {@code #.#.#.dev#} (e.g. {@code 1.2.3.dev4}). + */ private static final Pattern PEP440_DEV_VERSION_PATTERN = Pattern.compile("\\d+\\.\\d+\\.\\d+\\.dev\\d+"); private PythonCodeGeneratorUtil() { @@ -73,12 +81,15 @@ public static String createImports() { # pylint: disable=wildcard-import, wrong-import-order, missing-class-docstring # pylint: disable=missing-module-docstring from __future__ import annotations + import builtins as _builtins import datetime + import datetime as _datetime + import decimal as _decimal import functools import inspect import sys from decimal import Decimal - from typing import Annotated, Optional + from typing import Annotated, ClassVar, Optional from pydantic import ConfigDict, Field, validate_call, InstanceOf @@ -177,7 +188,7 @@ public static String createPYProjectTomlFile(String namespace, String version, S requires-python = ">= 3.11" dependencies = [ "pydantic>=2.10.3", - "rune.runtime>=3.0.0,<4.0.0" + "rune.runtime>=3.1.0,<4.0.0" ] readme = "README.md" classifiers = [ @@ -196,6 +207,27 @@ public static String createPYProjectTomlFile(String namespace, String version, S where = ["src"]""".formatted(name, version).stripIndent(); } + /** + * Creates the README that pyproject.toml names as the package's description, so that the + * generated package is complete without any file supplied from elsewhere. + * + * @param projectName the package name, as in pyproject.toml + * @param version the package version, as in pyproject.toml + * @return the README content + */ + public static String createReadmeFile(String projectName, String version) { + return """ + # %s + + Python package generated from a Rune model by \ + [rune-python-generator](https://github.com/finos/rune-python-generator). + + Version: %s + + The generated code depends on the `rune.runtime` package. + """.formatted(projectName, version); + } + public static String cleanVersion(String version) { if (version == null || version.equals("${project.version}")) { return "0.0.0"; diff --git a/src/main/java/com/regnosys/rosetta/generator/python/util/PythonDependencyCollector.java b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonDependencyCollector.java new file mode 100644 index 00000000..7633ce73 --- /dev/null +++ b/src/main/java/com/regnosys/rosetta/generator/python/util/PythonDependencyCollector.java @@ -0,0 +1,567 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.util; + +import java.util.LinkedHashSet; +import java.util.Set; + +import com.regnosys.rosetta.rosetta.RosettaEnumValue; +import com.regnosys.rosetta.rosetta.RosettaEnumeration; +import com.regnosys.rosetta.rosetta.RosettaNamed; +import com.regnosys.rosetta.rosetta.expression.ArithmeticOperation; +import com.regnosys.rosetta.rosetta.expression.AsKeyOperation; +import com.regnosys.rosetta.rosetta.expression.AsOperation; +import com.regnosys.rosetta.rosetta.expression.ChoiceOperation; +import com.regnosys.rosetta.rosetta.expression.ComparisonOperation; +import com.regnosys.rosetta.rosetta.expression.DefaultOperation; +import com.regnosys.rosetta.rosetta.expression.DistinctOperation; +import com.regnosys.rosetta.rosetta.expression.EqualityOperation; +import com.regnosys.rosetta.rosetta.expression.FilterOperation; +import com.regnosys.rosetta.rosetta.expression.FirstOperation; +import com.regnosys.rosetta.rosetta.expression.FlattenOperation; +import com.regnosys.rosetta.rosetta.expression.InlineFunction; +import com.regnosys.rosetta.rosetta.expression.JoinOperation; +import com.regnosys.rosetta.rosetta.expression.LastOperation; +import com.regnosys.rosetta.rosetta.expression.ListLiteral; +import com.regnosys.rosetta.rosetta.expression.LogicalOperation; +import com.regnosys.rosetta.rosetta.expression.MapOperation; +import com.regnosys.rosetta.rosetta.expression.MaxOperation; +import com.regnosys.rosetta.rosetta.expression.MinOperation; +import com.regnosys.rosetta.rosetta.expression.OneOfOperation; +import com.regnosys.rosetta.rosetta.expression.ReduceOperation; +import com.regnosys.rosetta.rosetta.expression.ReverseOperation; +import com.regnosys.rosetta.rosetta.expression.RosettaAbsentExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaBinaryOperation; +import com.regnosys.rosetta.rosetta.expression.RosettaBooleanLiteral; +import com.regnosys.rosetta.rosetta.expression.RosettaConditionalExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaConstructorExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaContainsExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaCountOperation; +import com.regnosys.rosetta.rosetta.expression.RosettaDeepFeatureCall; +import com.regnosys.rosetta.rosetta.expression.RosettaDisjointExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaExistsExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaFeatureCall; +import com.regnosys.rosetta.rosetta.expression.RosettaFunctionalOperation; +import com.regnosys.rosetta.rosetta.expression.RosettaImplicitVariable; +import com.regnosys.rosetta.rosetta.expression.RosettaIntLiteral; +import com.regnosys.rosetta.rosetta.expression.RosettaNumberLiteral; +import com.regnosys.rosetta.rosetta.expression.RosettaOnlyElement; +import com.regnosys.rosetta.rosetta.expression.RosettaOnlyExistsExpression; +import com.regnosys.rosetta.rosetta.expression.RosettaStringLiteral; +import com.regnosys.rosetta.rosetta.expression.RosettaSuperCall; +import com.regnosys.rosetta.rosetta.expression.RosettaSymbolReference; +import com.regnosys.rosetta.rosetta.expression.RosettaUnaryOperation; +import com.regnosys.rosetta.rosetta.expression.SortOperation; +import com.regnosys.rosetta.rosetta.expression.SumOperation; +import com.regnosys.rosetta.rosetta.expression.SwitchOperation; +import com.regnosys.rosetta.rosetta.expression.ThenOperation; +import com.regnosys.rosetta.rosetta.expression.ToDateOperation; +import com.regnosys.rosetta.rosetta.expression.ToDateTimeOperation; +import com.regnosys.rosetta.rosetta.expression.ToEnumOperation; +import com.regnosys.rosetta.rosetta.expression.ToIntOperation; +import com.regnosys.rosetta.rosetta.expression.ToNumberOperation; +import com.regnosys.rosetta.rosetta.expression.ToStringOperation; +import com.regnosys.rosetta.rosetta.expression.ToTimeOperation; +import com.regnosys.rosetta.rosetta.expression.ToZonedDateTimeOperation; +import com.regnosys.rosetta.rosetta.expression.WithMetaOperation; +import com.regnosys.rosetta.rosetta.simple.Data; +import com.regnosys.rosetta.types.RType; +import com.regnosys.rosetta.types.RosettaTypeProvider; +import com.regnosys.rosetta.utils.RosettaExpressionSwitch; + +import jakarta.inject.Inject; + +/** + * Collects what a Rune expression refers to: the functions, types and enums that generated code + * needs to import, or that the dependency graph needs as edges. + * + *

There is one case for every Rune expression type, so a new expression type stops the build + * until it is handled here. Each case visits every sub-expression, including those held by + * elements that aren't expressions themselves, such as switch cases and with-meta entries. + * + *

References are kept by kind, because each caller uses a different selection of them. + */ +public final class PythonDependencyCollector + extends RosettaExpressionSwitch { + + /** + * The references found in one or more expressions. + */ + public static final class Dependencies { + /** The named elements referred to directly; see {@link #symbols()}. */ + private final Set symbols = new LinkedHashSet<>(); + /** The types built with a constructor; see {@link #constructedTypes()}. */ + private final Set constructedTypes = new LinkedHashSet<>(); + /** The data types named as the target of {@code as}. */ + private final Set asTargets = new LinkedHashSet<>(); + /** The enums named as the target of {@code to-enum}. */ + private final Set toEnumTargets = new LinkedHashSet<>(); + /** The enum types that enum values are written through; see {@link #enumValueTypes()}. */ + private final Set enumValueTypes = new LinkedHashSet<>(); + /** The data types that switch cases test the value against. */ + private final Set switchCaseTypes = new LinkedHashSet<>(); + + /** + * @return the named elements referred to directly, such as functions, types, enums and + * attributes; enum values are in {@link #enumValueTypes()} instead + */ + public Set symbols() { + return symbols; + } + + /** + * @return the types built with a constructor, {@code Type { ... }} + */ + public Set constructedTypes() { + return constructedTypes; + } + + /** + * @return the data types named as the target of {@code as} + */ + public Set asTargets() { + return asTargets; + } + + /** + * @return the enums named as the target of {@code to-enum} + */ + public Set toEnumTargets() { + return toEnumTargets; + } + + /** + * @return the data types that switch cases test the value against + */ + public Set switchCaseTypes() { + return switchCaseTypes; + } + + /** + * @return the enum types that enum values are written through: the expression's own enum + * type, which for an extended enum isn't the enum declaring the value + */ + public Set enumValueTypes() { + return enumValueTypes; + } + } + + /** Gives the enum type an enum value is written through. */ + @Inject + private RosettaTypeProvider typeProvider; + + /** + * Collects the references in an expression. + * + * @param expression the expression, or null + * @return the references found + */ + public Dependencies collect(RosettaExpression expression) { + Dependencies dependencies = new Dependencies(); + visit(expression, dependencies); + return dependencies; + } + + /** + * Collects the references in each expression. + * + * @param expressions the expressions; null elements are ignored + * @return the references found in all of them + */ + public Dependencies collect(Iterable expressions) { + Dependencies dependencies = new Dependencies(); + expressions.forEach(expression -> visit(expression, dependencies)); + return dependencies; + } + + private void visit(RosettaExpression expression, Dependencies dependencies) { + if (expression != null) { + doSwitch(expression, dependencies); + } + } + + private void visit(InlineFunction function, Dependencies dependencies) { + if (function != null) { + visit(function.getBody(), dependencies); + } + } + + private Void unary(RosettaUnaryOperation expr, Dependencies dependencies) { + visit(expr.getArgument(), dependencies); + return null; + } + + private Void binary(RosettaBinaryOperation expr, Dependencies dependencies) { + visit(expr.getLeft(), dependencies); + visit(expr.getRight(), dependencies); + return null; + } + + private Void functional(RosettaFunctionalOperation expr, Dependencies dependencies) { + visit(expr.getArgument(), dependencies); + visit(expr.getFunction(), dependencies); + return null; + } + + private Void none() { + return null; + } + + // ------------------------------------------------------------------------- + // Expressions that refer to something + // ------------------------------------------------------------------------- + + @Override + protected Void caseSymbolReference(RosettaSymbolReference expr, Dependencies dependencies) { + if (expr.getSymbol() instanceof RosettaEnumValue) { + dependencies.enumValueTypes.add(typeProvider.getRMetaAnnotatedType(expr).getRType()); + } else if (expr.getSymbol() instanceof RosettaNamed named) { + dependencies.symbols.add(named); + } + expr.getArgs().forEach(arg -> visit(arg, dependencies)); + return null; + } + + @Override + protected Void caseConstructorExpression(RosettaConstructorExpression expr, Dependencies dependencies) { + if (expr.getTypeCall() != null && expr.getTypeCall().getType() != null) { + dependencies.constructedTypes.add(expr.getTypeCall().getType()); + } + expr.getValues().forEach(pair -> visit(pair.getValue(), dependencies)); + return null; + } + + @Override + protected Void caseAsOperation(AsOperation expr, Dependencies dependencies) { + // A choice option target is just an attribute on the choice's own class + if (expr.getType() instanceof Data target) { + dependencies.asTargets.add(target); + } + return unary(expr, dependencies); + } + + @Override + protected Void caseToEnumOperation(ToEnumOperation expr, Dependencies dependencies) { + dependencies.toEnumTargets.add(expr.getEnumeration()); + return unary(expr, dependencies); + } + + @Override + protected Void caseSwitchOperation(SwitchOperation expr, Dependencies dependencies) { + visit(expr.getArgument(), dependencies); + expr.getCases().forEach(switchCase -> { + visit(switchCase.getExpression(), dependencies); + if (switchCase.getGuard() != null) { + visit(switchCase.getGuard().getLiteralGuard(), dependencies); + if (switchCase.getGuard().getDataGuard() != null) { + dependencies.switchCaseTypes.add(switchCase.getGuard().getDataGuard()); + } + if (switchCase.getGuard().getEnumGuard() != null) { + // The guard is written through the switch argument's enum type + dependencies.enumValueTypes.add( + typeProvider.getRMetaAnnotatedType(expr.getArgument()).getRType()); + } + } + }); + visit(expr.getDefault(), dependencies); + return null; + } + + @Override + protected Void caseWithMetaOperation(WithMetaOperation expr, Dependencies dependencies) { + visit(expr.getArgument(), dependencies); + expr.getEntries().forEach(entry -> visit(entry.getValue(), dependencies)); + return null; + } + + // ------------------------------------------------------------------------- + // Expressions that only contain other expressions + // ------------------------------------------------------------------------- + + @Override + protected Void caseListLiteral(ListLiteral expr, Dependencies dependencies) { + expr.getElements().forEach(element -> visit(element, dependencies)); + return null; + } + + @Override + protected Void caseConditionalExpression(RosettaConditionalExpression expr, Dependencies dependencies) { + visit(expr.getIf(), dependencies); + visit(expr.getIfthen(), dependencies); + visit(expr.getElsethen(), dependencies); + return null; + } + + @Override + protected Void caseFeatureCall(RosettaFeatureCall expr, Dependencies dependencies) { + visit(expr.getReceiver(), dependencies); + return null; + } + + @Override + protected Void caseDeepFeatureCall(RosettaDeepFeatureCall expr, Dependencies dependencies) { + visit(expr.getReceiver(), dependencies); + return null; + } + + @Override + protected Void caseOnlyExists(RosettaOnlyExistsExpression expr, Dependencies dependencies) { + expr.getArgs().forEach(arg -> visit(arg, dependencies)); + return null; + } + + @Override + protected Void caseSuperCall(RosettaSuperCall expr, Dependencies dependencies) { + expr.getArgs().forEach(arg -> visit(arg, dependencies)); + return null; + } + + @Override + protected Void caseImplicitVariable(RosettaImplicitVariable expr, Dependencies dependencies) { + return none(); + } + + @Override + protected Void caseBooleanLiteral(RosettaBooleanLiteral expr, Dependencies dependencies) { + return none(); + } + + @Override + protected Void caseIntLiteral(RosettaIntLiteral expr, Dependencies dependencies) { + return none(); + } + + @Override + protected Void caseNumberLiteral(RosettaNumberLiteral expr, Dependencies dependencies) { + return none(); + } + + @Override + protected Void caseStringLiteral(RosettaStringLiteral expr, Dependencies dependencies) { + return none(); + } + + // Binary operations + + @Override + protected Void caseAddOperation(ArithmeticOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseSubtractOperation(ArithmeticOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseMultiplyOperation(ArithmeticOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseDivideOperation(ArithmeticOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseJoinOperation(JoinOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseAndOperation(LogicalOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseOrOperation(LogicalOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseLessThanOperation(ComparisonOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseLessThanOrEqualOperation(ComparisonOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseGreaterThanOperation(ComparisonOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseGreaterThanOrEqualOperation(ComparisonOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseEqualsOperation(EqualityOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseNotEqualsOperation(EqualityOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseContainsOperation(RosettaContainsExpression expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseDisjointOperation(RosettaDisjointExpression expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + @Override + protected Void caseDefaultOperation(DefaultOperation expr, Dependencies dependencies) { + return binary(expr, dependencies); + } + + // Unary operations + + @Override + protected Void caseAsKeyOperation(AsKeyOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseChoiceOperation(ChoiceOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseOneOfOperation(OneOfOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseAbsentOperation(RosettaAbsentExpression expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseCountOperation(RosettaCountOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseExistsOperation(RosettaExistsExpression expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseDistinctOperation(DistinctOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseFirstOperation(FirstOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseFlattenOperation(FlattenOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseLastOperation(LastOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseReverseOperation(ReverseOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseOnlyElementOperation(RosettaOnlyElement expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseSumOperation(SumOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToStringOperation(ToStringOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToNumberOperation(ToNumberOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToIntOperation(ToIntOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToTimeOperation(ToTimeOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToDateOperation(ToDateOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToDateTimeOperation(ToDateTimeOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + @Override + protected Void caseToZonedDateTimeOperation(ToZonedDateTimeOperation expr, Dependencies dependencies) { + return unary(expr, dependencies); + } + + // Operations with an inline function + + @Override + protected Void caseFilterOperation(FilterOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseMapOperation(MapOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseMaxOperation(MaxOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseMinOperation(MinOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseReduceOperation(ReduceOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseSortOperation(SortOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } + + @Override + protected Void caseThenOperation(ThenOperation expr, Dependencies dependencies) { + return functional(expr, dependencies); + } +} diff --git a/src/main/java/com/regnosys/rosetta/generator/python/util/RuneToPythonMapper.java b/src/main/java/com/regnosys/rosetta/generator/python/util/RuneToPythonMapper.java index 67cc721e..d2f99ada 100644 --- a/src/main/java/com/regnosys/rosetta/generator/python/util/RuneToPythonMapper.java +++ b/src/main/java/com/regnosys/rosetta/generator/python/util/RuneToPythonMapper.java @@ -9,6 +9,7 @@ import com.regnosys.rosetta.rosetta.RosettaEnumeration; import com.regnosys.rosetta.rosetta.RosettaModel; +import com.regnosys.rosetta.rosetta.RosettaRootElement; import com.regnosys.rosetta.rosetta.RosettaNamed; import com.regnosys.rosetta.rosetta.RosettaTypeAlias; import com.regnosys.rosetta.types.RAttribute; @@ -113,6 +114,35 @@ private static String toPythonBasicTypeInnerFunction(String typeName) { }; } + /** + * Qualifies a basic Python type when an attribute's own name would shadow it (#331). + * + *

Generated modules use {@code from __future__ import annotations}, so pydantic resolves a + * field's annotation with the class's names in scope. An attribute named {@code int}, + * {@code str}, {@code bool}, {@code Decimal} or {@code datetime} is a {@code FieldInfo} there, + * so {@code int: Optional[int]} would not be validated, and {@code datetime: Optional[datetime.date]} + * would fail to resolve. The type is written through an alias imported by + * {@code PythonCodeGeneratorUtil.createImports()} instead. The aliases start with {@code _}, + * which {@link #mangleName} never leaves on an attribute name, so they can't clash. + * + * @param pythonType the Python type, as {@link #toPythonType} returns it + * @param attrPythonName the attribute's Python name, as {@link #mangleName} returns it + * @return the type, qualified through an alias if the attribute's name shadows it + */ + public static String qualifyShadowedType(String pythonType, String attrPythonName) { + int dot = pythonType.indexOf('.'); + String leading = (dot < 0) ? pythonType : pythonType.substring(0, dot); + if (!leading.equals(attrPythonName)) { + return pythonType; + } + return switch (leading) { + case "int", "str", "bool" -> "_builtins." + pythonType; + case "datetime" -> "_datetime" + pythonType.substring(leading.length()); + case "Decimal" -> "_decimal." + pythonType; + default -> pythonType; + }; + } + /** * Inner private function to convert from Rosetta type to Python type. * Returns null if no matching type. @@ -145,7 +175,7 @@ public static String getFullyQualifiedName(RosettaNamed rn, String namespacePref break; } } - RosettaModel model = (RosettaModel) rn.eContainer(); + RosettaModel model = (rn instanceof RosettaRootElement root) ? root.getModel() : null; if (model == null) { throw new RuntimeException("Rosetta model not found for data " + rn.getName()); } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonArithmeticExpressionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonArithmeticExpressionTest.java index 3caabb6d..6b3ccc0b 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonArithmeticExpressionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonArithmeticExpressionTest.java @@ -50,6 +50,7 @@ b int (1..1) testUtils.assertGeneratedContainsExpectedString(generatedPython, """ class TestMath(BaseDataClass): + model_config = ConfigDict(defer_build=True) a: int = Field(..., description='') b: int = Field(..., description='') @@ -84,6 +85,7 @@ b int (1..1) generatedPython, """ class ArithmeticTest(BaseDataClass): + model_config = ConfigDict(defer_build=True) a: int = Field(..., description='') b: int = Field(..., description='') @@ -118,6 +120,7 @@ s string (1..1) """, """ class TestConv(BaseDataClass): + model_config = ConfigDict(defer_build=True) val: int = Field(..., description='') s: str = Field(..., description='') @@ -142,6 +145,7 @@ s string (1..1) """, """ class TestDateConv(BaseDataClass): + model_config = ConfigDict(defer_build=True) s: str = Field(..., description='') @rune_condition @@ -165,11 +169,31 @@ s string (1..1) """, """ class TestEnumConv(BaseDataClass): + model_config = ConfigDict(defer_build=True) s: str = Field(..., description='') @rune_condition def condition_0_EnumConvCheck(self): item = self - return rune_all_elements(MyEnum(rune_resolve_attr(self, "s")), "=", com.rosetta.test.model.MyEnum.MyEnum.VALUE_1)"""); + return rune_all_elements((lambda v: None if v is None else next((m for m in com.rosetta.test.model.MyEnum.MyEnum if m.value == v), None))(rune_resolve_attr(self, "s")), "=", com.rosetta.test.model.MyEnum.MyEnum.VALUE_1)"""); + } + + /** + * Converting one enum to another matches the value name, and gives nothing if the target + * has no value of that name, as in Rune's Java generator. + */ + @Test + public void testEnumToEnumConversion() { + testUtils.assertBundleContainsExpectedString(""" + enum Source: + Red + enum Target: + Red + type TestEnumToEnumConv: + colour Source (1..1) + condition EnumConvCheck: + colour to-enum Target = Target -> Red + """, + "return rune_all_elements((lambda v: None if v is None else com.rosetta.test.model.Target.Target.__members__.get(rune_unwrap(v).name))(rune_resolve_attr(self, \"colour\")), \"=\", com.rosetta.test.model.Target.Target.RED)"); } } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonAsOperationTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonAsOperationTest.java index 0fad1823..43534ba3 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonAsOperationTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonAsOperationTest.java @@ -4,6 +4,8 @@ */ package com.regnosys.rosetta.generator.python.expressions; +import java.util.Map; + import org.eclipse.xtext.testing.InjectWith; import org.eclipse.xtext.testing.extensions.InjectionExtension; import org.junit.jupiter.api.Test; @@ -560,4 +562,117 @@ quxAttr int (0..1) result = rune_resolve_attr(rune_resolve_attr(rune_resolve_attr(self, "outer"), "Inner1"), "Bar") """); } + + // ------------------------------------------------------------------------- + // Type names and imports for data type narrowing + // ------------------------------------------------------------------------- + + /** + * DogB and OwnerB refer to each other, so they are bundled, and so is their supertype Pet. + * Shape and Circle are standalone. Checker is a standalone class, OwnerB a bundled one, and + * NarrowPet a function; each narrows to a bundled type and to a standalone one. + */ + private static final String NARROWING_MODEL = """ + type Pet: + + type DogB extends Pet: + owner OwnerB (0..1) + + type OwnerB: + dog DogB (0..1) + pet Pet (0..1) + shape Shape (0..1) + + condition NarrowsToBundledType: + if pet exists then (pet as DogB) exists + + condition NarrowsToStandaloneType: + if shape exists then (shape as Circle) exists + + type Shape: + + type Circle extends Shape: + radius number (0..1) + + type Checker: + pet Pet (0..1) + shape Shape (0..1) + + condition NarrowsToBundledType: + if pet exists then (pet as DogB) exists + + condition NarrowsToStandaloneType: + if shape exists then (shape as Circle) exists + + func NarrowPet: + inputs: + pet Pet (1..1) + shape Shape (1..1) + output: + result boolean (1..1) + set result: + (pet as DogB) exists and (shape as Circle) exists + """; + + private Map narrowing() { + return testUtils.generatePythonFromString(NARROWING_MODEL); + } + + /** + * A condition in the bundle narrows to a bundled type by its bundle name. + */ + @Test + public void testBundledConditionNarrowsToBundledTypeByBundleName() { + String bundle = narrowing().get("src/com/_bundle.py").toString(); + testUtils.assertAppearsAfter(bundle, "def condition_0_NarrowsToBundledType(self):", + "), com_rosetta_test_model_DogB) else None)"); + } + + /** + * A condition in the bundle imports a standalone type it narrows to, inside the condition. + */ + @Test + public void testBundledConditionImportsStandaloneTarget() { + String bundle = narrowing().get("src/com/_bundle.py").toString(); + testUtils.assertAppearsAfter(bundle, "def condition_1_NarrowsToStandaloneType(self):", + " from com.rosetta.test.model.Circle import Circle"); + } + + /** + * A standalone class condition imports the bundled type it narrows to, inside the condition, + * and uses its stub name. + */ + @Test + public void testStandaloneConditionImportsBundledTarget() { + String checker = narrowing().get("src/com/rosetta/test/model/Checker.py").toString(); + testUtils.assertAppearsAfter(checker, "def condition_0_NarrowsToBundledType(self):", + " from com.rosetta.test.model.DogB import DogB"); + testUtils.assertGeneratedContainsExpectedString(checker, "), DogB) else None)"); + testUtils.assertGeneratedDoesNotContain(checker, "com_rosetta_test_model_DogB"); + } + + /** + * A standalone class condition imports the standalone type it narrows to, inside the + * condition. + */ + @Test + public void testStandaloneConditionImportsStandaloneTarget() { + String checker = narrowing().get("src/com/rosetta/test/model/Checker.py").toString(); + testUtils.assertAppearsAfter(checker, "def condition_1_NarrowsToStandaloneType(self):", + " from com.rosetta.test.model.Circle import Circle"); + } + + /** + * Control: a function imports both types it narrows to and uses their stub names. + */ + @Test + public void testFunctionImportsNarrowingTargets() { + String narrowPet = narrowing().get("src/com/rosetta/test/model/functions/NarrowPet.py").toString(); + testUtils.assertGeneratedContainsExpectedString(narrowPet, + "from com.rosetta.test.model.DogB import DogB"); + testUtils.assertGeneratedContainsExpectedString(narrowPet, + "from com.rosetta.test.model.Circle import Circle"); + testUtils.assertGeneratedContainsExpectedString(narrowPet, "), DogB) else None)"); + testUtils.assertGeneratedDoesNotContain(narrowPet, "com_rosetta_test_model_DogB"); + } } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCollectionExpressionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCollectionExpressionTest.java index 58057b63..3bc67fe8 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCollectionExpressionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCollectionExpressionTest.java @@ -291,7 +291,7 @@ items Item (0..*) testUtils.assertGeneratedContainsExpectedString(generatedPython, "items: Annotated[Optional[list[Item | None]], Item.serializer(), Item.validator()] = Field(None, description='')"); testUtils.assertGeneratedContainsExpectedString(generatedPython, - "return rune_all_elements((lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))(item))(rune_filter(rune_resolve_attr(self, \"items\"), lambda item: rune_all_elements(rune_resolve_attr(item, \"val\"), \">\", 5))), \"=\", 0)"); + "return rune_all_elements((lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))(item))(rune_filter(rune_resolve_attr(self, \"items\"), lambda item: rune_all_elements(rune_resolve_attr(item, \"val\"), \">\", 5))), \"=\", 0)"); } /** @@ -312,7 +312,7 @@ val int (1..1) testUtils.assertGeneratedContainsExpectedString(generatedPython, "def TestNestedNested(items: list[InstanceOf[Item] | None] | None) -> int:"); testUtils.assertGeneratedContainsExpectedString(generatedPython, - "result = (lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))(item))(rune_filter(rune_resolve_attr(self, \"items\"), lambda item: rune_all_elements(rune_resolve_attr(item, \"val\"), \">\", 5)))"); + "result = (lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))(item))(rune_filter(rune_resolve_attr(self, \"items\"), lambda item: rune_all_elements(rune_resolve_attr(item, \"val\"), \">\", 5)))"); } // ------------------------------------------------------------------------- @@ -339,7 +339,7 @@ items Item (0..*) testUtils.assertGeneratedContainsExpectedString(generatedPython, "items: Annotated[Optional[list[Item | None]], Item.serializer(), Item.validator()] = Field(None, description='')"); testUtils.assertGeneratedContainsExpectedString(generatedPython, - "return rune_all_elements((lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))(item))([x for x in map(lambda item: rune_resolve_attr(item, \"val\"), rune_resolve_attr(self, \"items\") or []) if x is not None]), \"=\", 0)"); + "return rune_all_elements((lambda item: (lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))(item))([x for x in map(lambda item: rune_resolve_attr(item, \"val\"), rune_resolve_attr(self, \"items\") or []) if x is not None]), \"=\", 0)"); } // ------------------------------------------------------------------------- @@ -469,6 +469,7 @@ field2 string (1..1) <"Test boolean field2"> """, """ class Test(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test any operation condition \""" @@ -670,6 +671,7 @@ class C(rune.runtime.metadata.EnumWithMetaMixin, Enum): String expectedA = """ class A(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test type \""" @@ -685,6 +687,7 @@ class A(BaseDataClass): String expectedBPhase1 = """ class B(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test type B \""" @@ -700,6 +703,7 @@ class B(BaseDataClass): String expectedTestPhase1 = """ class Test(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test filter operation condition \""" @@ -754,6 +758,7 @@ field3 boolean (1..1) <"Test boolean field3"> """, """ class Test(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test disjoint binary expression condition \""" @@ -808,6 +813,7 @@ delimiter string (1..1) testUtils.assertGeneratedContainsExpectedString(generatedPython, """ class TestJoin(BaseDataClass): + model_config = ConfigDict(defer_build=True) field1: list[str | None] = Field(..., description='', min_length=1) delimiter: str = Field(..., description='') @@ -854,6 +860,7 @@ class TestEnum(rune.runtime.metadata.EnumWithMetaMixin, Enum): String expectedTest1 = """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test only-element condition. \""" @@ -977,4 +984,52 @@ vals extract (if item > 0 then item else 0) assertFalse(generatedPython.contains("def _then_fn"), "Conditional inside extract must not use pre-hoisted closure (item not in scope)"); } + + /** + * A filter on a single value applies the test to that value and keeps a single result. + * Passing the value to rune_filter would iterate it as if it were a list. + */ + @Test + public void testFilterOnSingleValueDoesNotIterate() { + Map python = testUtils.generatePythonFromString( + """ + type Leaf: + value number (0..1) + + func FilterSingle: + inputs: + lone Leaf (0..1) + output: + result Leaf (0..1) + set result: + lone filter value exists + """); + String generated = python.get("src/com/rosetta/test/model/functions/FilterSingle.py").toString(); + testUtils.assertGeneratedContainsExpectedString(generated, + "result = (lambda item: item if item is not None and (rune_attr_exists(rune_resolve_attr(item, \"value\"))) else None)(rune_resolve_attr(self, \"lone\"))"); + testUtils.assertGeneratedDoesNotContain(generated, "rune_filter("); + } + + /** + * A filter on a list still uses rune_filter. + */ + @Test + public void testFilterOnListUsesRuneFilter() { + Map python = testUtils.generatePythonFromString( + """ + type Leaf: + value number (0..1) + + func FilterMany: + inputs: + leaves Leaf (0..*) + output: + result Leaf (0..*) + add result: + leaves filter value exists + """); + String generated = python.get("src/com/rosetta/test/model/functions/FilterMany.py").toString(); + testUtils.assertGeneratedContainsExpectedString(generated, + "rune_filter(rune_resolve_attr(self, \"leaves\"), lambda item: rune_attr_exists(rune_resolve_attr(item, \"value\")))"); + } } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonConditionalExpressionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonConditionalExpressionTest.java index cb728196..de021532 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonConditionalExpressionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonConditionalExpressionTest.java @@ -45,6 +45,7 @@ field2 number (0..1) <"Test number field 2"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Test if-then condition. \"\"\" @@ -86,6 +87,7 @@ field2 number (0..1) <"Test number field 2"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Test if-then-else condition. \"\"\" @@ -127,6 +129,7 @@ field2 number (0..1) <"Test number field 2"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Test boolean condition. \"\"\" @@ -168,6 +171,7 @@ field2 number (0..1) <"Test number field 2"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Test absent condition. \"\"\" @@ -216,6 +220,7 @@ a int (1..1) <"Test field a"> """, """ class FooTest(BaseDataClass): + model_config = ConfigDict(defer_build=True) a: int = Field(..., description='Test field a') \"\"\" Test field a @@ -232,6 +237,8 @@ def _then_2(): def _then_default(): return False switchAttribute = rune_resolve_attr(self, "a") + if switchAttribute is None: + return None if switchAttribute == 1: return _then_1() elif switchAttribute == 2: @@ -243,6 +250,78 @@ def _then_default(): """); } + /** + * A switch on a type with subtypes tests the value's type, and passes the matched value to + * the case as item, so the case can use that type's attributes. + */ + @Test + public void testGenerateTypeSwitch() { + String bundle = testUtils.generatePythonAndExtractBundle( + """ + type Person: + name string (0..1) + + type Employee extends Person: + salary number (1..1) + + func Wage: + inputs: + person Person (0..1) + output: + wage number (0..1) + set wage: + person switch + Employee then salary, + default 0 + """); + testUtils.assertGeneratedContainsExpectedString(bundle, "def _then_1(item):\n return rune_resolve_attr(item, \"salary\")"); + testUtils.assertGeneratedContainsExpectedString(bundle, "if switchAttribute is None:\n return None"); + testUtils.assertGeneratedContainsExpectedString(bundle, + "if isinstance(rune_unwrap(switchAttribute), Employee):\n return _then_1(switchAttribute)"); + } + + /** + * A switch on a choice follows the path through nested choices to the option, and passes the + * option's value to the case as item. + */ + @Test + public void testGenerateNestedChoiceSwitch() { + String bundle = testUtils.generatePythonAndExtractBundle( + """ + choice PoweredVehicle: + PetrolCar + Motorcycle + + type PetrolCar: + fuelCapacity number (1..1) + + type Motorcycle: + fuelCapacity number (1..1) + + choice Vehicle: + PoweredVehicle + Bicycle + + type Bicycle: + weight number (1..1) + + func Mileage: + inputs: + vehicle Vehicle (1..1) + output: + mileage number (1..1) + set mileage: + vehicle switch + PetrolCar then 15 * fuelCapacity, + PoweredVehicle then 80, + Bicycle then 30 + """); + testUtils.assertGeneratedContainsExpectedString(bundle, + "if (caseValue := rune_resolve_attr(rune_resolve_attr(switchAttribute, \"PoweredVehicle\"), \"PetrolCar\")) is not None:\n return _then_1(caseValue)"); + testUtils.assertGeneratedContainsExpectedString(bundle, + "return (15 * rune_resolve_attr(item, \"fuelCapacity\"))"); + } + // ------------------------------------------------------------------------- // From RosettaChoiceExpressionTest // ------------------------------------------------------------------------- @@ -261,6 +340,7 @@ field3 string (0..1) <"Test string field 3"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test choice condition. \""" @@ -295,6 +375,7 @@ field1 string (0..1) <"Test string field 1"> """, """ class Test1(BaseDataClass): + model_config = ConfigDict(defer_build=True) _CHOICE_ALIAS_MAP ={"field1":[]} \""" Test one-of condition. diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCountOperationTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCountOperationTest.java index 3b6e952e..d53fa2bd 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCountOperationTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonCountOperationTest.java @@ -28,9 +28,13 @@ @SuppressWarnings("LineLength") public class PythonCountOperationTest { + /** + * The generated lambda that {@code count} applies: it counts the non-null items of a list, or 1 for a single value. + */ private static final String COUNT_LAMBDA_PREFIX = - "(lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))"; + "(lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))"; + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonDeepPathTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonDeepPathTest.java index 3351c704..05f4f05d 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonDeepPathTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonDeepPathTest.java @@ -29,6 +29,7 @@ @SuppressWarnings("LineLength") public class PythonDeepPathTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonEmptyEvaluationTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonEmptyEvaluationTest.java index cdff7fea..06992254 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonEmptyEvaluationTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonEmptyEvaluationTest.java @@ -29,6 +29,7 @@ @SuppressWarnings("LineLength") public class PythonEmptyEvaluationTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonSymbolReferenceMetaTypeTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonSymbolReferenceMetaTypeTest.java index f03cc8c0..dfdc5a95 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonSymbolReferenceMetaTypeTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonSymbolReferenceMetaTypeTest.java @@ -44,6 +44,7 @@ @SuppressWarnings("LineLength") public class PythonSymbolReferenceMetaTypeTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonTypeAliasConditionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonTypeAliasConditionTest.java index 75e2b4d4..50e4e1e4 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonTypeAliasConditionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonTypeAliasConditionTest.java @@ -39,6 +39,7 @@ @SuppressWarnings("LineLength") public class PythonTypeAliasConditionTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonWithMetaExpressionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonWithMetaExpressionTest.java index 391fe56d..956ceb48 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonWithMetaExpressionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/PythonWithMetaExpressionTest.java @@ -4,6 +4,8 @@ */ package com.regnosys.rosetta.generator.python.expressions; +import java.util.Map; + import org.eclipse.xtext.testing.InjectWith; import org.eclipse.xtext.testing.extensions.InjectionExtension; import org.junit.jupiter.api.Test; @@ -138,6 +140,38 @@ result MyEnum (0..1) "com.rosetta.test.model.MyEnum.MyEnum.deserialize({'@data': rune_resolve_attr(self, \"val\"), '@scheme': rune_resolve_attr(self, \"schemeVal\")}, allowed_meta={'@scheme'})"); } + /** + * With a namespace prefix, the enum must be referred to by its prefixed module path: the Rune + * namespace alone is not a Python package. + */ + @Test + public void testWithMetaEnumSchemeUsesNamespacePrefix() { + Map python = testUtils.generatePythonFromString(""" + enum MyEnum: + VALUE_A + VALUE_B + + func TestWithMetaEnum: + inputs: + val MyEnum (0..1) + schemeVal string (0..1) + output: + result MyEnum (0..1) + [metadata scheme] + set result: + val with-meta { scheme: schemeVal } + """, "finos", null); + String function = python.entrySet().stream() + .filter(e -> e.getKey().endsWith("/functions/TestWithMetaEnum.py")) + .map(e -> e.getValue().toString()) + .findFirst() + .orElseThrow(); + testUtils.assertGeneratedContainsExpectedString(function, + "result = finos.com.rosetta.test.model.MyEnum.MyEnum.deserialize("); + testUtils.assertGeneratedDoesNotContain(function, + "result = com.rosetta.test.model.MyEnum.MyEnum.deserialize("); + } + // ------------------------------------------------------------------------- // Complex type: with location (scoped key) // ------------------------------------------------------------------------- diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaReduceOperationTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaReduceOperationTest.java index 8e34a84e..3fe3d5a1 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaReduceOperationTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaReduceOperationTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ package com.regnosys.rosetta.generator.python.expressions; import org.eclipse.xtext.testing.InjectWith; @@ -14,6 +18,7 @@ @InjectWith(RosettaInjectorProvider.class) public class RosettaReduceOperationTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaWithMetaOperationTest.java b/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaWithMetaOperationTest.java index ad8801fb..a2ab4880 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaWithMetaOperationTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/expressions/RosettaWithMetaOperationTest.java @@ -1,3 +1,7 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ package com.regnosys.rosetta.generator.python.expressions; import org.eclipse.xtext.testing.InjectWith; @@ -24,6 +28,7 @@ @SuppressWarnings("LineLength") public class RosettaWithMetaOperationTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionCycleImportTest.java b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionCycleImportTest.java new file mode 100644 index 00000000..21564d45 --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionCycleImportTest.java @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.functions; + +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.util.Map; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * Functions that call each other import each other inside the calling function, so that either + * module can be imported first. Every other call keeps its import at the top of the file, + * including calls made by a function in a cycle to a function outside it, calls into a cycle + * from outside it, and a function calling itself. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +public class PythonFunctionCycleImportTest { + + /** The package of the generated functions. */ + private static final String PACKAGE = "com.rosetta.test.model.functions."; + + /** + * The Rune model the tests generate Python from. + * IsEven and IsOdd call each other. IsEven also calls Helper, which is outside the cycle. + * UsesIsEven calls into the cycle from outside it. UsesHelper has no cycle near it. + * Countdown calls itself. + */ + private static final String MODEL = """ + func IsEven: + inputs: + n int (1..1) + output: + result boolean (1..1) + set result: + if n = 0 then True else IsOdd(Helper(n) - 2) + + func IsOdd: + inputs: + n int (1..1) + output: + result boolean (1..1) + set result: + if n = 0 then False else IsEven(n - 1) + + func Helper: + inputs: + n int (1..1) + output: + result int (1..1) + set result: + n + 1 + + func UsesHelper: + inputs: + n int (1..1) + output: + result int (1..1) + set result: + Helper(n) + + func UsesIsEven: + inputs: + n int (1..1) + output: + result boolean (1..1) + set result: + IsEven(n) + + func Countdown: + inputs: + n int (1..1) + output: + result int (1..1) + set result: + if n = 0 then 0 else Countdown(n - 1) + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String functionName) { + Map python = testUtils.generatePythonFromString(MODEL); + return python.get("src/com/rosetta/test/model/functions/" + functionName + ".py").toString(); + } + + private static String importOf(String functionName) { + return "from " + PACKAGE + functionName + " import " + functionName; + } + + /** Asserts that the function's import is at the top of the file and not inside the function. */ + private void assertTopLevelImport(String generated, String functionName) { + testUtils.assertGeneratedContainsExpectedString(generated, "\n" + importOf(functionName)); + testUtils.assertGeneratedDoesNotContain(generated, " " + importOf(functionName)); + } + + /** Asserts that nothing is imported inside the generated function. */ + private static void assertNoImportInsideFunction(String generated, String functionName) { + String body = generated.substring(generated.indexOf("def " + functionName + "(")); + assertFalse(body.contains("\n from ") || body.contains("\n import "), + functionName + " should not import anything inside the function:\n" + body); + } + + @Test + public void testFunctionInCycleIsImportedInsideTheCaller() { + String isEven = generated("IsEven"); + testUtils.assertAppearsAfter(isEven, "def IsEven(", " " + importOf("IsOdd")); + testUtils.assertGeneratedDoesNotContain(isEven, "\n" + importOf("IsOdd")); + } + + @Test + public void testOtherFunctionInCycleIsImportedInsideTheCaller() { + String isOdd = generated("IsOdd"); + testUtils.assertAppearsAfter(isOdd, "def IsOdd(", " " + importOf("IsEven")); + testUtils.assertGeneratedDoesNotContain(isOdd, "\n" + importOf("IsEven")); + } + + @Test + public void testFunctionInCycleKeepsTopLevelImportForCallOutsideCycle() { + assertTopLevelImport(generated("IsEven"), "Helper"); + } + + @Test + public void testCallIntoCycleFromOutsideKeepsTopLevelImport() { + String usesIsEven = generated("UsesIsEven"); + assertTopLevelImport(usesIsEven, "IsEven"); + assertNoImportInsideFunction(usesIsEven, "UsesIsEven"); + } + + @Test + public void testFunctionOutsideCycleKeepsTopLevelImport() { + String usesHelper = generated("UsesHelper"); + assertTopLevelImport(usesHelper, "Helper"); + assertNoImportInsideFunction(usesHelper, "UsesHelper"); + } + + @Test + public void testRecursiveFunctionDoesNotImportItself() { + String countdown = generated("Countdown"); + testUtils.assertGeneratedDoesNotContain(countdown, importOf("Countdown")); + assertNoImportInsideFunction(countdown, "Countdown"); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionOutputTest.java b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionOutputTest.java index b3b17c15..25f56b20 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionOutputTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonFunctionOutputTest.java @@ -38,6 +38,53 @@ public class PythonFunctionOutputTest { /** * Test case for function with add operation. */ + /** + * Setting a whole multi-valued output from a single value must still give a list. + */ + @Test + public void testSetMultiValuedOutputFromSingleValueWrapsInList() { + Map gf = testUtils.generatePythonFromString( + """ + type Leaf: + value number (1..1) + + func SetFromSingle: + inputs: + leaf Leaf (0..1) + output: + result Leaf (0..*) + set result: + leaf + """); + String generatedPython = gf.get("src/com/rosetta/test/model/functions/SetFromSingle.py").toString(); + testUtils.assertGeneratedContainsExpectedString(generatedPython, + "result = (lambda v: [] if v is None else (v if isinstance(rune_unwrap(v), list) else [v]))" + + "(rune_resolve_attr(self, \"leaf\"))"); + } + + /** + * Setting a whole multi-valued output from a list leaves the assignment unchanged. + */ + @Test + public void testSetMultiValuedOutputFromListIsNotWrapped() { + Map gf = testUtils.generatePythonFromString( + """ + type Leaf: + value number (1..1) + + func SetFromList: + inputs: + leaves Leaf (0..*) + output: + result Leaf (0..*) + set result: + leaves + """); + String generatedPython = gf.get("src/com/rosetta/test/model/functions/SetFromList.py").toString(); + testUtils.assertGeneratedContainsExpectedString(generatedPython, + "result = rune_resolve_attr(self, \"leaves\")"); + } + @Test public void testGenerateAddOperation() { Map gf = testUtils.generatePythonFromString( diff --git a/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonReferenceResolutionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonReferenceResolutionTest.java new file mode 100644 index 00000000..fdf343b4 --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/functions/PythonReferenceResolutionTest.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.functions; + +import java.util.Map; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * Generated code must import every function and enum it refers to, and refer to an enum value + * through the enum written in the model. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +@SuppressWarnings("LineLength") +public class PythonReferenceResolutionTest { + + /** The Rune model the tests generate Python from. */ + private static final String MODEL = """ + enum Kind: + Plain + Fancy + + enum Mode: + Start + End + + type Settings: + mode Mode (0..1) + + enum Base: + Low + High + + enum Extended extends Base: + Middle + + type Qualified: + level Extended (0..1) + + condition KnownLevel: + if level exists + then level = Extended -> Middle or level = Extended -> High + + type Unqualified: + level Extended (0..1) + + condition KnownLevel: + if level exists + then level = Middle or level = High + + type Switched: + level Extended (0..1) + + condition KnownLevel: + level switch + High then True, + Middle then True, + default False + + func Helper: + inputs: + value string (0..1) + output: + result string (0..1) + set result: + value + + func MakeKey: + inputs: + value string (0..1) + output: + result string (0..1) + set result: + value + + func CallInSwitch: + inputs: + value string (0..1) + output: + result string (0..1) + set result: + value switch + "a" then Helper(value), + default empty + + func CallInWithMeta: + inputs: + value string (0..1) + output: + result string (0..1) + [metadata scheme] + set result: + value with-meta { scheme: MakeKey(value) } + + func Dispatch: + [codeImplementation] + inputs: + kind Kind (1..1) + settings Settings (0..1) + output: + result boolean (0..1) + + func Dispatch(kind: Kind -> Fancy): + alias isStart: settings -> mode = Mode -> Start + set result: isStart + + func Dispatch(kind: Kind -> Plain): + set result: False + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String path) { + Map python = testUtils.generatePythonFromString(MODEL); + return python.get("src/com/rosetta/test/model/" + path).toString(); + } + + @Test + public void testFunctionCalledInSwitchCaseIsImported() { + testUtils.assertGeneratedContainsExpectedString(generated("functions/CallInSwitch.py"), + "from com.rosetta.test.model.functions.Helper import Helper"); + } + + @Test + public void testFunctionCalledInWithMetaEntryIsImported() { + testUtils.assertGeneratedContainsExpectedString(generated("functions/CallInWithMeta.py"), + "from com.rosetta.test.model.functions.MakeKey import MakeKey"); + } + + @Test + public void testEnumUsedOnlyInFirstDispatchImplementationIsImported() { + testUtils.assertGeneratedContainsExpectedString(generated("functions/Dispatch.py"), + "import com.rosetta.test.model.Mode\n"); + } + + @Test + public void testInheritedEnumValueUsesEnumWrittenInModel() { + String python = generated("Qualified.py"); + testUtils.assertGeneratedContainsExpectedString(python, "com.rosetta.test.model.Extended.Extended.HIGH"); + testUtils.assertGeneratedDoesNotContain(python, "com.rosetta.test.model.Base.Base.HIGH"); + } + + @Test + public void testUnqualifiedInheritedEnumValueUsesExpressionEnum() { + String python = generated("Unqualified.py"); + testUtils.assertGeneratedContainsExpectedString(python, "com.rosetta.test.model.Extended.Extended.HIGH"); + testUtils.assertGeneratedDoesNotContain(python, "com.rosetta.test.model.Base.Base.HIGH"); + } + + @Test + public void testSwitchGuardComparesWithEnumMember() { + String python = generated("Switched.py"); + testUtils.assertGeneratedContainsExpectedString(python, + "if switchAttribute == com.rosetta.test.model.Extended.Extended.HIGH:"); + testUtils.assertGeneratedContainsExpectedString(python, + "elif switchAttribute == com.rosetta.test.model.Extended.Extended.MIDDLE:"); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBuiltinNameClashTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBuiltinNameClashTest.java new file mode 100644 index 00000000..e8432e8b --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBuiltinNameClashTest.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.object; + +import java.util.Map; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * An attribute whose Python name is a name its own annotation uses ({@code int}, {@code str}, + * {@code bool}, {@code datetime}, {@code Decimal}) must not shadow that name (#331). Generated + * modules use {@code from __future__ import annotations}, so pydantic resolves annotations with + * the class's own names in scope, where the attribute's name is its {@code FieldInfo} default. + * The annotation is qualified through an alias that can't be an attribute name instead. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +public class PythonBuiltinNameClashTest { + + /** + * The Rune model the tests generate Python from. + * BundledChoice and Node refer to each other, so both are bundled. The other types are standalone. + */ + private static final String MODEL = """ + choice ChoiceBasic: + string + int + + type Clash: + str string (0..1) + bool boolean (0..1) + + type DateClash: + datetime date (0..1) + + type TimeClash: + datetime time (0..1) + + type DecimalClash: + Decimal number (0..1) + + type NoClash: + quantity int (0..1) + name string (0..1) + tradeDate date (0..1) + amount number (0..1) + + type WithMeta: + str string (0..1) + [metadata scheme] + + choice BundledChoice: + int + Node + + type Node: + next BundledChoice (0..1) + str string (0..1) + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String file) { + Map python = testUtils.generatePythonFromString(MODEL); + return python.get("src/com/" + file).toString(); + } + + @Test + public void testIntOptionOfChoiceIsQualified() { + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/ChoiceBasic.py"), + " int: Optional[_builtins.int] = Field(None, description='')"); + } + + @Test + public void testStrAndBoolAttributesAreQualified() { + String clash = generated("rosetta/test/model/Clash.py"); + testUtils.assertGeneratedContainsExpectedString(clash, + " str: Optional[_builtins.str] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(clash, + " bool: Optional[_builtins.bool] = Field(None, description='')"); + } + + @Test + public void testDatetimeAttributeIsQualified() { + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/DateClash.py"), + " datetime: Optional[_datetime.date] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/TimeClash.py"), + " datetime: Optional[_datetime.time] = Field(None, description='')"); + } + + @Test + public void testDecimalAttributeIsQualified() { + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/DecimalClash.py"), + " Decimal: Optional[_decimal.Decimal] = Field(None, description='')"); + } + + @Test + public void testBundledClassesAreQualified() { + String bundle = generated("_bundle.py"); + testUtils.assertGeneratedContainsExpectedString(bundle, + " int: Optional[_builtins.int] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(bundle, + " str: Optional[_builtins.str] = Field(None, description='')"); + } + + @Test + public void testAliasesAreImported() { + for (String file : new String[] {"rosetta/test/model/ChoiceBasic.py", "_bundle.py"}) { + String python = generated(file); + testUtils.assertGeneratedContainsExpectedString(python, "import builtins as _builtins\n"); + testUtils.assertGeneratedContainsExpectedString(python, "import datetime as _datetime\n"); + testUtils.assertGeneratedContainsExpectedString(python, "import decimal as _decimal\n"); + } + } + + @Test + public void testAttributesWithoutAClashAreUnchanged() { + String noClash = generated("rosetta/test/model/NoClash.py"); + testUtils.assertGeneratedContainsExpectedString(noClash, + " quantity: Optional[int] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(noClash, + " name: Optional[str] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(noClash, + " tradeDate: Optional[datetime.date] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(noClash, + " amount: Optional[Decimal] = Field(None, description='')"); + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/ChoiceBasic.py"), + " string: Optional[str] = Field(None, description='')"); + } + + @Test + public void testAttributeWithMetadataIsUnchanged() { + // The type is StrWithMeta, so the attribute's name doesn't clash with it + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/WithMeta.py"), + " str: Annotated[Optional[StrWithMeta], StrWithMeta.serializer(), " + + "StrWithMeta.validator(('@scheme', ))] = Field(None, description='')"); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBundleRebuildOrderTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBundleRebuildOrderTest.java index b4cde3ca..34e4dada 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBundleRebuildOrderTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonBundleRebuildOrderTest.java @@ -38,6 +38,7 @@ @SuppressWarnings("LineLength") public class PythonBundleRebuildOrderTest { + /** Injected test utilities. */ @Inject private PythonGeneratorTestUtils testUtils; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchTest.java new file mode 100644 index 00000000..2cd4293e --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceDispatchTest.java @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.object; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Map; +import java.util.stream.Collectors; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * Every choice class gets a {@code _CHOICE_DISPATCH} table mapping each {@code @type} it can be + * read with to the attribute name of the option that holds it (#333). Keys are Rune names only: + * a data type or enum is keyed by its Rune FQN, a basic type by the option name, with or without + * metadata, and a type alias by the key of the type it names (keeping the option name for a basic + * type). A nested choice contributes its leaves; an intermediate choice also contributes its own FQN + * when the option has metadata, because that option's metadata/reference wrapper carries its type. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +@SuppressWarnings("checkstyle:LineLength") +public class PythonChoiceDispatchTest { + + /** The line that marks a choice class; its dispatch table follows it. */ + private static final String MARKER = " _RUNE_CHOICE: ClassVar[bool] = True\n"; + + /** + * Choices three levels deep: {@code OuterChoice} holds {@code ChoiceABCD} and {@code ChoiceEFGH}, which hold choices of {@code A} to {@code H}. + */ + private static final String NESTED_MODEL = """ + type A: + value string (1..1) + type B: + value string (1..1) + type C: + value string (1..1) + type D: + value string (1..1) + type E1: + value string (1..1) + type F: + value string (1..1) + type G: + value string (1..1) + type H: + value string (1..1) + + choice ChoiceAB: + A + B + choice ChoiceCD: + C + D + choice ChoiceEF: + E1 + F + choice ChoiceGH: + G + H + + choice ChoiceABCD: + ChoiceAB + ChoiceCD + choice ChoiceEFGH: + ChoiceEF + ChoiceGH + + choice OuterChoice: + ChoiceABCD + ChoiceEFGH + + type Root: + payload OuterChoice (0..1) + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String model, String file) { + Map python = testUtils.generatePythonFromString(model); + return python.get("src/com/" + file).toString(); + } + + @Test + public void testChoiceWithDirectOptions() { + testUtils.assertGeneratedContainsExpectedString( + generated(NESTED_MODEL, "rosetta/test/model/ChoiceAB.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "A", + "com.rosetta.test.model.B": "B", + } + """); + } + + @Test + public void testChoiceOfChoices() { + testUtils.assertGeneratedContainsExpectedString( + generated(NESTED_MODEL, "rosetta/test/model/ChoiceABCD.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "ChoiceAB", + "com.rosetta.test.model.B": "ChoiceAB", + "com.rosetta.test.model.C": "ChoiceCD", + "com.rosetta.test.model.D": "ChoiceCD", + } + """); + } + + @Test + public void testThreeLevelChoice() { + testUtils.assertGeneratedContainsExpectedString( + generated(NESTED_MODEL, "rosetta/test/model/OuterChoice.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "ChoiceABCD", + "com.rosetta.test.model.B": "ChoiceABCD", + "com.rosetta.test.model.C": "ChoiceABCD", + "com.rosetta.test.model.D": "ChoiceABCD", + "com.rosetta.test.model.E1": "ChoiceEFGH", + "com.rosetta.test.model.F": "ChoiceEFGH", + "com.rosetta.test.model.G": "ChoiceEFGH", + "com.rosetta.test.model.H": "ChoiceEFGH", + } + """); + } + + @Test + public void testBasicTypeOptionsAreKeyedByOptionNameAndEnumsByFullyQualifiedName() { + String model = """ + enum CurrencyEnum: + USD + EUR + + type A: + value string (1..1) + + choice Pick: + A + CurrencyEnum + string + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Pick.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "A", + "com.rosetta.test.model.CurrencyEnum": "CurrencyEnum", + "string": "string", + } + """); + } + + /** + * Metadata doesn't change an option's key: a basic type keeps its option name and an enum its + * Rune FQN, as without metadata, and never a language-specific name such as a Java class name. + */ + @Test + public void testMetadataDoesNotChangeKeys() { + String model = """ + enum CurrencyEnum: + USD + EUR + + choice WithMeta: + CurrencyEnum + [metadata scheme] + string + [metadata scheme] + number + [metadata scheme] + int + [metadata scheme] + boolean + [metadata scheme] + date + [metadata scheme] + time + [metadata scheme] + dateTime + [metadata scheme] + zonedDateTime + [metadata scheme] + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/WithMeta.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "boolean": "boolean", + "com.rosetta.test.model.CurrencyEnum": "CurrencyEnum", + "date": "date", + "dateTime": "dateTime", + "int": "int", + "number": "number", + "string": "string", + "time": "time", + "zonedDateTime": "zonedDateTime", + } + """); + } + + /** An integer's digits don't change its key. */ + @Test + public void testIntegerWithMetadataIsKeyedByOptionNameWhateverItsDigits() { + String model = """ + typeAlias Medium: int(digits: 12) + typeAlias Large: int(digits: 20) + + choice Sized: + Medium + [metadata scheme] + Large + [metadata scheme] + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Sized.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "Large": "Large", + "Medium": "Medium", + } + """); + } + + @Test + public void testAliasOptionsFollowTheTypeTheyNameWithOrWithoutMetadata() { + String model = """ + enum CurrencyEnum: + USD + EUR + + typeAlias Money: CurrencyEnum + typeAlias AliasOfMoney: Money + typeAlias Text: string + + choice Plain: + AliasOfMoney + Text + + choice Meta: + AliasOfMoney + [metadata scheme] + Text + [metadata scheme] + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Plain.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "Text": "Text", + "com.rosetta.test.model.CurrencyEnum": "AliasOfMoney", + } + """); + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Meta.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "Text": "Text", + "com.rosetta.test.model.CurrencyEnum": "AliasOfMoney", + } + """); + } + + @Test + public void testNestedEnumOptionIsKeyedByFullyQualifiedName() { + // From finos/rune-python-generator#338 + String model = """ + enum CurrencyEnum: + USD + EUR + + choice Inner: + CurrencyEnum + string + choice Outer: + Inner + int + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Outer.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.CurrencyEnum": "Inner", + "int": "int", + "string": "Inner", + } + """); + } + + /** A namespace segment that is a Java keyword gets no special treatment: the key is the Rune name. */ + @Test + public void testDataTypeInNamespaceWithJavaKeywordIsKeyedByRuneName() { + String model = """ + namespace test.new.model + + type A: + value string (1..1) + + choice Pick: + A + string + """; + Map python = testUtils.generatePythonFromString(model); + testUtils.assertGeneratedContainsExpectedString( + python.get("src/test/new/model/Pick.py").toString(), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "string": "string", + "test.new.model.A": "A", + } + """); + } + + @Test + public void testPlainNestedChoicesDoNotExposeIntermediateDiscriminators() { + String outer = generated(NESTED_MODEL, "rosetta/test/model/OuterChoice.py"); + for (String intermediate : new String[] { + "ChoiceAB", "ChoiceCD", "ChoiceEF", "ChoiceGH", "ChoiceABCD", "ChoiceEFGH" + }) { + testUtils.assertGeneratedDoesNotContain(outer, + "\"com.rosetta.test.model." + intermediate + "\":"); + } + } + + @Test + public void testLocatedChoiceOptionKeepsItsOwnDiscriminator() { + String model = """ + type A: + value string (1..1) + type B: + value string (1..1) + choice Inner: + A + B + choice Located: + Inner + [metadata location] + int + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Located.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "Inner", + "com.rosetta.test.model.B": "Inner", + "com.rosetta.test.model.Inner": "Inner", + "int": "int", + } + """); + } + + @Test + public void testWrappedDescendantDiscriminatorSurvivesPlainAncestor() { + String model = """ + type A: + value string (1..1) + type B: + value string (1..1) + type C: + value string (1..1) + choice Inner: + A + B + choice Located: + Inner + [metadata location] + string + choice Ancestor: + Located + C + """; + String ancestor = generated(model, "rosetta/test/model/Ancestor.py"); + testUtils.assertGeneratedContainsExpectedString(ancestor, + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "Located", + "com.rosetta.test.model.B": "Located", + "com.rosetta.test.model.C": "C", + "com.rosetta.test.model.Inner": "Located", + "string": "Located", + } + """); + testUtils.assertGeneratedDoesNotContain(ancestor, "\"com.rosetta.test.model.Located\":"); + } + + @Test + public void testReferenceChoiceOptionKeepsItsOwnDiscriminator() { + String model = """ + type A: + value string (1..1) + type B: + value string (1..1) + choice Inner: + A + B + choice Referenced: + Inner + [metadata address "pointsTo"=Root->located] + string + type Root: + located Inner (0..1) + [metadata location] + payload Referenced (0..1) + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Referenced.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "Inner", + "com.rosetta.test.model.B": "Inner", + "com.rosetta.test.model.Inner": "Inner", + "string": "string", + } + """); + } + + @Test + public void testAliasOfPlainChoiceOnlyContributesLeafDiscriminators() { + String model = """ + type A: + value string (1..1) + type B: + value string (1..1) + type C: + value string (1..1) + choice Inner: + A + B + typeAlias AliasOfInner: Inner + choice Pick: + AliasOfInner + C + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/Pick.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "AliasOfInner", + "com.rosetta.test.model.B": "AliasOfInner", + "com.rosetta.test.model.C": "C", + } + """); + } + + @Test + public void testTypeAliasOptions() { + String model = """ + typeAlias Text: string + + type A: + value string (1..1) + + typeAlias AliasOfA: A + + choice WithAliases: + Text + AliasOfA + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "rosetta/test/model/WithAliases.py"), + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "Text": "Text", + "com.rosetta.test.model.A": "AliasOfA", + } + """); + } + + @Test + public void testBundledChoice() { + // LoopOption and LoopHolder refer to each other through LoopChoice, so all three are bundled. + String model = """ + type B: + value string (1..1) + + type LoopOption: + back LoopHolder (0..1) + + choice LoopChoice: + LoopOption + B + + type LoopHolder: + option LoopChoice (0..1) + """; + testUtils.assertGeneratedContainsExpectedString( + generated(model, "_bundle.py"), + "class com_rosetta_test_model_LoopChoice(BaseDataClass):\n" + + " model_config = ConfigDict(defer_build=True)\n" + + MARKER + """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.B": "B", + "com.rosetta.test.model.LoopOption": "LoopOption", + } + """); + } + + @Test + public void testNamespacePrefixIsNotPartOfKeys() { + Map python = testUtils.generatePythonFromString(NESTED_MODEL, "finos", null); + String all = python.values().stream().map(CharSequence::toString).collect(Collectors.joining("\n")); + testUtils.assertGeneratedContainsExpectedString(all, """ + _CHOICE_DISPATCH: ClassVar[dict[str, str]] = { + "com.rosetta.test.model.A": "A", + "com.rosetta.test.model.B": "B", + } + """); + testUtils.assertGeneratedDoesNotContain(all, "\"finos.com.rosetta.test.model."); + } + + @Test + public void testTypesThatAreNotChoicesHaveNoTable() { + String model = """ + type A: + value string (1..1) + + type B: + value string (1..1) + + type ViaOneOf: + A A (0..1) + B B (0..1) + + condition Choice: + one-of + """; + testUtils.assertGeneratedDoesNotContain(generated(model, "rosetta/test/model/ViaOneOf.py"), "_CHOICE_DISPATCH"); + testUtils.assertGeneratedDoesNotContain(generated(model, "rosetta/test/model/A.py"), "_CHOICE_DISPATCH"); + } + + @Test + public void testSameLeafThroughTwoOptionsFailsGeneration() { + String model = """ + type A: + value string (1..1) + type B: + value string (1..1) + type C: + value string (1..1) + + choice ChoiceAB: + A + B + choice ChoiceAC: + A + C + + choice Ambiguous: + ChoiceAB + ChoiceAC + """; + RuntimeException e = assertThrows(RuntimeException.class, () -> testUtils.generatePythonFromString(model)); + String messages = causeMessages(e); + assertTrue(messages.contains("com.rosetta.test.model.Ambiguous") + && messages.contains("'com.rosetta.test.model.A'") + && messages.contains("'ChoiceAB'") + && messages.contains("'ChoiceAC'"), + "Expected the choice, the key and both options in the error, but was: " + messages); + } + + private static String causeMessages(Throwable t) { + StringBuilder messages = new StringBuilder(); + for (Throwable c = t; c != null; c = c.getCause()) { + messages.append(c.getMessage()).append('\n'); + } + return messages.toString(); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceMarkerTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceMarkerTest.java new file mode 100644 index 00000000..6780d609 --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonChoiceMarkerTest.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.object; + +import java.util.Map; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * A class generated from a {@code choice} is marked as one, because the runtime reads and writes + * it in a different JSON format. A type with optional attributes and a {@code one-of} condition + * generates the same fields and condition, so the marker is the only thing that tells them apart. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +public class PythonChoiceMarkerTest { + + /** The class attribute that marks a class generated from a choice. */ + private static final String MARKER = "_RUNE_CHOICE: ClassVar[bool] = True"; + + /** + * The Rune model the tests generate Python from. + * LoopOption and LoopHolder refer to each other through LoopChoice, so all three are bundled. + */ + private static final String MODEL = """ + type A: + fieldA string (1..1) + + type B: + fieldB string (1..1) + + choice ChoiceData: + A + B + + choice ChoiceNested: + ChoiceData + string + + type ViaOneOf: + A A (0..1) + B B (0..1) + + condition Choice: + one-of + + type LoopOption: + back LoopHolder (0..1) + + choice LoopChoice: + LoopOption + B + + type LoopHolder: + option LoopChoice (0..1) + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String file) { + Map python = testUtils.generatePythonFromString(MODEL); + return python.get("src/com/" + file).toString(); + } + + @Test + public void testChoiceIsMarked() { + testUtils.assertAppearsAfter(generated("rosetta/test/model/ChoiceData.py"), + "class ChoiceData(BaseDataClass):", " " + MARKER); + } + + @Test + public void testNestedChoiceIsMarked() { + testUtils.assertAppearsAfter(generated("rosetta/test/model/ChoiceNested.py"), + "class ChoiceNested(BaseDataClass):", " " + MARKER); + } + + @Test + public void testBundledChoiceIsMarked() { + testUtils.assertAppearsAfter(generated("_bundle.py"), + "class com_rosetta_test_model_LoopChoice(BaseDataClass):", " " + MARKER); + } + + @Test + public void testTypeWithOneOfConditionIsNotMarked() { + testUtils.assertGeneratedDoesNotContain(generated("rosetta/test/model/ViaOneOf.py"), "_RUNE_CHOICE"); + } + + @Test + public void testOrdinaryTypeIsNotMarked() { + testUtils.assertGeneratedDoesNotContain(generated("rosetta/test/model/A.py"), "_RUNE_CHOICE"); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonCircularDependencyTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonCircularDependencyTest.java index b22e86de..66ca9e0a 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonCircularDependencyTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonCircularDependencyTest.java @@ -144,12 +144,14 @@ a CircularA (1..1) testUtils.assertGeneratedContainsExpectedString(proxyCircularA, "_v = _b.com_rosetta_test_model_CircularA"); testUtils.assertGeneratedContainsExpectedString(proxyCircularA, "globals()['CircularA'] = _v"); testUtils.assertGeneratedContainsExpectedString(proxyCircularA, "# EOF"); + testUtils.assertGeneratedDoesNotContain(proxyCircularA, "class "); String proxyCircularB = gfAttr.get("src/com/rosetta/test/model/CircularB.py").toString(); testUtils.assertGeneratedContainsExpectedString(proxyCircularB, "def __getattr__(name: str):"); testUtils.assertGeneratedContainsExpectedString(proxyCircularB, "_v = _b.com_rosetta_test_model_CircularB"); testUtils.assertGeneratedContainsExpectedString(proxyCircularB, "globals()['CircularB'] = _v"); testUtils.assertGeneratedContainsExpectedString(proxyCircularB, "# EOF"); + testUtils.assertGeneratedDoesNotContain(proxyCircularB, "class "); // Inheritance cycle: Parent ↔ Child (default namespace → com._bundle) Map gfInh = testUtils.generatePythonFromString( @@ -261,4 +263,124 @@ a A (0..1) testUtils.assertGeneratedContainsExpectedString(proxyB, "globals()['B'] = _v"); testUtils.assertGeneratedContainsExpectedString(proxyB, "# EOF"); } + + /** + * A standalone function refers to a bundled type by its stub name, which its file imports. + * The bundle name is only defined in _bundle.py. + */ + @Test + public void testStandaloneFunctionUsesStubNameForBundledType() { + Map python = testUtils.generatePythonFromString( + """ + type GroupB1: + value number (0..1) + groupB2 GroupB2 (0..1) + + type GroupB2: + groupB1 GroupB1 (0..1) + + func MakeGroupB1: + inputs: + value number (0..1) + output: + result GroupB1 (0..1) + set result: + GroupB1 { value: value, ... } + + func SetPathGroupB1: + inputs: + value number (0..1) + output: + result GroupB1 (0..1) + set result -> value: + value + """); + String make = python.get("src/com/rosetta/test/model/functions/MakeGroupB1.py").toString(); + testUtils.assertGeneratedContainsExpectedString(make, + "from com.rosetta.test.model.GroupB1 import GroupB1"); + testUtils.assertGeneratedContainsExpectedString(make, "result = GroupB1(value="); + testUtils.assertGeneratedDoesNotContain(make, "com_rosetta_test_model_GroupB1"); + + String setPath = python.get("src/com/rosetta/test/model/functions/SetPathGroupB1.py").toString(); + testUtils.assertGeneratedContainsExpectedString(setPath, "result = ObjectBuilder(GroupB1)"); + testUtils.assertGeneratedDoesNotContain(setPath, "com_rosetta_test_model_GroupB1"); + } + + /** + * A standalone class condition imports a bundled type it builds, inside the condition, and a + * bundled condition does the same for a standalone type. The bundle itself keeps the bundle + * name for bundled types. + */ + @Test + public void testConditionsImportTypesTheyBuild() { + Map python = testUtils.generatePythonFromString( + """ + type GroupB1: + value number (0..1) + groupB2 GroupB2 (0..1) + + type GroupB2: + groupB1 GroupB1 (0..1) + + condition BuildsStandaloneType: + Leaf { value: 1 } -> value = 1 + + condition BuildsBundledType: + GroupB1 { value: 1, ... } -> value = 1 + + type Leaf: + value number (0..1) + + type Checker: + value number (0..1) + + condition BuildsBundledType: + GroupB1 { value: 1, ... } -> value = 1 + """); + String checker = python.get("src/com/rosetta/test/model/Checker.py").toString(); + testUtils.assertAppearsAfter(checker, "def condition_0_BuildsBundledType(self):", + " from com.rosetta.test.model.GroupB1 import GroupB1"); + testUtils.assertGeneratedDoesNotContain(checker, "com_rosetta_test_model_GroupB1"); + + String bundle = python.get("src/com/_bundle.py").toString(); + testUtils.assertAppearsAfter(bundle, "def condition_0_BuildsStandaloneType(self):", + " from com.rosetta.test.model.Leaf import Leaf"); + testUtils.assertGeneratedContainsExpectedString(bundle, "com_rosetta_test_model_GroupB1(value=1)"); + } + + /** + * A bundled field gets its metadata serializer and validator on the field pydantic builds, + * not only in __annotations__, so its metadata (such as @key:scoped) is written. A bundled + * field inherited by a bundled subclass gets them on the subclass too. A field with no + * serializer or validator gets no metadata line. + */ + @Test + public void testBundledFieldGetsMetadataSerializerAndValidator() { + Map python = testUtils.generatePythonFromString( + """ + type Measure: + value number (0..1) + + type Holder: + measure Measure (0..1) + [metadata location] + back Back (0..1) + + type ChildHolder extends Holder: + extra number (0..1) + + type Back: + holder Holder (0..1) + """); + String bundle = python.get("src/com/_bundle.py").toString(); + testUtils.assertGeneratedContainsExpectedString(bundle, + "com_rosetta_test_model_Holder.model_fields[\"measure\"].metadata += " + + "[Measure.serializer(), Measure.validator(('@key:scoped', ))]"); + testUtils.assertGeneratedContainsExpectedString(bundle, + "com_rosetta_test_model_ChildHolder.model_fields[\"measure\"].metadata += " + + "[Measure.serializer(), Measure.validator(('@key:scoped', ))]"); + testUtils.assertAppearsAfter(bundle, + "com_rosetta_test_model_Holder.__annotations__[\"measure\"] = ", + "com_rosetta_test_model_Holder.model_fields[\"measure\"].metadata += "); + } } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonConditionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonConditionTest.java index f88319e5..6c96d612 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonConditionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonConditionTest.java @@ -309,7 +309,7 @@ multiAttr number (1..*) testUtils.assertGeneratedContainsExpectedString(pythonString, "class CondTest(BaseDataClass):"); testUtils.assertGeneratedContainsExpectedString(pythonString, - "return rune_all_elements((lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(items, (str, dict, bytes, bytearray))) else ([items] if items is not None else [])) if x is not None))(rune_resolve_attr(self, \"multiAttr\")), \">=\", 0)"); + "return rune_all_elements((lambda items: sum(1 for x in (items if (hasattr(items, '__iter__') and not isinstance(rune_unwrap(items), (str, dict, bytes, bytearray, BaseDataClass))) else ([items] if items is not None else [])) if x is not None))(rune_resolve_attr(self, \"multiAttr\")), \">=\", 0)"); } /** @@ -390,6 +390,7 @@ a1 string (0..1) """, """ class A(BaseDataClass): + model_config = ConfigDict(defer_build=True) a0: Optional[str] = Field(None, description='') a1: Optional[str] = Field(None, description='') @@ -415,6 +416,7 @@ testTypeValue2 string (0..1) <"Test optional string"> pythonString, """ class TestType(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Test type description. \""" @@ -451,6 +453,7 @@ attr2 string (0..1) """, """ class AttributeIfThenTest(BaseDataClass): + model_config = ConfigDict(defer_build=True) attr1: Optional[str] = Field(None, description='') attr2: Optional[str] = Field(None, description='') @@ -494,6 +497,7 @@ aValue A (1..1) pythonString, """ class A(BaseDataClass): + model_config = ConfigDict(defer_build=True) a0: Optional[int] = Field(None, description='') a1: Optional[int] = Field(None, description='') @@ -506,6 +510,7 @@ def condition_0_(self): pythonString, """ class B(BaseDataClass): + model_config = ConfigDict(defer_build=True) intValue1: Optional[int] = Field(None, description='') intValue2: Optional[int] = Field(None, description='') aValue: Annotated[A, A.serializer(), A.validator()] = Field(..., description='') @@ -557,6 +562,7 @@ attr3 string (0..1) """, """ class AttributeIfThenElseTest(BaseDataClass): + model_config = ConfigDict(defer_build=True) attr1: Optional[str] = Field(None, description='') attr2: Optional[str] = Field(None, description='') attr3: Optional[str] = Field(None, description='') @@ -586,6 +592,7 @@ b number (0..1) """, """ class Foo(BaseDataClass): + model_config = ConfigDict(defer_build=True) a: Optional[Decimal] = Field(None, description='') b: Optional[Decimal] = Field(None, description='') @@ -620,6 +627,7 @@ stringType string (0..1) String choicePython = python.get("src/test/generated_syntax/semantic/Choice.py").toString(); String expectedChoice = """ class Choice(BaseDataClass): + model_config = ConfigDict(defer_build=True) intType: Optional[int] = Field(None, description='') stringType: Optional[str] = Field(None, description='') @@ -632,13 +640,13 @@ def condition_0_Choice(self): } /** - * When a bundled type's condition calls a standalone function, the bundle must contain - * a deferred import for that function so it is resolvable at runtime. - * A and B form a cycle (both bundled); IsPositive is acyclic (standalone). + * When a bundled type's condition calls a standalone function, the condition imports the + * function itself. A module-level import would make the bundle import a function that + * imports the bundle. A and B form a cycle (both bundled); IsPositive is acyclic (standalone). */ @Test public void testBundledConditionWithStandaloneFunctionCallGeneratesImport() { - testUtils.assertBundleContainsExpectedString( + String bundle = testUtils.generatePythonAndExtractBundle( """ func IsPositive: inputs: @@ -656,8 +664,40 @@ then IsPositive( value ) = True type B: a A (0..1) - """, - "from com.rosetta.test.model.functions.IsPositive import IsPositive"); + """); + testUtils.assertAppearsAfter(bundle, + "def condition_0_PositiveValue(self):", + " from com.rosetta.test.model.functions.IsPositive import IsPositive"); + testUtils.assertGeneratedDoesNotContain(bundle, + "\nfrom com.rosetta.test.model.functions.IsPositive import IsPositive"); + } + + /** + * A standalone type's condition imports the function it calls, inside the condition, so + * the module can be imported first whatever the function itself imports. + */ + @Test + public void testStandaloneConditionWithFunctionCallGeneratesImport() { + Map python = testUtils.generatePythonFromString( + """ + func IsPositive: + inputs: + value number (0..1) + output: + result boolean (1..1) + + type Holder: + value number (0..1) + + condition PositiveValue: + IsPositive( value ) + """); + String holder = python.get("src/com/rosetta/test/model/Holder.py").toString(); + testUtils.assertAppearsAfter(holder, + "def condition_0_PositiveValue(self):", + " from com.rosetta.test.model.functions.IsPositive import IsPositive"); + testUtils.assertGeneratedDoesNotContain(holder, + "\nfrom com.rosetta.test.model.functions.IsPositive import IsPositive"); } /** diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonEnumTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonEnumTest.java index 871f33d5..97ebe30c 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonEnumTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonEnumTest.java @@ -64,6 +64,7 @@ legalEntity LegalEntity (0..1) String expectedTestType1 = """ class LegalEntity(BaseDataClass): + model_config = ConfigDict(defer_build=True) _ALLOWED_METADATA = {'@key', '@key:external'} \""" A class to specify a legal entity, with a required name and an optional entity identifier (such as the LEI). @@ -79,6 +80,7 @@ A legal entity identifier (e.g. RED entity code). """; String expectedTestType2 = """ class TelephoneNumber(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" A class to specify a telephone number as a type of phone number (e.g. work, personal, ...) alongside with the actual number. \""" @@ -93,6 +95,7 @@ A class to specify a telephone number as a type of phone number (e.g. work, pers """; String expectedTestType3 = """ class AncillaryEntity(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Holds an identifier for an ancillary entity, either identified directly via its ancillary role or directly as a legal entity. \""" @@ -100,7 +103,7 @@ class AncillaryEntity(BaseDataClass): \""" Identifies a party via its ancillary role on a transaction (e.g. CCP or DCO through which the trade test be cleared.) \""" - legalEntity: Annotated[Optional[LegalEntity], LegalEntity.serializer(), LegalEntity.validator()] = Field(None, description='') + legalEntity: Annotated[Optional[LegalEntity], LegalEntity.serializer(), LegalEntity.validator(('@key', '@key:external'))] = Field(None, description='') @rune_condition def condition_0_(self): @@ -174,6 +177,7 @@ weatherUnit WeatherUnitEnum (0..1) <"Provides an enumerated values for a weather String expectedTestType = """ class UnitType(BaseDataClass): + model_config = ConfigDict(defer_build=True) \""" Defines the unit to be used for price, quantity, or other purposes \""" diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonInheritanceTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonInheritanceTest.java index 9c0643b9..d0cc5312 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonInheritanceTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonInheritanceTest.java @@ -43,24 +43,28 @@ aa string (0..1) String expectedA = """ class A(BaseDataClass): + model_config = ConfigDict(defer_build=True) aa: Optional[str] = Field(None, description='') """; String expectedB = """ from com.rosetta.test.model.A import A class B(A): + model_config = ConfigDict(defer_build=True) bb: Optional[str] = Field(None, description='') """; String expectedC = """ from com.rosetta.test.model.B import B class C(B): + model_config = ConfigDict(defer_build=True) cc: Optional[str] = Field(None, description='') """; String expectedD = """ from com.rosetta.test.model.C import C class D(C): + model_config = ConfigDict(defer_build=True) dd: Optional[str] = Field(None, description='') """; testUtils.assertGeneratedContainsExpectedString(pythonString, expectedA); @@ -88,6 +92,7 @@ public void testSuperClasses() { String expectedBaz = """ class Baz(BaseDataClass): + model_config = ConfigDict(defer_build=True) pass """; @@ -95,6 +100,7 @@ class Baz(BaseDataClass): from test.Baz import Baz class Bar(Baz): + model_config = ConfigDict(defer_build=True) pass """; @@ -102,6 +108,7 @@ class Bar(Baz): from test.Bar import Bar class Foo(Bar): + model_config = ConfigDict(defer_build=True) pass """; @@ -168,6 +175,7 @@ b string (1..1) """, """ class B(BaseDataClass): + model_config = ConfigDict(defer_build=True) b: str = Field(..., description='') """); @@ -181,6 +189,7 @@ b string (1..1) """, """ class A(B): + model_config = ConfigDict(defer_build=True) a: str = Field(..., description='') """); } @@ -198,6 +207,7 @@ override a string (0..1) """, """ class Foo(BaseDataClass): + model_config = ConfigDict(defer_build=True) a: Optional[str] = Field(None, description='') b: Optional[str] = Field(None, description='') """); @@ -213,6 +223,7 @@ override a string (0..1) """, """ class Bar(Foo): + model_config = ConfigDict(defer_build=True) a: Optional[str] = Field(None, description='') """); } @@ -228,6 +239,7 @@ b string (1..1) """, """ class A(B): + model_config = ConfigDict(defer_build=True) pass """); } @@ -270,6 +282,7 @@ enum FinancialUnitEnum: <"Provides enumerated values for financial units, genera testUtils.assertBundleContainsExpectedString(model, """ class MeasureBase(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Provides an abstract type to define a measure as a number associated to a unit. This type is abstract because all its attributes are optional. The types that extend it can specify further existence constraints. \"\"\" @@ -286,6 +299,7 @@ class MeasureBase(BaseDataClass): testUtils.assertBundleContainsExpectedString(model, """ class Measure(MeasureBase): + model_config = ConfigDict(defer_build=True) \"\"\" Defines a concrete measure as a number associated to a unit. It extends MeasureBase by requiring the value attribute to be present. A measure may be unit-less so the unit attribute is still optional. \"\"\" @@ -299,6 +313,7 @@ def condition_0_ValueExists(self): testUtils.assertBundleContainsExpectedString(model, """ class UnitType(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Defines the unit to be used for price, quantity, or other purposes \"\"\" diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonMetadataTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonMetadataTest.java index 17b5e9a1..01db2815 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonMetadataTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonMetadataTest.java @@ -101,7 +101,7 @@ public void testReferenceMetadata() { String nodeRefPython = getPython().get("src/test/generated_syntax/metadata/NodeRef.py").toString(); testUtils.assertGeneratedContainsExpectedString(nodeRefPython, "class NodeRef(BaseDataClass):"); testUtils.assertGeneratedContainsExpectedString(nodeRefPython, - "typeA: Annotated[Optional[A], A.serializer(), A.validator()] = Field(None, description='')"); + "typeA: Annotated[Optional[A], A.serializer(), A.validator(('@key', '@key:external'))] = Field(None, description='')"); testUtils.assertGeneratedContainsExpectedString(nodeRefPython, "aReference: Annotated[Optional[A | BaseReference], A.serializer(), A.validator(('@key', '@key:external', '@ref', '@ref:external'))] = Field(None, description='')"); } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonNameCollisionTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonNameCollisionTest.java index a0b23033..9291e94b 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonNameCollisionTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonNameCollisionTest.java @@ -51,7 +51,7 @@ result CollidingName(1..1) String classPython = gf.get("src/test/collision/CollidingName.py").toString(); testUtils.assertGeneratedContainsExpectedString(classPython, "class CollidingName(BaseDataClass):"); testUtils.assertGeneratedContainsExpectedString(classPython, - "other: Annotated[Optional[CollidingName | BaseReference], CollidingName.serializer(), CollidingName.validator(('@key', '@key:external', '@ref', '@ref:external'))] = Field(None, description='')"); + "other: Annotated[Optional[CollidingName | BaseReference], CollidingName.serializer(), CollidingName.validator(('@key', '@key:external', '@ref', '@ref:external'))] = Field(None, description='')"); // CollidingName function is also standalone. String funcPython = gf.get("src/test/collision/functions/CollidingName.py").toString(); diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonPartitioningTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonPartitioningTest.java index ae02b16c..a8612d8b 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonPartitioningTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonPartitioningTest.java @@ -4,6 +4,7 @@ */ package com.regnosys.rosetta.generator.python.object; +import java.util.List; import java.util.Map; import org.eclipse.xtext.testing.InjectWith; @@ -276,14 +277,15 @@ a CycleA (1..1) } // ----------------------------------------------------------------------- - // Test 12 — standalone types referenced by bundled types are reverse-promoted - // into the bundle; no scattered imports remain in the bundle body + // Test 12 — standalone attribute types stay standalone with a deferred import, + // unless they are on a path between bundled types // ----------------------------------------------------------------------- /** * A standalone own-type used only as an attribute type (not as a direct base class) - * of a bundled type is NOT reverse-promoted. Attribute annotations are lazy strings - * under PEP 563 and do not need the type at class-definition time. The import is - * deferred to a consolidated section after all bundled class definitions. + * of a bundled type, and which does not depend on a bundled type, is NOT promoted. + * Attribute annotations are lazy strings under PEP 563 and do not need the type at + * class-definition time. The import is deferred to a consolidated section after all + * bundled class definitions. * * Model structure: * CycleA1 ↔ CycleA2 (first SCC — bundled) @@ -309,7 +311,7 @@ s Standalone (0..1) b1 CycleB1 (0..1) type Standalone: - a1 CycleA1 (0..1) + value int (0..1) """); String bundlePython = gf.get("src/com/_bundle.py").toString(); @@ -330,6 +332,69 @@ a1 CycleA1 (0..1) "from com.rosetta.test.model.Standalone import Standalone"); } + /** + * Types on a path between bundled types are promoted, however long the path. The two + * cycle groups are separate in the dependency graph, so the path is not a cycle, but + * they share one _bundle.py, so leaving Outer or Inner standalone would make the files + * circular. + * + * Model structure: + * CycleB1 ↔ CycleB2 (first SCC — bundled; CycleB1 has attribute of type Outer) + * Outer → Inner → CycleA1 (promoted — each depends on a bundled type) + * CycleA1 ↔ CycleA2 (second SCC — bundled) + * Unrelated (used by CycleB1, depends on nothing — stays standalone) + */ + @Test + public void testTypesOnPathBetweenBundledTypesArePromoted() { + Map gf = testUtils.generatePythonFromString(""" + namespace com.rosetta.test.model + + type CycleA1: + a2 CycleA2 (0..1) + + type CycleA2: + a1 CycleA1 (0..1) + + type CycleB1: + b2 CycleB2 (0..1) + outer Outer (0..1) + unrelated Unrelated (0..1) + + type CycleB2: + b1 CycleB1 (0..1) + + type Outer: + inner Inner (0..1) + + type Inner: + a1 CycleA1 (0..1) + + type Unrelated: + value int (0..1) + """); + + String bundlePython = gf.get("src/com/_bundle.py").toString(); + + for (String promoted : List.of("Outer", "Inner")) { + testUtils.assertGeneratedContainsExpectedString(bundlePython, + "class com_rosetta_test_model_" + promoted + "(BaseDataClass):"); + testUtils.assertGeneratedContainsExpectedString(bundlePython, + "com_rosetta_test_model_" + promoted + ".model_rebuild(force=True)"); + testUtils.assertGeneratedDoesNotContain(bundlePython, + "from com.rosetta.test.model." + promoted + " import " + promoted); + + String stubPython = gf.get("src/com/rosetta/test/model/" + promoted + ".py").toString(); + testUtils.assertGeneratedContainsExpectedString(stubPython, "__getattr__"); + testUtils.assertGeneratedDoesNotContain(stubPython, "class " + promoted + "(BaseDataClass):"); + } + + // A type the bundle uses, but which does not lead back to it, stays standalone + String unrelatedPython = gf.get("src/com/rosetta/test/model/Unrelated.py").toString(); + testUtils.assertGeneratedContainsExpectedString(unrelatedPython, "class Unrelated(BaseDataClass):"); + testUtils.assertGeneratedContainsExpectedString(bundlePython, + "from com.rosetta.test.model.Unrelated import Unrelated"); + } + /** * A standalone own-type used as a direct base class of a bundled type is * reverse-promoted into the bundle. The bundled subclass references the base diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonScalarTypeTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonScalarTypeTest.java index 787be8d3..3c8acffd 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonScalarTypeTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonScalarTypeTest.java @@ -34,6 +34,7 @@ list string (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[str] = Field(None, description='') list: list[str | None] = Field(..., description='', min_length=1)"""); } @@ -48,6 +49,7 @@ list int (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[int] = Field(None, description='') list: list[int | None] = Field(..., description='', min_length=1)"""); } @@ -62,6 +64,7 @@ list number (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[Decimal] = Field(None, description='') list: list[Decimal | None] = Field(..., description='', min_length=1)"""); } @@ -76,6 +79,7 @@ list boolean (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[bool] = Field(None, description='') list: list[bool | None] = Field(..., description='', min_length=1)"""); } @@ -90,6 +94,7 @@ list date (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[datetime.date] = Field(None, description='') list: list[datetime.date | None] = Field(..., description='', min_length=1)"""); } @@ -105,6 +110,7 @@ zoned zonedDateTime (0..1) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[datetime.datetime] = Field(None, description='') list: list[datetime.datetime | None] = Field(..., description='', min_length=1) zoned: Optional[datetime.datetime] = Field(None, description='')"""); @@ -120,6 +126,7 @@ list time (1..*) """, """ class Tester(BaseDataClass): + model_config = ConfigDict(defer_build=True) one: Optional[datetime.time] = Field(None, description='') list: list[datetime.time | None] = Field(..., description='', min_length=1)"""); } @@ -133,6 +140,7 @@ withoutGlobalKey string (1..1) """, """ class AttributeGlobalKeyTest(BaseDataClass): + model_config = ConfigDict(defer_build=True) withoutGlobalKey: str = Field(..., description='')"""); } @@ -145,6 +153,7 @@ bar calculation (0..1) """, """ class Foo(BaseDataClass): + model_config = ConfigDict(defer_build=True) bar: Optional[str] = Field(None, description='')"""); } @@ -170,6 +179,7 @@ enum TestEnum: testUtils.assertBundleContainsExpectedString(model, """ class TestType2(BaseDataClass): + model_config = ConfigDict(defer_build=True) testType2Value1: list[Decimal | None] = Field(..., description='', min_length=1) testType2Value2: Optional[datetime.date] = Field(None, description='') testEnum: Optional[com.rosetta.test.model.TestEnum.TestEnum] = Field(None, description='') @@ -178,6 +188,7 @@ class TestType2(BaseDataClass): testUtils.assertBundleContainsExpectedString(model, """ class TestType(BaseDataClass): + model_config = ConfigDict(defer_build=True) \"\"\" Test type description. \"\"\" @@ -207,12 +218,14 @@ multiplierUnit UnitType (0..1) testUtils.assertBundleContainsExpectedString(model, """ class UnitType(BaseDataClass): + model_config = ConfigDict(defer_build=True) currency: Optional[str] = Field(None, description='') """); testUtils.assertBundleContainsExpectedString(model, """ class MeasureBase(BaseDataClass): + model_config = ConfigDict(defer_build=True) amount: Decimal = Field(..., description='') unitOfAmount: Annotated[UnitType, UnitType.serializer(), UnitType.validator()] = Field(..., description='') """); @@ -220,6 +233,7 @@ class MeasureBase(BaseDataClass): testUtils.assertBundleContainsExpectedString(model, """ class Quantity(MeasureBase): + model_config = ConfigDict(defer_build=True) multiplier: Optional[Decimal] = Field(None, description='') multiplierUnit: Annotated[Optional[UnitType], UnitType.serializer(), UnitType.validator()] = Field(None, description='') """); @@ -235,6 +249,7 @@ field string (1..1) <"This is a multiline """, """ class MultilineDefinition(BaseDataClass): + model_config = ConfigDict(defer_build=True) field: str = Field(..., description='This is a multiline definition for a field.')"""); } } diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonStandaloneStructureTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonStandaloneStructureTest.java index f32be8dc..cfff9aca 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonStandaloneStructureTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonStandaloneStructureTest.java @@ -121,6 +121,7 @@ public void testExpectedBasicSingle() { String generatedPython = gf.get("src/test/generated_syntax/basic/BasicSingle.py").toString(); String expectedBasicSingle = """ class BasicSingle(BaseDataClass): + model_config = ConfigDict(defer_build=True) booleanType: bool = Field(..., description='') numberType: Decimal = Field(..., description='') parameterisedNumberType: Decimal = Field(..., description='', max_digits=18, decimal_places=2) @@ -140,6 +141,7 @@ public void testExpectedBasicList() { String generatedPython = gf.get("src/test/generated_syntax/basic/BasicList.py").toString(); String expectedBasicList = """ class BasicList(BaseDataClass): + model_config = ConfigDict(defer_build=True) booleanTypes: list[bool | None] = Field(..., description='', min_length=1) numberTypes: list[Decimal | None] = Field(..., description='', min_length=1) parameterisedNumberTypes: list[Annotated[Decimal, Field(max_digits=18, decimal_places=2)] | None] = Field(..., description='', min_length=1) @@ -160,6 +162,7 @@ public void testExpectedRoot() { String generatedPython = gf.get("src/test/generated_syntax/basic/Root.py").toString(); String expectedRoot = """ class Root(BaseDataClass): + model_config = ConfigDict(defer_build=True) basicSingle: Annotated[Optional[BasicSingle], BasicSingle.serializer(), BasicSingle.validator()] = Field(None, description='') basicList: Annotated[Optional[BasicList], BasicList.serializer(), BasicList.validator()] = Field(None, description='') """; diff --git a/src/test/java/com/regnosys/rosetta/generator/python/object/PythonTypeLevelKeyTest.java b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonTypeLevelKeyTest.java new file mode 100644 index 00000000..c489fe81 --- /dev/null +++ b/src/test/java/com/regnosys/rosetta/generator/python/object/PythonTypeLevelKeyTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC + * SPDX-License-Identifier: Apache-2.0 + */ +package com.regnosys.rosetta.generator.python.object; + +import java.util.Map; + +import org.eclipse.xtext.testing.InjectWith; +import org.eclipse.xtext.testing.extensions.InjectionExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; +import com.regnosys.rosetta.tests.RosettaInjectorProvider; + +import jakarta.inject.Inject; + +/** + * A field whose type has {@code [metadata key]} gets the runtime's serializer and a validator + * that allows the type's keys, so that {@code @key} and {@code @key:external} are written. The + * runtime writes an object's keys only through the field's serializer. + */ +@ExtendWith(InjectionExtension.class) +@InjectWith(RosettaInjectorProvider.class) +public class PythonTypeLevelKeyTest { + + /** The serializer and validator of a field whose type, {@code Party}, has type-level keys. */ + private static final String HOOKS = "Party.serializer(), Party.validator(('@key', '@key:external'))"; + + /** + * The Rune model the tests generate Python from. + * SingleParty and PartyList are standalone. BundledSingle and Loop refer to each other, so + * they are bundled. + */ + private static final String MODEL = """ + type Party: + [metadata key] + name string (0..1) + + type SingleParty: + party Party (0..1) + + type PartyList: + parties Party (0..*) + + type BundledSingle: + party Party (0..1) + loop Loop (0..1) + + type Loop: + back BundledSingle (0..1) + """; + + /** Injected test utilities. */ + @Inject + private PythonGeneratorTestUtils testUtils; + + private String generated(String file) { + Map python = testUtils.generatePythonFromString(MODEL); + return python.get("src/com/" + file).toString(); + } + + @Test + public void testStandaloneFieldOfKeyedTypeGetsHooks() { + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/SingleParty.py"), + "party: Annotated[Optional[Party], " + HOOKS + "] = Field(None, description='')"); + } + + @Test + public void testStandaloneListOfKeyedTypeGetsHooks() { + testUtils.assertGeneratedContainsExpectedString(generated("rosetta/test/model/PartyList.py"), + "parties: Annotated[Optional[list[Party | None]], " + HOOKS + "] = Field(None, description='')"); + } + + @Test + public void testBundledFieldOfKeyedTypeAllowsTheTypesKeys() { + String bundle = generated("_bundle.py"); + testUtils.assertGeneratedContainsExpectedString(bundle, + "com_rosetta_test_model_BundledSingle.__annotations__[\"party\"] = Annotated[Optional[Party], " + + HOOKS + "]"); + testUtils.assertGeneratedContainsExpectedString(bundle, + "com_rosetta_test_model_BundledSingle.model_fields[\"party\"].metadata += [" + HOOKS + "]"); + } +} diff --git a/src/test/java/com/regnosys/rosetta/generator/python/PythonTomlGeneratorTest.java b/src/test/java/com/regnosys/rosetta/generator/python/util/PythonTomlGeneratorTest.java similarity index 65% rename from src/test/java/com/regnosys/rosetta/generator/python/PythonTomlGeneratorTest.java rename to src/test/java/com/regnosys/rosetta/generator/python/util/PythonTomlGeneratorTest.java index 9aedea4a..bac991d0 100644 --- a/src/test/java/com/regnosys/rosetta/generator/python/PythonTomlGeneratorTest.java +++ b/src/test/java/com/regnosys/rosetta/generator/python/util/PythonTomlGeneratorTest.java @@ -2,8 +2,9 @@ * Copyright (c) 2023-2026 CLOUDRISK Limited and FT Advisory LLC * SPDX-License-Identifier: Apache-2.0 */ -package com.regnosys.rosetta.generator.python; +package com.regnosys.rosetta.generator.python.util; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Map; @@ -13,6 +14,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import com.regnosys.rosetta.generator.python.PythonGeneratorTestUtils; import com.regnosys.rosetta.tests.RosettaInjectorProvider; import jakarta.inject.Inject; @@ -67,8 +69,8 @@ void testTomlVersionWhenSpecified() { void testTomlRequiresRuntimeVersionThreeOrLater() { Map python = testUtils.generatePythonFromString(MODEL); String toml = python.get("pyproject.toml").toString(); - assertTrue(toml.contains("\"rune.runtime>=3.0.0,<4.0.0\""), - "Expected pyproject.toml to require rune.runtime>=3.0.0,<4.0.0 but was:\n" + toml); + assertTrue(toml.contains("\"rune.runtime>=3.1.0,<4.0.0\""), + "Expected pyproject.toml to require rune.runtime>=3.1.0,<4.0.0 but was:\n" + toml); } /** @@ -86,4 +88,33 @@ void testTomlProjectNameWithNamespacePrefix() { assertTrue(toml.contains("name = \"python-finos\""), "Expected pyproject.toml to contain name = \"python-finos\" but was:\n" + toml); } + + /** + * pyproject.toml names README.md as the package description, so the generator must write it. + * Without it, building the package warns that the file cannot be found. + */ + @Test + void testReadmeNamedByTomlIsGenerated() { + Map python = testUtils.generatePythonFromString(MODEL, null, "1.2.3"); + String toml = python.get("pyproject.toml").toString(); + assertTrue(toml.contains("readme = \"README.md\""), + "Expected pyproject.toml to name README.md but was:\n" + toml); + CharSequence readme = python.get("README.md"); + assertNotNull(readme, "Expected README.md to be generated alongside pyproject.toml"); + assertTrue(readme.toString().contains("# python-test"), + "Expected README.md to name the package but was:\n" + readme); + assertTrue(readme.toString().contains("Version: 1.2.3"), + "Expected README.md to give the version but was:\n" + readme); + } + + /** + * The README names the same package as pyproject.toml, including any namespace prefix. + */ + @Test + void testReadmeUsesProjectNameWithNamespacePrefix() { + Map python = testUtils.generatePythonFromString(MODEL, "finos", null); + String readme = python.get("README.md").toString(); + assertTrue(readme.contains("# python-finos"), + "Expected README.md to name python-finos but was:\n" + readme); + } }