From 240a4829e646a2d0e5fbf723b5037657a3c4fda7 Mon Sep 17 00:00:00 2001 From: martinv13 Date: Tue, 20 Oct 2026 22:39:11 +0200 Subject: [PATCH 1/2] Keep simple values when global transform is false (#81) With the global "transform": false option, simple columns lost their "join" transform, which the parser uses to read simple element values, so these values were loaded as NULL. The global option now only disables elevation and choice collapsing. Add an orders sample model version with global "transform": false so that round-trip tests cover it. Co-Authored-By: Claude Opus 5.5 --- docs/configuring.md | 2 +- src/xml2db/table/transformed_table.py | 11 +- tests/sample_models/models.py | 16 ++ .../orders/orders_ddl_mssql_version3.sql | 251 ++++++++++++++++++ .../orders/orders_ddl_mysql_version3.sql | 251 ++++++++++++++++++ .../orders/orders_ddl_postgresql_version3.sql | 251 ++++++++++++++++++ .../orders/orders_erd_version3.md | 86 ++++++ .../orders/orders_source_tree_version3.txt | 107 ++++++++ .../orders/orders_target_tree_version3.txt | 102 +++++++ 9 files changed, 1071 insertions(+), 6 deletions(-) create mode 100644 tests/sample_models/orders/orders_ddl_mssql_version3.sql create mode 100644 tests/sample_models/orders/orders_ddl_mysql_version3.sql create mode 100644 tests/sample_models/orders/orders_ddl_postgresql_version3.sql create mode 100644 tests/sample_models/orders/orders_erd_version3.md create mode 100644 tests/sample_models/orders/orders_source_tree_version3.txt create mode 100644 tests/sample_models/orders/orders_target_tree_version3.txt diff --git a/docs/configuring.md b/docs/configuring.md index cf0ae91..8d74027 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -106,7 +106,7 @@ functions (defaults to `hashlib.sha1`). deduplication of rows is opted out. This allows recording the original order of elements in the source XML, which is not always respected otherwise. It was implemented primarily for round-trip tests, but could serve other purposes. The default value is `False` (disabled). -* `transform` (`false` or `"auto"`): set to `false` to disable all automatic field transformations globally: no joining of multi-value columns, no elevation of child tables, no collapsing of choice groups. The default `"auto"` applies all of these where applicable. Per-field `transform` and per-table `choice_transform` still override the global setting. +* `transform` (`false` or `"auto"`): set to `false` to disable automatic field transformations globally: no elevation of child tables, no collapsing of choice groups. Multi-value columns are still joined, as opting out of `join` is not supported. The default `"auto"` applies all of these where applicable. Per-field `transform` and per-table `choice_transform` still override the global setting. ## Fields configuration diff --git a/src/xml2db/table/transformed_table.py b/src/xml2db/table/transformed_table.py index 16fc806..389b8d5 100644 --- a/src/xml2db/table/transformed_table.py +++ b/src/xml2db/table/transformed_table.py @@ -150,12 +150,13 @@ def _get_field_transform( f"Transform value '{field_transform}' cannot be applied" f" to field '{field_name}' of table '{self.name}'." ) + # "join" is how simple values are read at parsing stage, so global transform:false does not turn it off + if field_type == "col": + if self._can_transform_field("col", field_name, "join"): + return "join" # "auto" at field level always runs auto-detection, overriding global transform:false - if field_transform == "auto" or self.data_model.model_config.get("transform", True): - if field_type == "col": - if self._can_transform_field("col", field_name, "join"): - return "join" - elif field_type == "rel1": + elif field_transform == "auto" or self.data_model.model_config.get("transform", True): + if field_type == "rel1": if ( self.relations_1[field_name].occurs[0] == 1 or len(self.relations_1[field_name].other_table.columns) <= 4 diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index af1e4c6..3461a84 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -90,6 +90,22 @@ def wrapped(): ], } }, + { + "config": { + "transform": False, + "tables": { + "item": {"fields": {"comment": {"transform": "skip"}}}, + # without elevation, an empty extra element would add a row + "orderperson": {"fields": {"extra": {"transform": "skip"}}}, + }, + "metadata_columns": [ + { + "name": "input_file_path", + "type": sqlalchemy.String(256), + } + ], + } + }, ], }, { diff --git a/tests/sample_models/orders/orders_ddl_mssql_version3.sql b/tests/sample_models/orders/orders_ddl_mssql_version3.sql new file mode 100644 index 0000000..f417f4d --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_mssql_version3.sql @@ -0,0 +1,251 @@ + +CREATE TABLE [carrierId] ( + [pk_carrierId] INTEGER NOT NULL IDENTITY, + vat VARCHAR(1000) NULL, + registry VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT [cx_pk_carrierId] PRIMARY KEY CLUSTERED ([pk_carrierId]), + CONSTRAINT [carrierId_xml2db_record_hash] UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL IDENTITY, + reference VARCHAR(1000) NULL, + carrier VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip INTEGER NOT NULL IDENTITY, + [codingSystem] VARCHAR(1000) NULL, + state VARCHAR(1000) NULL, + value VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_zip PRIMARY KEY CLUSTERED (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE [companyId] ( + [pk_companyId] INTEGER NOT NULL IDENTITY, + ace VARCHAR(1000) NULL, + bic VARCHAR(1000) NULL, + lei VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT [cx_pk_companyId] PRIMARY KEY CLUSTERED ([pk_companyId]), + CONSTRAINT [companyId_xml2db_record_hash] UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + urn VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY CLUSTERED (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail INTEGER NOT NULL IDENTITY, + weight DOUBLE PRECISION NULL, + unit VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail PRIMARY KEY CLUSTERED (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_max_length ( + pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + value INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length PRIMARY KEY CLUSTERED (pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + value VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_stringfeature PRIMARY KEY CLUSTERED (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson INTEGER NOT NULL IDENTITY, + name_attr VARCHAR(1000) NULL, + name VARCHAR(1000) NULL, + address VARCHAR(1000) NULL, + city VARCHAR(1000) NULL, + fk_zip INTEGER NULL, + country VARCHAR(1000) NULL, + [phoneNumber] VARCHAR(8000) NULL, + [fk_companyId] INTEGER NULL, + coordinates VARCHAR(1000) NULL, + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_orderperson PRIMARY KEY CLUSTERED (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY([fk_companyId]) REFERENCES [companyId] ([pk_companyId]) +) + + +CREATE TABLE features ( + pk_features INTEGER NOT NULL IDENTITY, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_features PRIMARY KEY CLUSTERED (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) REFERENCES intfeature_with_peculiarly_long_suffix_which_overflow_max_length (pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery INTEGER NOT NULL IDENTITY, + from_fk_orderperson INTEGER NULL, + to_fk_orderperson INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_delivery PRIMARY KEY CLUSTERED (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product INTEGER NOT NULL IDENTITY, + name VARCHAR(1000) NULL, + version VARCHAR(1000) NULL, + fk_features INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_product PRIMARY KEY CLUSTERED (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item INTEGER NOT NULL IDENTITY, + fk_product INTEGER NULL, + note VARCHAR(1000) NULL, + quantity INTEGER NULL, + price DOUBLE PRECISION NULL, + currency CHAR(3) NULL, + fk_delivery INTEGER NULL, + fk_catalog_ref INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_item PRIMARY KEY CLUSTERED (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder INTEGER NOT NULL IDENTITY, + orderid VARCHAR(1000) NULL, + processed_at DATETIMEOFFSET NULL, + fk_orderperson INTEGER NULL, + shipto_fk_orderperson INTEGER NULL, + [fk_carrierId] INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_shiporder PRIMARY KEY CLUSTERED (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY([fk_carrierId]) REFERENCES [carrierId] ([pk_carrierId]) +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders INTEGER NOT NULL IDENTITY, + batch_id VARCHAR(1000) NULL, + version INTEGER NULL, + input_file_path VARCHAR(256) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_orders PRIMARY KEY CLUSTERED (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length_fk_intfeature_with_peculiarly_long_suffix_w_ef47 ON features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) + +CREATE CLUSTERED INDEX ix_fk_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ON features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_features, fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE CLUSTERED INDEX ix_fk_features_stringfeature ON features_stringfeature (fk_features, fk_stringfeature) + +CREATE CLUSTERED INDEX ix_fk_item_detail ON item_detail (fk_item, fk_detail) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE CLUSTERED INDEX ix_fk_shiporder_item ON shiporder_item (fk_shiporder, fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE CLUSTERED INDEX ix_fk_shiporder_detail_detail_1 ON shiporder_detail_detail_1 (fk_shiporder, fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE CLUSTERED INDEX ix_fk_orders_shiporder ON orders_shiporder (fk_orders, fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_ddl_mysql_version3.sql b/tests/sample_models/orders/orders_ddl_mysql_version3.sql new file mode 100644 index 0000000..0bb1e49 --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_mysql_version3.sql @@ -0,0 +1,251 @@ + +CREATE TABLE `carrierId` ( + `pk_carrierId` INTEGER NOT NULL AUTO_INCREMENT, + vat VARCHAR(255), + registry VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT `cx_pk_carrierId` PRIMARY KEY (`pk_carrierId`), + CONSTRAINT `carrierId_xml2db_record_hash` UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, + reference VARCHAR(255), + carrier VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip INTEGER NOT NULL AUTO_INCREMENT, + `codingSystem` VARCHAR(255), + state VARCHAR(255), + value VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_zip PRIMARY KEY (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE `companyId` ( + `pk_companyId` INTEGER NOT NULL AUTO_INCREMENT, + ace VARCHAR(255), + bic VARCHAR(255), + lei VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT `cx_pk_companyId` PRIMARY KEY (`pk_companyId`), + CONSTRAINT `companyId_xml2db_record_hash` UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + urn VARCHAR(1000), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail INTEGER NOT NULL AUTO_INCREMENT, + weight DOUBLE, + unit VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail PRIMARY KEY (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_ove_5868736 ( + pk_intfeature_with_peculiarly_long_suffix_which__85b659b INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + value INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_whi_ecb17be PRIMARY KEY (pk_intfeature_with_peculiarly_long_suffix_which__85b659b), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + value VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_stringfeature PRIMARY KEY (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson INTEGER NOT NULL AUTO_INCREMENT, + name_attr VARCHAR(255), + name VARCHAR(255), + address VARCHAR(255), + city VARCHAR(255), + fk_zip INTEGER, + country VARCHAR(255), + `phoneNumber` VARCHAR(4000), + `fk_companyId` INTEGER, + coordinates VARCHAR(255), + a_very_long_field_type_that_makes_col_name_excee_223ada0 VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_orderperson PRIMARY KEY (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY(`fk_companyId`) REFERENCES `companyId` (`pk_companyId`) +) + + +CREATE TABLE features ( + pk_features INTEGER NOT NULL AUTO_INCREMENT, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_features PRIMARY KEY (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix__4166a78 ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which__00590e9 INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which__00590e9) REFERENCES intfeature_with_peculiarly_long_suffix_which_ove_5868736 (pk_intfeature_with_peculiarly_long_suffix_which__85b659b) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery INTEGER NOT NULL AUTO_INCREMENT, + from_fk_orderperson INTEGER, + to_fk_orderperson INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_delivery PRIMARY KEY (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product INTEGER NOT NULL AUTO_INCREMENT, + name VARCHAR(255), + version VARCHAR(255), + fk_features INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_product PRIMARY KEY (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item INTEGER NOT NULL AUTO_INCREMENT, + fk_product INTEGER, + note VARCHAR(255), + quantity INTEGER, + price DOUBLE, + currency VARCHAR(3), + fk_delivery INTEGER, + fk_catalog_ref INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder INTEGER NOT NULL AUTO_INCREMENT, + orderid VARCHAR(255), + processed_at DATETIME, + fk_orderperson INTEGER, + shipto_fk_orderperson INTEGER, + `fk_carrierId` INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(`fk_carrierId`) REFERENCES `carrierId` (`pk_carrierId`) +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders INTEGER NOT NULL AUTO_INCREMENT, + batch_id VARCHAR(255), + version INTEGER, + input_file_path VARCHAR(256), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix__4166_6094 ON features_intfeature_with_peculiarly_long_suffix__4166a78 (fk_features) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix__4166_1469 ON features_intfeature_with_peculiarly_long_suffix__4166a78 (fk_intfeature_with_peculiarly_long_suffix_which__00590e9) + +CREATE INDEX ix_features_stringfeature_fk_features ON features_stringfeature (fk_features) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE INDEX ix_item_detail_fk_item ON item_detail (fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE INDEX ix_shiporder_item_fk_shiporder ON shiporder_item (fk_shiporder) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_shiporder ON shiporder_detail_detail_1 (fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_orders ON orders_shiporder (fk_orders) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version3.sql b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql new file mode 100644 index 0000000..7da1b61 --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql @@ -0,0 +1,251 @@ + +CREATE TABLE "carrierId" ( + "pk_carrierId" SERIAL NOT NULL, + vat VARCHAR(1000), + registry VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT "cx_pk_carrierId" PRIMARY KEY ("pk_carrierId"), + CONSTRAINT "carrierId_xml2db_record_hash" UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 SERIAL NOT NULL, + reference VARCHAR(1000), + carrier VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip SERIAL NOT NULL, + "codingSystem" VARCHAR(1000), + state VARCHAR(1000), + value VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_zip PRIMARY KEY (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE "companyId" ( + "pk_companyId" SERIAL NOT NULL, + ace VARCHAR(1000), + bic VARCHAR(1000), + lei VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT "cx_pk_companyId" PRIMARY KEY ("pk_companyId"), + CONSTRAINT "companyId_xml2db_record_hash" UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref SERIAL NOT NULL, + id VARCHAR(1000), + urn VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail SERIAL NOT NULL, + weight DOUBLE PRECISION, + unit VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail PRIMARY KEY (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_m_5868736 ( + pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b SERIAL NOT NULL, + id VARCHAR(1000), + value INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_which_over_ecb17be PRIMARY KEY (pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature SERIAL NOT NULL, + id VARCHAR(1000), + value VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_stringfeature PRIMARY KEY (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson SERIAL NOT NULL, + name_attr VARCHAR(1000), + name VARCHAR(1000), + address VARCHAR(1000), + city VARCHAR(1000), + fk_zip INTEGER, + country VARCHAR(1000), + "phoneNumber" VARCHAR(8000), + "fk_companyId" INTEGER, + coordinates VARCHAR(1000), + a_very_long_field_type_that_makes_col_name_exceeds_max__223ada0 VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_orderperson PRIMARY KEY (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY("fk_companyId") REFERENCES "companyId" ("pk_companyId") +) + + +CREATE TABLE features ( + pk_features SERIAL NOT NULL, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_features PRIMARY KEY (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9 INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9) REFERENCES intfeature_with_peculiarly_long_suffix_which_overflow_m_5868736 (pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery SERIAL NOT NULL, + from_fk_orderperson INTEGER, + to_fk_orderperson INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_delivery PRIMARY KEY (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product SERIAL NOT NULL, + name VARCHAR(1000), + version VARCHAR(1000), + fk_features INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_product PRIMARY KEY (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item SERIAL NOT NULL, + fk_product INTEGER, + note VARCHAR(1000), + quantity INTEGER, + price DOUBLE PRECISION, + currency VARCHAR(3), + fk_delivery INTEGER, + fk_catalog_ref INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder SERIAL NOT NULL, + orderid VARCHAR(1000), + processed_at TIMESTAMP WITH TIME ZONE, + fk_orderperson INTEGER, + shipto_fk_orderperson INTEGER, + "fk_carrierId" INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY("fk_carrierId") REFERENCES "carrierId" ("pk_carrierId") +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders SERIAL NOT NULL, + batch_id VARCHAR(1000), + version INTEGER, + input_file_path VARCHAR(256), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_whic_c836 ON features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 (fk_features) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_whic_b883 ON features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 (fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9) + +CREATE INDEX ix_features_stringfeature_fk_features ON features_stringfeature (fk_features) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE INDEX ix_item_detail_fk_item ON item_detail (fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE INDEX ix_shiporder_item_fk_shiporder ON shiporder_item (fk_shiporder) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_shiporder ON shiporder_detail_detail_1 (fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_orders ON orders_shiporder (fk_orders) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_erd_version3.md b/tests/sample_models/orders/orders_erd_version3.md new file mode 100644 index 0000000..87502de --- /dev/null +++ b/tests/sample_models/orders/orders_erd_version3.md @@ -0,0 +1,86 @@ +```mermaid +erDiagram + orders ||--o{ shiporder : "shiporder*" + orders { + string batch_id + int version + } + shiporder ||--|| orderperson : "orderperson" + shiporder ||--o| orderperson : "shipto" + shiporder ||--o| carrierId : "carrierId" + shiporder ||--|{ item : "item*" + shiporder ||--o{ detail_1 : "detail*" + shiporder { + string orderid + dateTime processed_at + } + item ||--|| product : "product" + item ||--o| delivery : "delivery" + item ||--o| catalog_ref : "catalog_ref" + item ||--o{ detail : "detail*" + item { + string note + integer quantity + decimal price + string currency + } + product ||--o| features : "features" + product { + string name + string version + } + delivery ||--o| orderperson : "from" + delivery ||--o| orderperson : "to" + delivery { + } + features ||--o{ intfeature_with_peculiarly_long_suffix_which_overflow_max_length : "intfeature_with_peculiarly_long_suffix_which_overflow_max_length*" + features ||--o{ stringfeature : "stringfeature*" + features { + } + orderperson ||--|| zip : "zip" + orderperson ||--o| companyId : "companyId" + orderperson { + string name_attr + string name + string address + string city + string country + string-N phoneNumber + string coordinates + string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length + } + stringfeature { + string id + string value + } + intfeature_with_peculiarly_long_suffix_which_overflow_max_length { + string id + integer value + } + detail { + decimal weight + string unit + } + catalog_ref { + string id + anyURI urn + } + companyId { + string ace + string bic + string lei + } + zip { + string codingSystem + string state + string value + } + detail_1 { + string reference + string carrier + } + carrierId { + string vat + string registry + } +``` \ No newline at end of file diff --git a/tests/sample_models/orders/orders_source_tree_version3.txt b/tests/sample_models/orders/orders_source_tree_version3.txt new file mode 100644 index 0000000..2635b02 --- /dev/null +++ b/tests/sample_models/orders/orders_source_tree_version3.txt @@ -0,0 +1,107 @@ +orders: + batch_id[0, 1]: string + version[1, 1]: int + shiporder[0, None]: + orderid[0, 1]: string + processed_at[0, 1]: dateTime + orderperson[1, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + item[1, None]: + product[1, 1]: + name[1, 1]: string + version[1, 1]: string + features[0, 1]: + intfeature_with_peculiarly_long_suffix_which_overflow_max_length[0, None]: + id[1, 1]: string + value[1, 1]: integer + stringfeature[0, None]: + id[1, 1]: string + value[1, 1]: string + note[0, 1]: string + comment[0, 1]: string + quantity[1, 1]: integer + price[1, 1]: decimal + currency[1, 1]: string + delivery[0, 1] (choice): + from[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + to[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + detail[0, None]: + weight[1, 1]: decimal + unit[1, 1]: string + catalog_ref[0, 1] (choice): + id[0, 1]: string + urn[0, 1]: anyURI + detail[0, None]: + reference[1, 1]: string + carrier[1, 1]: string + carrierId[0, 1]: + vat[1, 1]: string + registry[1, 1]: string \ No newline at end of file diff --git a/tests/sample_models/orders/orders_target_tree_version3.txt b/tests/sample_models/orders/orders_target_tree_version3.txt new file mode 100644 index 0000000..27fe315 --- /dev/null +++ b/tests/sample_models/orders/orders_target_tree_version3.txt @@ -0,0 +1,102 @@ +orders: + batch_id[0, 1]: string + version[1, 1]: int + shiporder[0, None]: + orderid[0, 1]: string + processed_at[0, 1]: dateTime + orderperson[1, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + item[1, None]: + product[1, 1]: + name[1, 1]: string + version[1, 1]: string + features[0, 1]: + intfeature_with_peculiarly_long_suffix_which_overflow_max_length[0, None]: + id[1, 1]: string + value[1, 1]: integer + stringfeature[0, None]: + id[1, 1]: string + value[1, 1]: string + note[0, 1]: string + quantity[1, 1]: integer + price[1, 1]: decimal + currency[1, 1]: string + delivery[0, 1] (choice): + from[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + to[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + detail[0, None]: + weight[1, 1]: decimal + unit[1, 1]: string + catalog_ref[0, 1] (choice): + id[0, 1]: string + urn[0, 1]: anyURI + detail[0, None]: + reference[1, 1]: string + carrier[1, 1]: string + carrierId[0, 1]: + vat[1, 1]: string + registry[1, 1]: string \ No newline at end of file From 61e93f580bb960c6a22d893bcdb3ddb8a2f530a8 Mon Sep 17 00:00:00 2001 From: martinv13 Date: Tue, 20 Oct 2026 22:41:42 +0200 Subject: [PATCH 2/2] Cover more config options in tests Extend the orders sample model configs with a forced "elevate" on an optional child, a column type override, and choice_transform: True overriding the global transform: false. Add a test running document_tree_hook and document_tree_node_hook during parsing. Co-Authored-By: Claude Opus 5.5 --- tests/sample_models/models.py | 9 ++++- .../orders/orders_ddl_mssql_version2.sql | 39 ++++++++++++------- .../orders/orders_ddl_mssql_version3.sql | 5 +-- .../orders/orders_ddl_mysql_version2.sql | 39 ++++++++++++------- .../orders/orders_ddl_mysql_version3.sql | 5 +-- .../orders/orders_ddl_postgresql_version2.sql | 39 ++++++++++++------- .../orders/orders_ddl_postgresql_version3.sql | 5 +-- .../orders/orders_erd_version2.md | 22 ++++++++--- .../orders/orders_erd_version3.md | 5 +-- .../orders/orders_target_tree_version2.txt | 27 +++++++------ .../orders/orders_target_tree_version3.txt | 20 ++++------ tests/test_conversions.py | 29 ++++++++++++++ 12 files changed, 158 insertions(+), 86 deletions(-) diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index 3461a84..f768a4d 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -74,11 +74,16 @@ def wrapped(): { "config": { "tables": { - "shiporder": {"reuse": False}, + "shiporder": { + "reuse": False, + # shipto is optional with many columns: auto-detection would not elevate it + "fields": {"shipto": {"transform": "elevate"}}, + }, "item": { "fields": { "product": {"transform": False}, "comment": {"transform": "skip"}, + "quantity": {"type": "BigInteger"}, } }, }, @@ -97,6 +102,8 @@ def wrapped(): "item": {"fields": {"comment": {"transform": "skip"}}}, # without elevation, an empty extra element would add a row "orderperson": {"fields": {"extra": {"transform": "skip"}}}, + # overrides the global transform: false + "companyId": {"choice_transform": True}, }, "metadata_columns": [ { diff --git a/tests/sample_models/orders/orders_ddl_mssql_version2.sql b/tests/sample_models/orders/orders_ddl_mssql_version2.sql index 230a6ac..f4d51e7 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL IDENTITY, + reference VARCHAR(1000) NULL, + carrier VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson INTEGER NOT NULL IDENTITY, name_attr VARCHAR(1000) NULL, @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 INTEGER NOT NULL IDENTITY, - reference VARCHAR(1000) NULL, - carrier VARCHAR(1000) NULL, - xml2db_record_hash BINARY(20) NULL, - CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail INTEGER NOT NULL IDENTITY, weight DOUBLE PRECISION NULL, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item INTEGER NOT NULL IDENTITY, fk_product INTEGER NULL, note VARCHAR(1000) NULL, - quantity INTEGER NULL, + quantity BIGINT NULL, price DOUBLE PRECISION NULL, currency CHAR(3) NULL, delivery_from_fk_orderperson INTEGER NULL, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( [orderperson_companyId_value] VARCHAR(1000) NULL, orderperson_coordinates VARCHAR(1000) NULL, orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, - shipto_fk_orderperson INTEGER NULL, + shipto_name_attr VARCHAR(1000) NULL, + shipto_name VARCHAR(1000) NULL, + shipto_address VARCHAR(1000) NULL, + shipto_city VARCHAR(1000) NULL, + [shipto_zip_codingSystem] VARCHAR(1000) NULL, + shipto_zip_state VARCHAR(1000) NULL, + shipto_zip_value VARCHAR(1000) NULL, + shipto_country VARCHAR(1000) NULL, + [shipto_phoneNumber] VARCHAR(8000) NULL, + [shipto_companyId_type] CHAR(3) NULL, + [shipto_companyId_value] VARCHAR(1000) NULL, + shipto_coordinates VARCHAR(1000) NULL, + shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, [carrierId_vat] VARCHAR(1000) NULL, [carrierId_registry] VARCHAR(1000) NULL, CONSTRAINT cx_pk_shiporder PRIMARY KEY CLUSTERED (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_mssql_version3.sql b/tests/sample_models/orders/orders_ddl_mssql_version3.sql index f417f4d..149c64f 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version3.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version3.sql @@ -32,9 +32,8 @@ CREATE TABLE zip ( CREATE TABLE [companyId] ( [pk_companyId] INTEGER NOT NULL IDENTITY, - ace VARCHAR(1000) NULL, - bic VARCHAR(1000) NULL, - lei VARCHAR(1000) NULL, + type CHAR(3) NULL, + value VARCHAR(1000) NULL, xml2db_record_hash BINARY(20) NULL, CONSTRAINT [cx_pk_companyId] PRIMARY KEY CLUSTERED ([pk_companyId]), CONSTRAINT [companyId_xml2db_record_hash] UNIQUE (xml2db_record_hash) diff --git a/tests/sample_models/orders/orders_ddl_mysql_version2.sql b/tests/sample_models/orders/orders_ddl_mysql_version2.sql index e6f9df1..b3cc5cf 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, + reference VARCHAR(255), + carrier VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson INTEGER NOT NULL AUTO_INCREMENT, name_attr VARCHAR(255), @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, - reference VARCHAR(255), - carrier VARCHAR(255), - xml2db_record_hash BINARY(20), - CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail INTEGER NOT NULL AUTO_INCREMENT, weight DOUBLE, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item INTEGER NOT NULL AUTO_INCREMENT, fk_product INTEGER, note VARCHAR(255), - quantity INTEGER, + quantity BIGINT, price DOUBLE, currency VARCHAR(3), delivery_from_fk_orderperson INTEGER, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( `orderperson_companyId_value` VARCHAR(255), orderperson_coordinates VARCHAR(255), orderperson_a_very_long_field_type_that_makes_co_ee3c2ee VARCHAR(255), - shipto_fk_orderperson INTEGER, + shipto_name_attr VARCHAR(255), + shipto_name VARCHAR(255), + shipto_address VARCHAR(255), + shipto_city VARCHAR(255), + `shipto_zip_codingSystem` VARCHAR(255), + shipto_zip_state VARCHAR(255), + shipto_zip_value VARCHAR(255), + shipto_country VARCHAR(255), + `shipto_phoneNumber` VARCHAR(4000), + `shipto_companyId_type` VARCHAR(3), + `shipto_companyId_value` VARCHAR(255), + shipto_coordinates VARCHAR(255), + shipto_a_very_long_field_type_that_makes_col_nam_65b3e2f VARCHAR(255), `carrierId_vat` VARCHAR(255), `carrierId_registry` VARCHAR(255), CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_mysql_version3.sql b/tests/sample_models/orders/orders_ddl_mysql_version3.sql index 0bb1e49..691c4de 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version3.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version3.sql @@ -32,9 +32,8 @@ CREATE TABLE zip ( CREATE TABLE `companyId` ( `pk_companyId` INTEGER NOT NULL AUTO_INCREMENT, - ace VARCHAR(255), - bic VARCHAR(255), - lei VARCHAR(255), + type VARCHAR(3), + value VARCHAR(255), xml2db_record_hash BINARY(20), CONSTRAINT `cx_pk_companyId` PRIMARY KEY (`pk_companyId`), CONSTRAINT `companyId_xml2db_record_hash` UNIQUE (xml2db_record_hash) diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version2.sql b/tests/sample_models/orders/orders_ddl_postgresql_version2.sql index c7812c8..4663c21 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 SERIAL NOT NULL, + reference VARCHAR(1000), + carrier VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson SERIAL NOT NULL, name_attr VARCHAR(1000), @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 SERIAL NOT NULL, - reference VARCHAR(1000), - carrier VARCHAR(1000), - xml2db_record_hash BYTEA, - CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail SERIAL NOT NULL, weight DOUBLE PRECISION, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item SERIAL NOT NULL, fk_product INTEGER, note VARCHAR(1000), - quantity INTEGER, + quantity BIGINT, price DOUBLE PRECISION, currency VARCHAR(3), delivery_from_fk_orderperson INTEGER, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( "orderperson_companyId_value" VARCHAR(1000), orderperson_coordinates VARCHAR(1000), orderperson_a_very_long_field_type_that_makes_col_name__ee3c2ee VARCHAR(1000), - shipto_fk_orderperson INTEGER, + shipto_name_attr VARCHAR(1000), + shipto_name VARCHAR(1000), + shipto_address VARCHAR(1000), + shipto_city VARCHAR(1000), + "shipto_zip_codingSystem" VARCHAR(1000), + shipto_zip_state VARCHAR(1000), + shipto_zip_value VARCHAR(1000), + shipto_country VARCHAR(1000), + "shipto_phoneNumber" VARCHAR(8000), + "shipto_companyId_type" VARCHAR(3), + "shipto_companyId_value" VARCHAR(1000), + shipto_coordinates VARCHAR(1000), + shipto_a_very_long_field_type_that_makes_col_name_excee_65b3e2f VARCHAR(1000), "carrierId_vat" VARCHAR(1000), "carrierId_registry" VARCHAR(1000), CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version3.sql b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql index 7da1b61..3d75ca1 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version3.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql @@ -32,9 +32,8 @@ CREATE TABLE zip ( CREATE TABLE "companyId" ( "pk_companyId" SERIAL NOT NULL, - ace VARCHAR(1000), - bic VARCHAR(1000), - lei VARCHAR(1000), + type VARCHAR(3), + value VARCHAR(1000), xml2db_record_hash BYTEA, CONSTRAINT "cx_pk_companyId" PRIMARY KEY ("pk_companyId"), CONSTRAINT "companyId_xml2db_record_hash" UNIQUE (xml2db_record_hash) diff --git a/tests/sample_models/orders/orders_erd_version2.md b/tests/sample_models/orders/orders_erd_version2.md index c3a761c..688bdf4 100644 --- a/tests/sample_models/orders/orders_erd_version2.md +++ b/tests/sample_models/orders/orders_erd_version2.md @@ -1,6 +1,5 @@ ```mermaid erDiagram - shiporder ||--o| orderperson : "shipto" shiporder ||--|{ item : "item*" shiporder ||--o{ detail_1 : "detail*" shiporder { @@ -19,6 +18,19 @@ erDiagram string orderperson_companyId_value string orderperson_coordinates string orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length + string shipto_name_attr + string shipto_name + string shipto_address + string shipto_city + string shipto_zip_codingSystem + string shipto_zip_state + string shipto_zip_value + string shipto_country + string-N shipto_phoneNumber + string shipto_companyId_type + string shipto_companyId_value + string shipto_coordinates + string shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length string carrierId_vat string carrierId_registry } @@ -52,10 +64,6 @@ erDiagram decimal weight string unit } - detail_1 { - string reference - string carrier - } orderperson { string name_attr string name @@ -71,6 +79,10 @@ erDiagram string coordinates string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length } + detail_1 { + string reference + string carrier + } orders ||--o{ shiporder : "shiporder" orders { string batch_id diff --git a/tests/sample_models/orders/orders_erd_version3.md b/tests/sample_models/orders/orders_erd_version3.md index 87502de..22cb640 100644 --- a/tests/sample_models/orders/orders_erd_version3.md +++ b/tests/sample_models/orders/orders_erd_version3.md @@ -66,9 +66,8 @@ erDiagram anyURI urn } companyId { - string ace - string bic - string lei + string type + string value } zip { string codingSystem diff --git a/tests/sample_models/orders/orders_target_tree_version2.txt b/tests/sample_models/orders/orders_target_tree_version2.txt index 80b555d..6209d74 100644 --- a/tests/sample_models/orders/orders_target_tree_version2.txt +++ b/tests/sample_models/orders/orders_target_tree_version2.txt @@ -17,20 +17,19 @@ orders: orderperson_companyId_value[0, 1]: string orderperson_coordinates[0, 1]: string orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string - shipto[0, 1]: - name_attr[0, 1]: string - name[1, 1]: string - address[1, 1]: string - city[1, 1]: string - zip_codingSystem[0, 1]: string - zip_state[0, 1]: string - zip_value[0, 1]: string - country[1, 1]: string - phoneNumber[0, None]: string - companyId_type[0, 1]: string - companyId_value[0, 1]: string - coordinates[0, 1]: string - a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto_name_attr[0, 1]: string + shipto_name[0, 1]: string + shipto_address[0, 1]: string + shipto_city[0, 1]: string + shipto_zip_codingSystem[0, 1]: string + shipto_zip_state[0, 1]: string + shipto_zip_value[0, 1]: string + shipto_country[0, 1]: string + shipto_phoneNumber[0, None]: string + shipto_companyId_type[0, 1]: string + shipto_companyId_value[0, 1]: string + shipto_coordinates[0, 1]: string + shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string item[1, None]: product[1, 1]: name[1, 1]: string diff --git a/tests/sample_models/orders/orders_target_tree_version3.txt b/tests/sample_models/orders/orders_target_tree_version3.txt index 27fe315..f7b0744 100644 --- a/tests/sample_models/orders/orders_target_tree_version3.txt +++ b/tests/sample_models/orders/orders_target_tree_version3.txt @@ -16,9 +16,8 @@ orders: country[1, 1]: string phoneNumber[0, None]: string companyId[0, 1] (choice): - ace[0, 1]: string - bic[0, 1]: string - lei[0, 1]: string + type[1, 1]: string + value[1, 1]: string coordinates[0, 1]: string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string shipto[0, 1]: @@ -33,9 +32,8 @@ orders: country[1, 1]: string phoneNumber[0, None]: string companyId[0, 1] (choice): - ace[0, 1]: string - bic[0, 1]: string - lei[0, 1]: string + type[1, 1]: string + value[1, 1]: string coordinates[0, 1]: string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string item[1, None]: @@ -66,9 +64,8 @@ orders: country[1, 1]: string phoneNumber[0, None]: string companyId[0, 1] (choice): - ace[0, 1]: string - bic[0, 1]: string - lei[0, 1]: string + type[1, 1]: string + value[1, 1]: string coordinates[0, 1]: string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string to[0, 1]: @@ -83,9 +80,8 @@ orders: country[1, 1]: string phoneNumber[0, None]: string companyId[0, 1] (choice): - ace[0, 1]: string - bic[0, 1]: string - lei[0, 1]: string + type[1, 1]: string + value[1, 1]: string coordinates[0, 1]: string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string detail[0, None]: diff --git a/tests/test_conversions.py b/tests/test_conversions.py index b880b34..8d95fc2 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -145,6 +145,35 @@ def test_field_rename(): assert "orderid" in records[0] +def test_document_tree_hooks(): + """Test that document_tree_hook and document_tree_node_hook can modify parsed data""" + + def node_hook(node): + node_type, content, node_hash = node + if node_type == "itemtype": + content["currency"] = ["USD"] + return node + + def tree_hook(tree): + tree[1]["version"] = [3] + return tree + + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "document_tree_hook": tree_hook, + "document_tree_node_hook": node_hook, + }, + ) + for iterparse in (True, False): + doc = model.parse_xml( + str(os.path.join(models_path, "orders", "xml", "order1.xml")), + iterparse=iterparse, + ) + assert doc.data["orderstype"]["records"][0]["version"] == 3 + assert {r["currency"] for r in doc.data["itemtype"]["records"]} == {"USD"} + + def test_field_skip_relation(): """Test that transform='skip' on a relation removes its generated columns and prunes the child table""" model = DataModel(