diff --git a/docs/configuring.md b/docs/configuring.md index cf0ae91..8d74027 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -106,7 +106,7 @@ functions (defaults to `hashlib.sha1`). deduplication of rows is opted out. This allows recording the original order of elements in the source XML, which is not always respected otherwise. It was implemented primarily for round-trip tests, but could serve other purposes. The default value is `False` (disabled). -* `transform` (`false` or `"auto"`): set to `false` to disable all automatic field transformations globally: no joining of multi-value columns, no elevation of child tables, no collapsing of choice groups. The default `"auto"` applies all of these where applicable. Per-field `transform` and per-table `choice_transform` still override the global setting. +* `transform` (`false` or `"auto"`): set to `false` to disable automatic field transformations globally: no elevation of child tables, no collapsing of choice groups. Multi-value columns are still joined, as opting out of `join` is not supported. The default `"auto"` applies all of these where applicable. Per-field `transform` and per-table `choice_transform` still override the global setting. ## Fields configuration diff --git a/src/xml2db/table/transformed_table.py b/src/xml2db/table/transformed_table.py index 16fc806..389b8d5 100644 --- a/src/xml2db/table/transformed_table.py +++ b/src/xml2db/table/transformed_table.py @@ -150,12 +150,13 @@ def _get_field_transform( f"Transform value '{field_transform}' cannot be applied" f" to field '{field_name}' of table '{self.name}'." ) + # "join" is how simple values are read at parsing stage, so global transform:false does not turn it off + if field_type == "col": + if self._can_transform_field("col", field_name, "join"): + return "join" # "auto" at field level always runs auto-detection, overriding global transform:false - if field_transform == "auto" or self.data_model.model_config.get("transform", True): - if field_type == "col": - if self._can_transform_field("col", field_name, "join"): - return "join" - elif field_type == "rel1": + elif field_transform == "auto" or self.data_model.model_config.get("transform", True): + if field_type == "rel1": if ( self.relations_1[field_name].occurs[0] == 1 or len(self.relations_1[field_name].other_table.columns) <= 4 diff --git a/tests/sample_models/models.py b/tests/sample_models/models.py index af1e4c6..f768a4d 100644 --- a/tests/sample_models/models.py +++ b/tests/sample_models/models.py @@ -74,11 +74,16 @@ def wrapped(): { "config": { "tables": { - "shiporder": {"reuse": False}, + "shiporder": { + "reuse": False, + # shipto is optional with many columns: auto-detection would not elevate it + "fields": {"shipto": {"transform": "elevate"}}, + }, "item": { "fields": { "product": {"transform": False}, "comment": {"transform": "skip"}, + "quantity": {"type": "BigInteger"}, } }, }, @@ -90,6 +95,24 @@ def wrapped(): ], } }, + { + "config": { + "transform": False, + "tables": { + "item": {"fields": {"comment": {"transform": "skip"}}}, + # without elevation, an empty extra element would add a row + "orderperson": {"fields": {"extra": {"transform": "skip"}}}, + # overrides the global transform: false + "companyId": {"choice_transform": True}, + }, + "metadata_columns": [ + { + "name": "input_file_path", + "type": sqlalchemy.String(256), + } + ], + } + }, ], }, { diff --git a/tests/sample_models/orders/orders_ddl_mssql_version2.sql b/tests/sample_models/orders/orders_ddl_mssql_version2.sql index 230a6ac..f4d51e7 100644 --- a/tests/sample_models/orders/orders_ddl_mssql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_mssql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL IDENTITY, + reference VARCHAR(1000) NULL, + carrier VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson INTEGER NOT NULL IDENTITY, name_attr VARCHAR(1000) NULL, @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 INTEGER NOT NULL IDENTITY, - reference VARCHAR(1000) NULL, - carrier VARCHAR(1000) NULL, - xml2db_record_hash BINARY(20) NULL, - CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail INTEGER NOT NULL IDENTITY, weight DOUBLE PRECISION NULL, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item INTEGER NOT NULL IDENTITY, fk_product INTEGER NULL, note VARCHAR(1000) NULL, - quantity INTEGER NULL, + quantity BIGINT NULL, price DOUBLE PRECISION NULL, currency CHAR(3) NULL, delivery_from_fk_orderperson INTEGER NULL, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( [orderperson_companyId_value] VARCHAR(1000) NULL, orderperson_coordinates VARCHAR(1000) NULL, orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, - shipto_fk_orderperson INTEGER NULL, + shipto_name_attr VARCHAR(1000) NULL, + shipto_name VARCHAR(1000) NULL, + shipto_address VARCHAR(1000) NULL, + shipto_city VARCHAR(1000) NULL, + [shipto_zip_codingSystem] VARCHAR(1000) NULL, + shipto_zip_state VARCHAR(1000) NULL, + shipto_zip_value VARCHAR(1000) NULL, + shipto_country VARCHAR(1000) NULL, + [shipto_phoneNumber] VARCHAR(8000) NULL, + [shipto_companyId_type] CHAR(3) NULL, + [shipto_companyId_value] VARCHAR(1000) NULL, + shipto_coordinates VARCHAR(1000) NULL, + shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, [carrierId_vat] VARCHAR(1000) NULL, [carrierId_registry] VARCHAR(1000) NULL, CONSTRAINT cx_pk_shiporder PRIMARY KEY CLUSTERED (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_mssql_version3.sql b/tests/sample_models/orders/orders_ddl_mssql_version3.sql new file mode 100644 index 0000000..149c64f --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_mssql_version3.sql @@ -0,0 +1,250 @@ + +CREATE TABLE [carrierId] ( + [pk_carrierId] INTEGER NOT NULL IDENTITY, + vat VARCHAR(1000) NULL, + registry VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT [cx_pk_carrierId] PRIMARY KEY CLUSTERED ([pk_carrierId]), + CONSTRAINT [carrierId_xml2db_record_hash] UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL IDENTITY, + reference VARCHAR(1000) NULL, + carrier VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY CLUSTERED (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip INTEGER NOT NULL IDENTITY, + [codingSystem] VARCHAR(1000) NULL, + state VARCHAR(1000) NULL, + value VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_zip PRIMARY KEY CLUSTERED (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE [companyId] ( + [pk_companyId] INTEGER NOT NULL IDENTITY, + type CHAR(3) NULL, + value VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT [cx_pk_companyId] PRIMARY KEY CLUSTERED ([pk_companyId]), + CONSTRAINT [companyId_xml2db_record_hash] UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + urn VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY CLUSTERED (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail INTEGER NOT NULL IDENTITY, + weight DOUBLE PRECISION NULL, + unit VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_detail PRIMARY KEY CLUSTERED (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_max_length ( + pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + value INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length PRIMARY KEY CLUSTERED (pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature INTEGER NOT NULL IDENTITY, + id VARCHAR(1000) NULL, + value VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_stringfeature PRIMARY KEY CLUSTERED (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson INTEGER NOT NULL IDENTITY, + name_attr VARCHAR(1000) NULL, + name VARCHAR(1000) NULL, + address VARCHAR(1000) NULL, + city VARCHAR(1000) NULL, + fk_zip INTEGER NULL, + country VARCHAR(1000) NULL, + [phoneNumber] VARCHAR(8000) NULL, + [fk_companyId] INTEGER NULL, + coordinates VARCHAR(1000) NULL, + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length VARCHAR(1000) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_orderperson PRIMARY KEY CLUSTERED (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY([fk_companyId]) REFERENCES [companyId] ([pk_companyId]) +) + + +CREATE TABLE features ( + pk_features INTEGER NOT NULL IDENTITY, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_features PRIMARY KEY CLUSTERED (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) REFERENCES intfeature_with_peculiarly_long_suffix_which_overflow_max_length (pk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery INTEGER NOT NULL IDENTITY, + from_fk_orderperson INTEGER NULL, + to_fk_orderperson INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_delivery PRIMARY KEY CLUSTERED (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product INTEGER NOT NULL IDENTITY, + name VARCHAR(1000) NULL, + version VARCHAR(1000) NULL, + fk_features INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_product PRIMARY KEY CLUSTERED (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item INTEGER NOT NULL IDENTITY, + fk_product INTEGER NULL, + note VARCHAR(1000) NULL, + quantity INTEGER NULL, + price DOUBLE PRECISION NULL, + currency CHAR(3) NULL, + fk_delivery INTEGER NULL, + fk_catalog_ref INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_item PRIMARY KEY CLUSTERED (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder INTEGER NOT NULL IDENTITY, + orderid VARCHAR(1000) NULL, + processed_at DATETIMEOFFSET NULL, + fk_orderperson INTEGER NULL, + shipto_fk_orderperson INTEGER NULL, + [fk_carrierId] INTEGER NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_shiporder PRIMARY KEY CLUSTERED (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY([fk_carrierId]) REFERENCES [carrierId] ([pk_carrierId]) +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders INTEGER NOT NULL IDENTITY, + batch_id VARCHAR(1000) NULL, + version INTEGER NULL, + input_file_path VARCHAR(256) NULL, + xml2db_record_hash BINARY(20) NULL, + CONSTRAINT cx_pk_orders PRIMARY KEY CLUSTERED (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length_fk_intfeature_with_peculiarly_long_suffix_w_ef47 ON features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) + +CREATE CLUSTERED INDEX ix_fk_features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length ON features_intfeature_with_peculiarly_long_suffix_which_overflow_max_length (fk_features, fk_intfeature_with_peculiarly_long_suffix_which_overflow_max_length) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE CLUSTERED INDEX ix_fk_features_stringfeature ON features_stringfeature (fk_features, fk_stringfeature) + +CREATE CLUSTERED INDEX ix_fk_item_detail ON item_detail (fk_item, fk_detail) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE CLUSTERED INDEX ix_fk_shiporder_item ON shiporder_item (fk_shiporder, fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE CLUSTERED INDEX ix_fk_shiporder_detail_detail_1 ON shiporder_detail_detail_1 (fk_shiporder, fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE CLUSTERED INDEX ix_fk_orders_shiporder ON orders_shiporder (fk_orders, fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_ddl_mysql_version2.sql b/tests/sample_models/orders/orders_ddl_mysql_version2.sql index e6f9df1..b3cc5cf 100644 --- a/tests/sample_models/orders/orders_ddl_mysql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_mysql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, + reference VARCHAR(255), + carrier VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson INTEGER NOT NULL AUTO_INCREMENT, name_attr VARCHAR(255), @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, - reference VARCHAR(255), - carrier VARCHAR(255), - xml2db_record_hash BINARY(20), - CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail INTEGER NOT NULL AUTO_INCREMENT, weight DOUBLE, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item INTEGER NOT NULL AUTO_INCREMENT, fk_product INTEGER, note VARCHAR(255), - quantity INTEGER, + quantity BIGINT, price DOUBLE, currency VARCHAR(3), delivery_from_fk_orderperson INTEGER, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( `orderperson_companyId_value` VARCHAR(255), orderperson_coordinates VARCHAR(255), orderperson_a_very_long_field_type_that_makes_co_ee3c2ee VARCHAR(255), - shipto_fk_orderperson INTEGER, + shipto_name_attr VARCHAR(255), + shipto_name VARCHAR(255), + shipto_address VARCHAR(255), + shipto_city VARCHAR(255), + `shipto_zip_codingSystem` VARCHAR(255), + shipto_zip_state VARCHAR(255), + shipto_zip_value VARCHAR(255), + shipto_country VARCHAR(255), + `shipto_phoneNumber` VARCHAR(4000), + `shipto_companyId_type` VARCHAR(3), + `shipto_companyId_value` VARCHAR(255), + shipto_coordinates VARCHAR(255), + shipto_a_very_long_field_type_that_makes_col_nam_65b3e2f VARCHAR(255), `carrierId_vat` VARCHAR(255), `carrierId_registry` VARCHAR(255), CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_mysql_version3.sql b/tests/sample_models/orders/orders_ddl_mysql_version3.sql new file mode 100644 index 0000000..691c4de --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_mysql_version3.sql @@ -0,0 +1,250 @@ + +CREATE TABLE `carrierId` ( + `pk_carrierId` INTEGER NOT NULL AUTO_INCREMENT, + vat VARCHAR(255), + registry VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT `cx_pk_carrierId` PRIMARY KEY (`pk_carrierId`), + CONSTRAINT `carrierId_xml2db_record_hash` UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 INTEGER NOT NULL AUTO_INCREMENT, + reference VARCHAR(255), + carrier VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip INTEGER NOT NULL AUTO_INCREMENT, + `codingSystem` VARCHAR(255), + state VARCHAR(255), + value VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_zip PRIMARY KEY (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE `companyId` ( + `pk_companyId` INTEGER NOT NULL AUTO_INCREMENT, + type VARCHAR(3), + value VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT `cx_pk_companyId` PRIMARY KEY (`pk_companyId`), + CONSTRAINT `companyId_xml2db_record_hash` UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + urn VARCHAR(1000), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail INTEGER NOT NULL AUTO_INCREMENT, + weight DOUBLE, + unit VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_detail PRIMARY KEY (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_ove_5868736 ( + pk_intfeature_with_peculiarly_long_suffix_which__85b659b INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + value INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_whi_ecb17be PRIMARY KEY (pk_intfeature_with_peculiarly_long_suffix_which__85b659b), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature INTEGER NOT NULL AUTO_INCREMENT, + id VARCHAR(255), + value VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_stringfeature PRIMARY KEY (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson INTEGER NOT NULL AUTO_INCREMENT, + name_attr VARCHAR(255), + name VARCHAR(255), + address VARCHAR(255), + city VARCHAR(255), + fk_zip INTEGER, + country VARCHAR(255), + `phoneNumber` VARCHAR(4000), + `fk_companyId` INTEGER, + coordinates VARCHAR(255), + a_very_long_field_type_that_makes_col_name_excee_223ada0 VARCHAR(255), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_orderperson PRIMARY KEY (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY(`fk_companyId`) REFERENCES `companyId` (`pk_companyId`) +) + + +CREATE TABLE features ( + pk_features INTEGER NOT NULL AUTO_INCREMENT, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_features PRIMARY KEY (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix__4166a78 ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which__00590e9 INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which__00590e9) REFERENCES intfeature_with_peculiarly_long_suffix_which_ove_5868736 (pk_intfeature_with_peculiarly_long_suffix_which__85b659b) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery INTEGER NOT NULL AUTO_INCREMENT, + from_fk_orderperson INTEGER, + to_fk_orderperson INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_delivery PRIMARY KEY (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product INTEGER NOT NULL AUTO_INCREMENT, + name VARCHAR(255), + version VARCHAR(255), + fk_features INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_product PRIMARY KEY (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item INTEGER NOT NULL AUTO_INCREMENT, + fk_product INTEGER, + note VARCHAR(255), + quantity INTEGER, + price DOUBLE, + currency VARCHAR(3), + fk_delivery INTEGER, + fk_catalog_ref INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder INTEGER NOT NULL AUTO_INCREMENT, + orderid VARCHAR(255), + processed_at DATETIME, + fk_orderperson INTEGER, + shipto_fk_orderperson INTEGER, + `fk_carrierId` INTEGER, + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(`fk_carrierId`) REFERENCES `carrierId` (`pk_carrierId`) +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders INTEGER NOT NULL AUTO_INCREMENT, + batch_id VARCHAR(255), + version INTEGER, + input_file_path VARCHAR(256), + xml2db_record_hash BINARY(20), + CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix__4166_6094 ON features_intfeature_with_peculiarly_long_suffix__4166a78 (fk_features) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix__4166_1469 ON features_intfeature_with_peculiarly_long_suffix__4166a78 (fk_intfeature_with_peculiarly_long_suffix_which__00590e9) + +CREATE INDEX ix_features_stringfeature_fk_features ON features_stringfeature (fk_features) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE INDEX ix_item_detail_fk_item ON item_detail (fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE INDEX ix_shiporder_item_fk_shiporder ON shiporder_item (fk_shiporder) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_shiporder ON shiporder_detail_detail_1 (fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_orders ON orders_shiporder (fk_orders) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version2.sql b/tests/sample_models/orders/orders_ddl_postgresql_version2.sql index c7812c8..4663c21 100644 --- a/tests/sample_models/orders/orders_ddl_postgresql_version2.sql +++ b/tests/sample_models/orders/orders_ddl_postgresql_version2.sql @@ -10,6 +10,16 @@ CREATE TABLE orders ( ) +CREATE TABLE detail_1 ( + pk_detail_1 SERIAL NOT NULL, + reference VARCHAR(1000), + carrier VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + CREATE TABLE orderperson ( pk_orderperson SERIAL NOT NULL, name_attr VARCHAR(1000), @@ -31,16 +41,6 @@ CREATE TABLE orderperson ( ) -CREATE TABLE detail_1 ( - pk_detail_1 SERIAL NOT NULL, - reference VARCHAR(1000), - carrier VARCHAR(1000), - xml2db_record_hash BYTEA, - CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), - CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) -) - - CREATE TABLE detail ( pk_detail SERIAL NOT NULL, weight DOUBLE PRECISION, @@ -101,7 +101,7 @@ CREATE TABLE item ( pk_item SERIAL NOT NULL, fk_product INTEGER, note VARCHAR(1000), - quantity INTEGER, + quantity BIGINT, price DOUBLE PRECISION, currency VARCHAR(3), delivery_from_fk_orderperson INTEGER, @@ -144,12 +144,23 @@ CREATE TABLE shiporder ( "orderperson_companyId_value" VARCHAR(1000), orderperson_coordinates VARCHAR(1000), orderperson_a_very_long_field_type_that_makes_col_name__ee3c2ee VARCHAR(1000), - shipto_fk_orderperson INTEGER, + shipto_name_attr VARCHAR(1000), + shipto_name VARCHAR(1000), + shipto_address VARCHAR(1000), + shipto_city VARCHAR(1000), + "shipto_zip_codingSystem" VARCHAR(1000), + shipto_zip_state VARCHAR(1000), + shipto_zip_value VARCHAR(1000), + shipto_country VARCHAR(1000), + "shipto_phoneNumber" VARCHAR(8000), + "shipto_companyId_type" VARCHAR(3), + "shipto_companyId_value" VARCHAR(1000), + shipto_coordinates VARCHAR(1000), + shipto_a_very_long_field_type_that_makes_col_name_excee_65b3e2f VARCHAR(1000), "carrierId_vat" VARCHAR(1000), "carrierId_registry" VARCHAR(1000), CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), - FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders), - FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson) + FOREIGN KEY(fk_parent_orders) REFERENCES orders (pk_orders) ) diff --git a/tests/sample_models/orders/orders_ddl_postgresql_version3.sql b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql new file mode 100644 index 0000000..3d75ca1 --- /dev/null +++ b/tests/sample_models/orders/orders_ddl_postgresql_version3.sql @@ -0,0 +1,250 @@ + +CREATE TABLE "carrierId" ( + "pk_carrierId" SERIAL NOT NULL, + vat VARCHAR(1000), + registry VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT "cx_pk_carrierId" PRIMARY KEY ("pk_carrierId"), + CONSTRAINT "carrierId_xml2db_record_hash" UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail_1 ( + pk_detail_1 SERIAL NOT NULL, + reference VARCHAR(1000), + carrier VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail_1 PRIMARY KEY (pk_detail_1), + CONSTRAINT detail_1_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE zip ( + pk_zip SERIAL NOT NULL, + "codingSystem" VARCHAR(1000), + state VARCHAR(1000), + value VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_zip PRIMARY KEY (pk_zip), + CONSTRAINT zip_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE "companyId" ( + "pk_companyId" SERIAL NOT NULL, + type VARCHAR(3), + value VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT "cx_pk_companyId" PRIMARY KEY ("pk_companyId"), + CONSTRAINT "companyId_xml2db_record_hash" UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE catalog_ref ( + pk_catalog_ref SERIAL NOT NULL, + id VARCHAR(1000), + urn VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_catalog_ref PRIMARY KEY (pk_catalog_ref), + CONSTRAINT catalog_ref_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE detail ( + pk_detail SERIAL NOT NULL, + weight DOUBLE PRECISION, + unit VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_detail PRIMARY KEY (pk_detail), + CONSTRAINT detail_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE intfeature_with_peculiarly_long_suffix_which_overflow_m_5868736 ( + pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b SERIAL NOT NULL, + id VARCHAR(1000), + value INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_intfeature_with_peculiarly_long_suffix_which_over_ecb17be PRIMARY KEY (pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b), + CONSTRAINT intfeature_with_peculia_0c087_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE stringfeature ( + pk_stringfeature SERIAL NOT NULL, + id VARCHAR(1000), + value VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_stringfeature PRIMARY KEY (pk_stringfeature), + CONSTRAINT stringfeature_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orderperson ( + pk_orderperson SERIAL NOT NULL, + name_attr VARCHAR(1000), + name VARCHAR(1000), + address VARCHAR(1000), + city VARCHAR(1000), + fk_zip INTEGER, + country VARCHAR(1000), + "phoneNumber" VARCHAR(8000), + "fk_companyId" INTEGER, + coordinates VARCHAR(1000), + a_very_long_field_type_that_makes_col_name_exceeds_max__223ada0 VARCHAR(1000), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_orderperson PRIMARY KEY (pk_orderperson), + CONSTRAINT orderperson_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_zip) REFERENCES zip (pk_zip), + FOREIGN KEY("fk_companyId") REFERENCES "companyId" ("pk_companyId") +) + + +CREATE TABLE features ( + pk_features SERIAL NOT NULL, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_features PRIMARY KEY (pk_features), + CONSTRAINT features_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 ( + fk_features INTEGER NOT NULL, + fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9 INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9) REFERENCES intfeature_with_peculiarly_long_suffix_which_overflow_m_5868736 (pk_intfeature_with_peculiarly_long_suffix_which_overflo_85b659b) +) + + +CREATE TABLE features_stringfeature ( + fk_features INTEGER NOT NULL, + fk_stringfeature INTEGER NOT NULL, + FOREIGN KEY(fk_features) REFERENCES features (pk_features), + FOREIGN KEY(fk_stringfeature) REFERENCES stringfeature (pk_stringfeature) +) + + +CREATE TABLE delivery ( + pk_delivery SERIAL NOT NULL, + from_fk_orderperson INTEGER, + to_fk_orderperson INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_delivery PRIMARY KEY (pk_delivery), + CONSTRAINT delivery_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(from_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(to_fk_orderperson) REFERENCES orderperson (pk_orderperson) +) + + +CREATE TABLE product ( + pk_product SERIAL NOT NULL, + name VARCHAR(1000), + version VARCHAR(1000), + fk_features INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_product PRIMARY KEY (pk_product), + CONSTRAINT product_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_features) REFERENCES features (pk_features) +) + + +CREATE TABLE item ( + pk_item SERIAL NOT NULL, + fk_product INTEGER, + note VARCHAR(1000), + quantity INTEGER, + price DOUBLE PRECISION, + currency VARCHAR(3), + fk_delivery INTEGER, + fk_catalog_ref INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_item PRIMARY KEY (pk_item), + CONSTRAINT item_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_product) REFERENCES product (pk_product), + FOREIGN KEY(fk_delivery) REFERENCES delivery (pk_delivery), + FOREIGN KEY(fk_catalog_ref) REFERENCES catalog_ref (pk_catalog_ref) +) + + +CREATE TABLE item_detail ( + fk_item INTEGER NOT NULL, + fk_detail INTEGER NOT NULL, + FOREIGN KEY(fk_item) REFERENCES item (pk_item), + FOREIGN KEY(fk_detail) REFERENCES detail (pk_detail) +) + + +CREATE TABLE shiporder ( + pk_shiporder SERIAL NOT NULL, + orderid VARCHAR(1000), + processed_at TIMESTAMP WITH TIME ZONE, + fk_orderperson INTEGER, + shipto_fk_orderperson INTEGER, + "fk_carrierId" INTEGER, + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_shiporder PRIMARY KEY (pk_shiporder), + CONSTRAINT shiporder_xml2db_record_hash UNIQUE (xml2db_record_hash), + FOREIGN KEY(fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY(shipto_fk_orderperson) REFERENCES orderperson (pk_orderperson), + FOREIGN KEY("fk_carrierId") REFERENCES "carrierId" ("pk_carrierId") +) + + +CREATE TABLE shiporder_item ( + fk_shiporder INTEGER NOT NULL, + fk_item INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_item) REFERENCES item (pk_item) +) + + +CREATE TABLE shiporder_detail_detail_1 ( + fk_shiporder INTEGER NOT NULL, + fk_detail_1 INTEGER NOT NULL, + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder), + FOREIGN KEY(fk_detail_1) REFERENCES detail_1 (pk_detail_1) +) + + +CREATE TABLE orders ( + pk_orders SERIAL NOT NULL, + batch_id VARCHAR(1000), + version INTEGER, + input_file_path VARCHAR(256), + xml2db_record_hash BYTEA, + CONSTRAINT cx_pk_orders PRIMARY KEY (pk_orders), + CONSTRAINT orders_xml2db_record_hash UNIQUE (xml2db_record_hash) +) + + +CREATE TABLE orders_shiporder ( + fk_orders INTEGER NOT NULL, + fk_shiporder INTEGER NOT NULL, + FOREIGN KEY(fk_orders) REFERENCES orders (pk_orders), + FOREIGN KEY(fk_shiporder) REFERENCES shiporder (pk_shiporder) +) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_whic_c836 ON features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 (fk_features) + +CREATE INDEX ix_features_intfeature_with_peculiarly_long_suffix_whic_b883 ON features_intfeature_with_peculiarly_long_suffix_which_o_4166a78 (fk_intfeature_with_peculiarly_long_suffix_which_overflo_00590e9) + +CREATE INDEX ix_features_stringfeature_fk_features ON features_stringfeature (fk_features) + +CREATE INDEX ix_features_stringfeature_fk_stringfeature ON features_stringfeature (fk_stringfeature) + +CREATE INDEX ix_item_detail_fk_detail ON item_detail (fk_detail) + +CREATE INDEX ix_item_detail_fk_item ON item_detail (fk_item) + +CREATE INDEX ix_shiporder_item_fk_item ON shiporder_item (fk_item) + +CREATE INDEX ix_shiporder_item_fk_shiporder ON shiporder_item (fk_shiporder) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_detail_1 ON shiporder_detail_detail_1 (fk_detail_1) + +CREATE INDEX ix_shiporder_detail_detail_1_fk_shiporder ON shiporder_detail_detail_1 (fk_shiporder) + +CREATE INDEX ix_orders_shiporder_fk_orders ON orders_shiporder (fk_orders) + +CREATE INDEX ix_orders_shiporder_fk_shiporder ON orders_shiporder (fk_shiporder) + diff --git a/tests/sample_models/orders/orders_erd_version2.md b/tests/sample_models/orders/orders_erd_version2.md index c3a761c..688bdf4 100644 --- a/tests/sample_models/orders/orders_erd_version2.md +++ b/tests/sample_models/orders/orders_erd_version2.md @@ -1,6 +1,5 @@ ```mermaid erDiagram - shiporder ||--o| orderperson : "shipto" shiporder ||--|{ item : "item*" shiporder ||--o{ detail_1 : "detail*" shiporder { @@ -19,6 +18,19 @@ erDiagram string orderperson_companyId_value string orderperson_coordinates string orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length + string shipto_name_attr + string shipto_name + string shipto_address + string shipto_city + string shipto_zip_codingSystem + string shipto_zip_state + string shipto_zip_value + string shipto_country + string-N shipto_phoneNumber + string shipto_companyId_type + string shipto_companyId_value + string shipto_coordinates + string shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length string carrierId_vat string carrierId_registry } @@ -52,10 +64,6 @@ erDiagram decimal weight string unit } - detail_1 { - string reference - string carrier - } orderperson { string name_attr string name @@ -71,6 +79,10 @@ erDiagram string coordinates string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length } + detail_1 { + string reference + string carrier + } orders ||--o{ shiporder : "shiporder" orders { string batch_id diff --git a/tests/sample_models/orders/orders_erd_version3.md b/tests/sample_models/orders/orders_erd_version3.md new file mode 100644 index 0000000..22cb640 --- /dev/null +++ b/tests/sample_models/orders/orders_erd_version3.md @@ -0,0 +1,85 @@ +```mermaid +erDiagram + orders ||--o{ shiporder : "shiporder*" + orders { + string batch_id + int version + } + shiporder ||--|| orderperson : "orderperson" + shiporder ||--o| orderperson : "shipto" + shiporder ||--o| carrierId : "carrierId" + shiporder ||--|{ item : "item*" + shiporder ||--o{ detail_1 : "detail*" + shiporder { + string orderid + dateTime processed_at + } + item ||--|| product : "product" + item ||--o| delivery : "delivery" + item ||--o| catalog_ref : "catalog_ref" + item ||--o{ detail : "detail*" + item { + string note + integer quantity + decimal price + string currency + } + product ||--o| features : "features" + product { + string name + string version + } + delivery ||--o| orderperson : "from" + delivery ||--o| orderperson : "to" + delivery { + } + features ||--o{ intfeature_with_peculiarly_long_suffix_which_overflow_max_length : "intfeature_with_peculiarly_long_suffix_which_overflow_max_length*" + features ||--o{ stringfeature : "stringfeature*" + features { + } + orderperson ||--|| zip : "zip" + orderperson ||--o| companyId : "companyId" + orderperson { + string name_attr + string name + string address + string city + string country + string-N phoneNumber + string coordinates + string a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length + } + stringfeature { + string id + string value + } + intfeature_with_peculiarly_long_suffix_which_overflow_max_length { + string id + integer value + } + detail { + decimal weight + string unit + } + catalog_ref { + string id + anyURI urn + } + companyId { + string type + string value + } + zip { + string codingSystem + string state + string value + } + detail_1 { + string reference + string carrier + } + carrierId { + string vat + string registry + } +``` \ No newline at end of file diff --git a/tests/sample_models/orders/orders_source_tree_version3.txt b/tests/sample_models/orders/orders_source_tree_version3.txt new file mode 100644 index 0000000..2635b02 --- /dev/null +++ b/tests/sample_models/orders/orders_source_tree_version3.txt @@ -0,0 +1,107 @@ +orders: + batch_id[0, 1]: string + version[1, 1]: int + shiporder[0, None]: + orderid[0, 1]: string + processed_at[0, 1]: dateTime + orderperson[1, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + item[1, None]: + product[1, 1]: + name[1, 1]: string + version[1, 1]: string + features[0, 1]: + intfeature_with_peculiarly_long_suffix_which_overflow_max_length[0, None]: + id[1, 1]: string + value[1, 1]: integer + stringfeature[0, None]: + id[1, 1]: string + value[1, 1]: string + note[0, 1]: string + comment[0, 1]: string + quantity[1, 1]: integer + price[1, 1]: decimal + currency[1, 1]: string + delivery[0, 1] (choice): + from[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + to[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + ace[0, 1]: string + bic[0, 1]: string + lei[0, 1]: string + coordinates[0, 1]: string + extra[0, 1]: + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + detail[0, None]: + weight[1, 1]: decimal + unit[1, 1]: string + catalog_ref[0, 1] (choice): + id[0, 1]: string + urn[0, 1]: anyURI + detail[0, None]: + reference[1, 1]: string + carrier[1, 1]: string + carrierId[0, 1]: + vat[1, 1]: string + registry[1, 1]: string \ No newline at end of file diff --git a/tests/sample_models/orders/orders_target_tree_version2.txt b/tests/sample_models/orders/orders_target_tree_version2.txt index 80b555d..6209d74 100644 --- a/tests/sample_models/orders/orders_target_tree_version2.txt +++ b/tests/sample_models/orders/orders_target_tree_version2.txt @@ -17,20 +17,19 @@ orders: orderperson_companyId_value[0, 1]: string orderperson_coordinates[0, 1]: string orderperson_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string - shipto[0, 1]: - name_attr[0, 1]: string - name[1, 1]: string - address[1, 1]: string - city[1, 1]: string - zip_codingSystem[0, 1]: string - zip_state[0, 1]: string - zip_value[0, 1]: string - country[1, 1]: string - phoneNumber[0, None]: string - companyId_type[0, 1]: string - companyId_value[0, 1]: string - coordinates[0, 1]: string - a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto_name_attr[0, 1]: string + shipto_name[0, 1]: string + shipto_address[0, 1]: string + shipto_city[0, 1]: string + shipto_zip_codingSystem[0, 1]: string + shipto_zip_state[0, 1]: string + shipto_zip_value[0, 1]: string + shipto_country[0, 1]: string + shipto_phoneNumber[0, None]: string + shipto_companyId_type[0, 1]: string + shipto_companyId_value[0, 1]: string + shipto_coordinates[0, 1]: string + shipto_a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string item[1, None]: product[1, 1]: name[1, 1]: string diff --git a/tests/sample_models/orders/orders_target_tree_version3.txt b/tests/sample_models/orders/orders_target_tree_version3.txt new file mode 100644 index 0000000..f7b0744 --- /dev/null +++ b/tests/sample_models/orders/orders_target_tree_version3.txt @@ -0,0 +1,98 @@ +orders: + batch_id[0, 1]: string + version[1, 1]: int + shiporder[0, None]: + orderid[0, 1]: string + processed_at[0, 1]: dateTime + orderperson[1, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + type[1, 1]: string + value[1, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + shipto[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + type[1, 1]: string + value[1, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + item[1, None]: + product[1, 1]: + name[1, 1]: string + version[1, 1]: string + features[0, 1]: + intfeature_with_peculiarly_long_suffix_which_overflow_max_length[0, None]: + id[1, 1]: string + value[1, 1]: integer + stringfeature[0, None]: + id[1, 1]: string + value[1, 1]: string + note[0, 1]: string + quantity[1, 1]: integer + price[1, 1]: decimal + currency[1, 1]: string + delivery[0, 1] (choice): + from[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + type[1, 1]: string + value[1, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + to[0, 1]: + name_attr[0, 1]: string + name[1, 1]: string + address[1, 1]: string + city[1, 1]: string + zip[1, 1]: + codingSystem[0, 1]: string + state[0, 1]: string + value[0, 1]: string + country[1, 1]: string + phoneNumber[0, None]: string + companyId[0, 1] (choice): + type[1, 1]: string + value[1, 1]: string + coordinates[0, 1]: string + a_very_long_field_type_that_makes_col_name_exceeds_max_identifier_length[0, 1]: string + detail[0, None]: + weight[1, 1]: decimal + unit[1, 1]: string + catalog_ref[0, 1] (choice): + id[0, 1]: string + urn[0, 1]: anyURI + detail[0, None]: + reference[1, 1]: string + carrier[1, 1]: string + carrierId[0, 1]: + vat[1, 1]: string + registry[1, 1]: string \ No newline at end of file diff --git a/tests/test_conversions.py b/tests/test_conversions.py index b880b34..8d95fc2 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -145,6 +145,35 @@ def test_field_rename(): assert "orderid" in records[0] +def test_document_tree_hooks(): + """Test that document_tree_hook and document_tree_node_hook can modify parsed data""" + + def node_hook(node): + node_type, content, node_hash = node + if node_type == "itemtype": + content["currency"] = ["USD"] + return node + + def tree_hook(tree): + tree[1]["version"] = [3] + return tree + + model = DataModel( + str(os.path.join(models_path, "orders", "orders.xsd")), + model_config={ + "document_tree_hook": tree_hook, + "document_tree_node_hook": node_hook, + }, + ) + for iterparse in (True, False): + doc = model.parse_xml( + str(os.path.join(models_path, "orders", "xml", "order1.xml")), + iterparse=iterparse, + ) + assert doc.data["orderstype"]["records"][0]["version"] == 3 + assert {r["currency"] for r in doc.data["itemtype"]["records"]} == {"USD"} + + def test_field_skip_relation(): """Test that transform='skip' on a relation removes its generated columns and prunes the child table""" model = DataModel(