diff --git a/docs/configuring.md b/docs/configuring.md index 8d74027..54c4e51 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -252,8 +252,8 @@ Any field (column or relation) can be excluded from the data model entirely by s The field will be absent from the target table schema and all data for it will be silently dropped during XML parsing. This is useful for PII columns, large binary blobs, or fields that are irrelevant for analysis. -For a skipped relation, the child table is also pruned from the model unless it is referenced by another relation -elsewhere in the schema. +For a skipped relation, the child table and the tables beneath it are also pruned from the model, unless they are +referenced by another relation elsewhere in the schema. Configuration: `"transform": "skip"` diff --git a/pyproject.toml b/pyproject.toml index 94c94b1..48f5a3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "xml2db" -version = "0.14.0" +version = "0.15.0" authors = [ { name="Commission de régulation de l'énergie", email="opensource@cre.fr" }, ] diff --git a/src/xml2db/table/transformed_table.py b/src/xml2db/table/transformed_table.py index 389b8d5..7f99223 100644 --- a/src/xml2db/table/transformed_table.py +++ b/src/xml2db/table/transformed_table.py @@ -307,6 +307,15 @@ def simplify_table(self) -> Tuple[dict, dict]: out_fields.append(("col", field_name, field)) else: + # skip before simplifying the child table, which would keep the tables beneath it + field_config = self.config.get("fields", {}).get(field_name, {}) + if field_config.get("transform") == "skip": + if field_type == "rel1": + del self.relations_1[field_name] + elif field_type == "reln": + del self.relations_n[field_name] + continue + # simplify child table ( types_transforms_child, @@ -317,13 +326,7 @@ def simplify_table(self) -> Tuple[dict, dict]: # check if children can be "elevated" to the upper level transform = self._get_field_transform(field_type, field_name) - if transform == "skip": - if field_type == "rel1": - del self.relations_1[field_name] - elif field_type == "reln": - del self.relations_n[field_name] - # don't add to out_fields, don't set keep_table, don't add to fields_transforms - elif transform is not None: + if transform is not None: if field_type == "rel1": elevated_fields = self._elevate_relation_1( field_name, transform diff --git a/tests/test_conversions.py b/tests/test_conversions.py index 8d95fc2..89aa1f2 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -183,6 +183,7 @@ def test_field_skip_relation(): "item": { "fields": { "delivery": {"transform": "skip"}, + "product": {"transform": "skip"}, } } } @@ -198,6 +199,10 @@ def test_field_skip_relation(): # deliveryType has no other parents, so it should be pruned from the model assert "deliveryType" not in model.tables + # types only reachable through a skipped field are pruned as well + assert "producttype" not in model.tables + assert "intfeaturetype" not in model.tables + assert "stringfeaturetype" not in model.tables # parsing works even for XML files that contain ; delivery data is silently dropped doc = model.parse_xml(str(os.path.join(models_path, "orders", "xml", "order3.xml")))