georgeRobertson · Aug 25, 2026 · Aug 27, 2026 · Aug 27, 2026 · Aug 27, 2026 · Sep 11, 2026
diff --git a/.mise.toml b/.mise.toml
@@ -1,4 +1,4 @@
 [tools]
 python="3.12"
 poetry="2.4.1"
-java="liberica-1.8.0"
+java="zulu-17.60.17"
diff --git a/.tool-versions b/.tool-versions
@@ -1,3 +1,3 @@
 python 3.12.12
 poetry 2.4.1
-java liberica-1.8.0
+java zulu-17.60.17
diff --git a/README.md b/README.md
@@ -17,6 +17,7 @@ __The DVE offers__:
 - Format normalization to Parquet for a unified data representation
 - Data modelling and typecasting
 - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
 - Deriving new fields and entities
 - Clear validation reporting, including summary insights and record-level error messages
 
@@ -41,7 +42,8 @@ Below is a list of features that we would like to implement or have been request
 | Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
 | Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
 | Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
-| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes        |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
 | Uplift Pyspark to 4.0+                                                          | TBA               | No        |
 | Polars upgrade to v1+ | TBA | No |
 | DuckDB upgrade to v1.5+ | TBA | No |

diff --git a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json
@@ -26,8 +26,15 @@
             "items": {
                 "type": "string"
             }
+        },
+        "reader_additional_checks": {
+            "description": "A mapping of additional checks to perform on entities after initial read",
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "reader_additional_checks.json"
         }
-    },
+    }
+},
 
     "required": ["fields"]
 }
diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_additional_checks.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_additional_checks.schema.json
@@ -0,0 +1,22 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:contract/components/reader_additional_checks.schema.json",
+    "title": "reader_additional_checks",
+    "description": "Additional checks to perform on initially read entities",
+    "type": "object",
+    "properties": {
+        "error_code": {
+            "description": "The code to be used for the additional check specified",
+            "type": "string"
+        },
+        "error_message": {
+            "description": "The message to be displayed for the additional check specified.",
+            "type": "string"
+        }
+    },
+    "required": [
+        "error_code",
+        "error_message"
+    ],
+    "additionalProperties": false
+}
diff --git a/...ce/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json b/...ce/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json
@@ -4,6 +4,11 @@
     "title": "Keyword Arguments used across all CSV readers",
     "description": "Arguments present in all CSV readers available.",
     "type": "object",
+    "anyOf": [
+        {
+            "$ref": "global_reader_args.schema.json"
+        }
+    ],
     "properties": {
         "field_check": {
             "type": "string",
@@ -13,14 +18,6 @@
                 "false"
             ]
         },
-        "field_check_error_code": {
-            "type": "string",
-            "description": "Allows you to customise the error code generated in the error report when the field check fails."
-        },
-        "field_check_error_message": {
-            "type": "string",
-            "description": "Allows you to customise the error message generated in the error report when the field check fails."
-        },
         "null_empty_strings": {
             "type": "boolean",
             "description": "Converts empty string values into 'Null' values."

diff --git a/...idance/json_schemas/contract/components/reader_constraints/global_reader_args.schema.json b/...idance/json_schemas/contract/components/reader_constraints/global_reader_args.schema.json
@@ -0,0 +1,19 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:contract/components/reader_constraints/global_reader_args.schema.json",
+    "title": "Keyword Arguments used across all readers",
+    "description": "Arguments present in all readers available.",
+    "type": "object",
+    "properties": {
+        "ft_error_code": {
+            "type": "string",
+            "description": "Error code to raise when a reader is unable to read the contents of a file.",
+            "minLength": 1
+        },
+        "ft_error_message": {
+            "type": "string",
+            "description": "Error message to raise when a reader is unable to read the contents of a file.",
+            "minLength": 1
+        }
+    }
+}
diff --git a/...ce/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json b/...ce/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json
@@ -4,6 +4,11 @@
     "title": "Keyword Arguments used across all XML readers",
     "description": "Arguments present in all XML readers.",
     "type": "object",
+    "anyOf": [
+        {
+            "$ref": "global_reader_args.schema.json"
+        }
+    ],
     "properties": {
         "record_tag": {
             "type": "string",
@@ -40,14 +45,6 @@
             "type": "string",
             "description": "The Relative path (to the dischema) for the XSD document."
         },
-        "xsd_error_code": {
-            "type": "string",
-            "description": "Allows you to customise the error code generated in the error report when the XSD check fails."
-        },
-        "xsd_error_message": {
-            "type": "string",
-            "description": "Allows you to customise the error message generated in the error report when the XSD check fails."
-        },
         "rules_location": {
             "type": "string",
             "description": "Allows you to prefix the directory which contains the XSD document."

diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json
@@ -10,6 +10,9 @@
         },
         "transformations": {
             "$ref": "transformations/transformations.schema.json"
+        },
+        "entity_relationships": {
+            "$ref": "entity_relationships.schema.json"
         }
     },
     "required": [

diff --git a/docs/advanced_guidance/json_schemas/entity_relationships.schema.json b/docs/advanced_guidance/json_schemas/entity_relationships.schema.json
@@ -0,0 +1,50 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:entity_relationships.schema.json",
+    "title": "entity_relationships",
+    "description": "Description of relationships to link normalised entities back to parent entities.",
+    "type": "object",
+    "patternProperties": {
+        "^[A-Za-z0-9_]+.$": {
+            "type": "object",
+            "properties": {
+                "parent_entity": {
+                    "type": "string"
+                },
+                "join_fields": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "is_root_entity": {
+                    "type": "boolean"
+                },
+                "mandatory": {
+                    "type": "boolean"
+                },
+                "missing_parent_id_error_code": {
+                    "type": "string"
+                },
+                "missing_parent_id_error_message": {
+                    "type": "string"
+                },
+                "no_valid_records_error_code": {
+                    "type": "string"
+                },
+                "no_valid_records_error_message": {
+                    "type": "string"
+                },
+                "empty_entity_error_code": {
+                    "type": "string",
+                    "minLength": 1
+                },
+                "empty_entity_error_message": {
+                    "type": "string",
+                    "minLength": 1
+                }
+            },
+            "additionalProperties": false
+        }
+    }
+}
diff --git a/docs/advanced_guidance/package_documentation/auditing.md b/docs/advanced_guidance/package_documentation/auditing.md
@@ -11,3 +11,4 @@
             - get_all_business_rule_submissions
             - get_all_error_report_submissions
             - get_current_processing_info
+        show_root_toc_entry: false
diff --git a/docs/advanced_guidance/package_documentation/business_rules/index.md b/docs/advanced_guidance/package_documentation/business_rules/index.md
@@ -0,0 +1,31 @@
+---
+title: Business Rules Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Operations__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Operation` objects here](operations.md)
+
+-   :material-language-python:{ .lg .middle } __Reference Data Loaders__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Business Rules` objects here](refdata_loaders.md)
+
+-   :material-language-python:{ .lg .middle } __Reference Data Types__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Data Contract` objects here](reference_data_types.md)
+
+-   :material-language-python:{ .lg .middle } __User Defined Functions (UDFs)__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `UDFs` objects here](udfs.md)
+
+</div>
diff --git a/...dance/package_documentation/operations.md → ...ocumentation/business_rules/operations.md b/...dance/package_documentation/operations.md → ...ocumentation/business_rules/operations.md
diff --git a/.../package_documentation/refdata_loaders.md → ...ntation/business_rules/refdata_loaders.md b/.../package_documentation/refdata_loaders.md → ...ntation/business_rules/refdata_loaders.md
diff --git a/...ckage_documentation/refence_data_types.md → ...on/business_rules/reference_data_types.md b/...ckage_documentation/refence_data_types.md → ...on/business_rules/reference_data_types.md
diff --git a/docs/advanced_guidance/package_documentation/business_rules/udfs.md b/docs/advanced_guidance/package_documentation/business_rules/udfs.md
@@ -0,0 +1,10 @@
+
+!!! warning
+
+    Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets.
+
+::: dve.core_engine.functions.implementations
+    handler: python
+    options:
+        show_root_heading: true
+        heading_level: 1
diff --git a/...nce/package_documentation/domain_types.md → ...cumentation/data_contract/domain_types.md b/...nce/package_documentation/domain_types.md → ...cumentation/data_contract/domain_types.md
diff --git a/docs/advanced_guidance/package_documentation/data_contract/index.md b/docs/advanced_guidance/package_documentation/data_contract/index.md
@@ -0,0 +1,20 @@
+---
+title: Data Contract Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Readers__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Readers` objects here](readers.md)
+
+-   :material-language-python:{ .lg .middle } __Domain Types__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Domain Types` objects here](domain_types.md)
+
+
+</div>
diff --git a/...guidance/package_documentation/readers.md → ...ge_documentation/data_contract/readers.md b/...guidance/package_documentation/readers.md → ...ge_documentation/data_contract/readers.md
diff --git a/docs/advanced_guidance/package_documentation/entity_hierarchy.md b/docs/advanced_guidance/package_documentation/entity_hierarchy.md
@@ -0,0 +1,5 @@
+::: dve.core_engine.configuration.v1.hierarchy
+    handler: python
+    options:
+        show_root_heading: true
+        heading_level: 1
diff --git a/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md
@@ -0,0 +1,9 @@
+
+::: dve.core_engine.message
+    handler: python
+    options:
+        heading_level: 2
+        members:
+            - FeedbackMessage
+            - UserMessage
+        show_root_toc_entry: false
diff --git a/docs/advanced_guidance/package_documentation/feedback/index.md b/docs/advanced_guidance/package_documentation/feedback/index.md
@@ -0,0 +1,13 @@
+---
+title: User Feedback Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-language-python:{ .lg .middle } __Feedback Messages__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Operation` objects here](feedback_messages.md)
+
+</div>
diff --git a/docs/advanced_guidance/package_documentation/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback_messages.md
diff --git a/docs/advanced_guidance/package_documentation/index.md b/docs/advanced_guidance/package_documentation/index.md
@@ -4,16 +4,46 @@ title: Package Documentation
 
 <div class="grid cards" markdown>
 
--   :material-language-python:{ .lg .middle } __Pipeline__
+-   :material-database-plus:{ .lg .middle } __Auditing__
 
     ---
 
-    [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
+    [:octicons-arrow-right-24: Read about the `Auditing` objects here](auditing.md)
+
+-   :material-clipboard-check:{ .lg .middle } __Business Rules__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Business Rules` objects here](./business_rules/index.md)
+
+-   :material-file-document-check:{ .lg .middle } __Data Contract__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Data Contract` objects here](./data_contract/index.md)
+
+-   :material-relation-one-to-many:{ .lg .middle } __Entity Relationships__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Entity Reliationships` objects here](entity_hierarchy.md)
+
+-   :material-message-alert:{ .lg .middle } __Submission Feedback__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](./feedback/index.md)
 
--   :material-set-all:{ .lg .middle } __Reference Data Loader__
+-   :material-package-variant-plus:{ .lg .middle } __Additional Objects__
 
     ---
 
-    [:octicons-arrow-right-24: Read about the `Reference Data Loader` objects here](refdata_loaders.md)
+    [:octicons-arrow-right-24: Read about the additional objects here](./other/index.md)
+
+-   :material-pipe:{ .lg .middle } __Pipeline__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
 
 </div>
diff --git a/docs/advanced_guidance/package_documentation/other/index.md b/docs/advanced_guidance/package_documentation/other/index.md
@@ -0,0 +1,13 @@
+---
+title: Additional Objects Documentation
+---
+
+<div class="grid cards" markdown>
+
+-   :material-code-braces-box:{ .lg .middle } __Models__
+
+    ---
+
+    [:octicons-arrow-right-24: Read about the Model objects here](./models.md)
+
+</div>
diff --git a/..._guidance/package_documentation/models.md → ...nce/package_documentation/other/models.md b/..._guidance/package_documentation/models.md → ...nce/package_documentation/other/models.md
@@ -2,5 +2,7 @@
 ::: dve.core_engine.models
     handler: python
     options:
-        show_root_heading: true
         heading_level: 2
+        members:
+            - SubmissionInfo
+        show_root_toc_entry: false