diff --git a/docs/advanced_guidance/package_documentation/auditing.md b/docs/advanced_guidance/package_documentation/auditing.md index a043e9ae..78230195 100644 --- a/docs/advanced_guidance/package_documentation/auditing.md +++ b/docs/advanced_guidance/package_documentation/auditing.md @@ -11,3 +11,4 @@ - get_all_business_rule_submissions - get_all_error_report_submissions - get_current_processing_info + show_root_toc_entry: false diff --git a/docs/advanced_guidance/package_documentation/business_rules/index.md b/docs/advanced_guidance/package_documentation/business_rules/index.md new file mode 100644 index 00000000..0b1f3aea --- /dev/null +++ b/docs/advanced_guidance/package_documentation/business_rules/index.md @@ -0,0 +1,31 @@ +--- +title: Business Rules Documentation +--- + +
+ +- :material-language-python:{ .lg .middle } __Operations__ + + --- + + [:octicons-arrow-right-24: Read about the `Operation` objects here](operations.md) + +- :material-language-python:{ .lg .middle } __Reference Data Loaders__ + + --- + + [:octicons-arrow-right-24: Read about the `Business Rules` objects here](refdata_loaders.md) + +- :material-language-python:{ .lg .middle } __Reference Data Types__ + + --- + + [:octicons-arrow-right-24: Read about the `Data Contract` objects here](reference_data_types.md) + +- :material-language-python:{ .lg .middle } __User Defined Functions (UDFs)__ + + --- + + [:octicons-arrow-right-24: Read about the `UDFs` objects here](udfs.md) + +
\ No newline at end of file diff --git a/docs/advanced_guidance/package_documentation/operations.md b/docs/advanced_guidance/package_documentation/business_rules/operations.md similarity index 100% rename from docs/advanced_guidance/package_documentation/operations.md rename to docs/advanced_guidance/package_documentation/business_rules/operations.md diff --git a/docs/advanced_guidance/package_documentation/refdata_loaders.md b/docs/advanced_guidance/package_documentation/business_rules/refdata_loaders.md similarity index 100% rename from docs/advanced_guidance/package_documentation/refdata_loaders.md rename to docs/advanced_guidance/package_documentation/business_rules/refdata_loaders.md diff --git a/docs/advanced_guidance/package_documentation/refence_data_types.md b/docs/advanced_guidance/package_documentation/business_rules/reference_data_types.md similarity index 100% rename from docs/advanced_guidance/package_documentation/refence_data_types.md rename to docs/advanced_guidance/package_documentation/business_rules/reference_data_types.md diff --git a/docs/advanced_guidance/package_documentation/business_rules/udfs.md b/docs/advanced_guidance/package_documentation/business_rules/udfs.md new file mode 100644 index 00000000..6a994a27 --- /dev/null +++ b/docs/advanced_guidance/package_documentation/business_rules/udfs.md @@ -0,0 +1,10 @@ + +!!! warning + + Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets. + +::: dve.core_engine.functions.implementations + handler: python + options: + show_root_heading: true + heading_level: 1 diff --git a/docs/advanced_guidance/package_documentation/domain_types.md b/docs/advanced_guidance/package_documentation/data_contract/domain_types.md similarity index 100% rename from docs/advanced_guidance/package_documentation/domain_types.md rename to docs/advanced_guidance/package_documentation/data_contract/domain_types.md diff --git a/docs/advanced_guidance/package_documentation/data_contract/index.md b/docs/advanced_guidance/package_documentation/data_contract/index.md new file mode 100644 index 00000000..a801650d --- /dev/null +++ b/docs/advanced_guidance/package_documentation/data_contract/index.md @@ -0,0 +1,20 @@ +--- +title: Data Contract Documentation +--- + +
+ +- :material-language-python:{ .lg .middle } __Readers__ + + --- + + [:octicons-arrow-right-24: Read about the `Readers` objects here](readers.md) + +- :material-language-python:{ .lg .middle } __Domain Types__ + + --- + + [:octicons-arrow-right-24: Read about the `Domain Types` objects here](domain_types.md) + + +
\ No newline at end of file diff --git a/docs/advanced_guidance/package_documentation/readers.md b/docs/advanced_guidance/package_documentation/data_contract/readers.md similarity index 100% rename from docs/advanced_guidance/package_documentation/readers.md rename to docs/advanced_guidance/package_documentation/data_contract/readers.md diff --git a/docs/advanced_guidance/package_documentation/entity_hierarchy.md b/docs/advanced_guidance/package_documentation/entity_hierarchy.md index 4080dcbe..b4f3ba48 100644 --- a/docs/advanced_guidance/package_documentation/entity_hierarchy.md +++ b/docs/advanced_guidance/package_documentation/entity_hierarchy.md @@ -2,4 +2,4 @@ handler: python options: show_root_heading: true - heading_level: 2 + heading_level: 1 diff --git a/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md new file mode 100644 index 00000000..cfc279a6 --- /dev/null +++ b/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md @@ -0,0 +1,9 @@ + +::: dve.core_engine.message + handler: python + options: + heading_level: 2 + members: + - FeedbackMessage + - UserMessage + show_root_toc_entry: false diff --git a/docs/advanced_guidance/package_documentation/feedback/index.md b/docs/advanced_guidance/package_documentation/feedback/index.md new file mode 100644 index 00000000..e0d810a2 --- /dev/null +++ b/docs/advanced_guidance/package_documentation/feedback/index.md @@ -0,0 +1,13 @@ +--- +title: User Feedback Documentation +--- + +
+ +- :material-language-python:{ .lg .middle } __Feedback Messages__ + + --- + + [:octicons-arrow-right-24: Read about the `Operation` objects here](feedback_messages.md) + +
\ No newline at end of file diff --git a/docs/advanced_guidance/package_documentation/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback_messages.md deleted file mode 100644 index ede7aad8..00000000 --- a/docs/advanced_guidance/package_documentation/feedback_messages.md +++ /dev/null @@ -1,12 +0,0 @@ - -::: dve.core_engine.message.FeedbackMessage - handler: python - options: - show_root_heading: true - heading_level: 2 - -::: dve.core_engine.message.UserMessage - handler: python - options: - show_root_heading: true - heading_level: 2 diff --git a/docs/advanced_guidance/package_documentation/index.md b/docs/advanced_guidance/package_documentation/index.md index c2e76c50..3c02ad0e 100644 --- a/docs/advanced_guidance/package_documentation/index.md +++ b/docs/advanced_guidance/package_documentation/index.md @@ -4,16 +4,46 @@ title: Package Documentation
-- :material-language-python:{ .lg .middle } __Pipeline__ +- :material-database-plus:{ .lg .middle } __Auditing__ --- - [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md) + [:octicons-arrow-right-24: Read about the `Auditing` objects here](auditing.md) + +- :material-clipboard-check:{ .lg .middle } __Business Rules__ + + --- + + [:octicons-arrow-right-24: Read about the `Business Rules` objects here](./business_rules/index.md) + +- :material-file-document-check:{ .lg .middle } __Data Contract__ + + --- + + [:octicons-arrow-right-24: Read about the `Data Contract` objects here](./data_contract/index.md) + +- :material-relation-one-to-many:{ .lg .middle } __Entity Relationships__ + + --- + + [:octicons-arrow-right-24: Read about the `Entity Reliationships` objects here](entity_hierarchy.md) + +- :material-message-alert:{ .lg .middle } __Submission Feedback__ + + --- + + [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](./feedback/index.md) -- :material-set-all:{ .lg .middle } __Reference Data Loader__ +- :material-package-variant-plus:{ .lg .middle } __Additional Objects__ --- - [:octicons-arrow-right-24: Read about the `Reference Data Loader` objects here](refdata_loaders.md) + [:octicons-arrow-right-24: Read about the additional objects here](./other/index.md) + +- :material-pipe:{ .lg .middle } __Pipeline__ + + --- + + [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
\ No newline at end of file diff --git a/docs/advanced_guidance/package_documentation/other/index.md b/docs/advanced_guidance/package_documentation/other/index.md new file mode 100644 index 00000000..bfb3a4e2 --- /dev/null +++ b/docs/advanced_guidance/package_documentation/other/index.md @@ -0,0 +1,13 @@ +--- +title: Additional Objects Documentation +--- + +
+ +- :material-code-braces-box:{ .lg .middle } __Models__ + + --- + + [:octicons-arrow-right-24: Read about the Model objects here](./models.md) + +
diff --git a/docs/advanced_guidance/package_documentation/models.md b/docs/advanced_guidance/package_documentation/other/models.md similarity index 51% rename from docs/advanced_guidance/package_documentation/models.md rename to docs/advanced_guidance/package_documentation/other/models.md index 42e94202..0fc02050 100644 --- a/docs/advanced_guidance/package_documentation/models.md +++ b/docs/advanced_guidance/package_documentation/other/models.md @@ -2,5 +2,7 @@ ::: dve.core_engine.models handler: python options: - show_root_heading: true heading_level: 2 + members: + - SubmissionInfo + show_root_toc_entry: false diff --git a/docs/user_guidance/business_rules.md b/docs/user_guidance/business_rules.md index 0f4093f5..9ea21a12 100644 --- a/docs/user_guidance/business_rules.md +++ b/docs/user_guidance/business_rules.md @@ -209,7 +209,27 @@ The difference between modifiying the existing entity and adding a new one is si ### Operations -For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/operations.md). +For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/business_rules/operations.md). + +### User Defined Functions (UDFs) + +!!! warning + + Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets. + + +The DVE currently has a number of UDFs available to be called during a rule check. For instance, if you wanted to use the [NHS Number](https://www.datadictionary.nhs.uk/attributes/nhs_number.html#ariaid-title2) mod 11 check then you can simply call the udf during the expression definition - such as: + +```json title="example.dischema.json" + { + "entity": "example", + "name": "check NHS number passes mod 11 check", + "expression": "nhsno_mod11_check()" + ... + } +``` + +A full list of UDF functions available can be found [here](../advanced_guidance/package_documentation/business_rules/udfs.md). ## Post Rule @@ -358,7 +378,7 @@ If your Business Rules are reliant on reference data, then you can add the `"ref - Refdata entities are also immutable. So, if you need to modify them in any way, you will always need to create a new entity from it -For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/refence_data_types.md). +For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/business_rules/reference_data_types.md). ## Complex Rules diff --git a/docs/user_guidance/data_contract.md b/docs/user_guidance/data_contract.md index 13e72efa..937fed10 100644 --- a/docs/user_guidance/data_contract.md +++ b/docs/user_guidance/data_contract.md @@ -278,7 +278,7 @@ As you can see, we can set the "type" for several fields to `MyConstrainedString #### Domain Types -Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/domain_types.md). +Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/data_contract/domain_types.md). ### Complex Types diff --git a/docs/user_guidance/feedback_messages.md b/docs/user_guidance/feedback_messages.md index ee812886..01128d27 100644 --- a/docs/user_guidance/feedback_messages.md +++ b/docs/user_guidance/feedback_messages.md @@ -28,4 +28,4 @@ If the DVE is crashing and the error message is either unreadable or is crashing ## Feedback Message Models -Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback_messages.md). +Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback/feedback_messages.md). diff --git a/docs/user_guidance/file_transformation.md b/docs/user_guidance/file_transformation.md index 41dc7de4..411b1e0f 100644 --- a/docs/user_guidance/file_transformation.md +++ b/docs/user_guidance/file_transformation.md @@ -155,7 +155,7 @@ The secondary use of the File Transformation stage is the ability to normalise y ``` !!! abstract "" - You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/readers.md). + You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/data_contract/readers.md). ## Supported Formats diff --git a/docs/user_guidance/implementations/duckdb.md b/docs/user_guidance/implementations/duckdb.md index ac75c587..48448e35 100644 --- a/docs/user_guidance/implementations/duckdb.md +++ b/docs/user_guidance/implementations/duckdb.md @@ -36,7 +36,7 @@ Now you have the DuckDB connection object setup, you are ready to setup the requ ## Generating SubmissionInfo objects -Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path: +Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/other/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path: ```py import glob from datetime import date, datetime @@ -112,7 +112,7 @@ DuckDBRefDataLoader.dataset_config_uri = Path("path", "to", "my", "rules").as_po The connection passed into the `DuckDBRefDataLoader` object will then be able to use various DuckDB readers to load data from an existing table on the connection OR loading data from reference data persisted in either `parquet` or `pyarrow` format. -If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/refdata_loaders.md). +If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md). Now we can move onto setting up the DuckDB DVE Pipeline object. @@ -171,4 +171,4 @@ error_reports = dve_pipeline.cluster_pipeline_run() For further details on the objects referenced above, you can use the following links to read more about the objects: - [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md) -- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md) +- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md) diff --git a/docs/user_guidance/implementations/spark.md b/docs/user_guidance/implementations/spark.md index 6bbdb300..866460e0 100644 --- a/docs/user_guidance/implementations/spark.md +++ b/docs/user_guidance/implementations/spark.md @@ -187,4 +187,4 @@ error_reports = dve_pipeline.cluster_pipeline_run() For further details on the objects referenced above, you can use the following links to read more about the objects: - [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md) -- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md) +- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md) diff --git a/src/dve/core_engine/functions/implementations.py b/src/dve/core_engine/functions/implementations.py index bae6d98b..9d5928d2 100644 --- a/src/dve/core_engine/functions/implementations.py +++ b/src/dve/core_engine/functions/implementations.py @@ -42,116 +42,6 @@ def date_in_current_financial_year(test_date: datetime.date) -> bool: return test_date >= current_fiscal_year_start -def is_valid_ods_code(check_ods_code: str) -> bool: - """A test ODS code lookup, contains some actual valid codes""" - if (not check_ods_code) or (not isinstance(check_ods_code, str)): - return False - valid_ods_codes = [ - "EE142976", - "EE144430", - "EE143473", - "EE148112", - "EE142863", - "EE147862", - "EE142472", - "EE141208", - "EE143149", - "EE140862", - "EE140319", - "EE144899", - "EE144475", - "EE141850", - "EE147934", - "EE141068", - "EE143825", - "EE147805", - "EE143489", - "EE146813", - "EE145703", - "EE148295", - "EE140156", - "EE145502", - "EE148396", - "EE144126", - "EE145590", - "EE141566", - "EE142081", - "EE143640", - "EE144911", - "EE145935", - "EE145279", - "EE143156", - "EE146556", - "EE140781", - "EE144734", - "EE144841", - "EE140419", - "EE140040", - "EE147342", - "EE143330", - "EE140926", - "EE140926", - "EE146438", - "EE142137", - "EE143856", - "EE141067", - "EE148534", - "EE141310", - "EE146899", - "EE146996", - "EE147487", - "EE148447", - "EE144311", - "EE142147", - "EE147605", - "EE142117", - "EE144087", - "EE147326", - "EE147614", - "EE143703", - "EE146135", - "EE140782", - "EE143603", - "EE143554", - "EE146659", - "EE140321", - "EE141185", - "EE147648", - "EE144527", - "EE142680", - "EE141620", - "EE145274", - "EE146251", - "EE148209", - "EE142574", - "EE148162", - "EE143118", - "EE142977", - "EE147798", - "EE147902", - "EE145780", - "EE146992", - "EE142916", - "EE144777", - "EE146935", - "EE145586", - "EE144570", - "EE147122", - "EE140874", - "EE141338", - "EE143244", - ] - return check_ods_code in valid_ods_codes - - -def is_valid_national_org(check_org_code: str) -> bool: - """Simple test org code lookup""" - if (not check_org_code) or (not isinstance(check_org_code, str)): - return False - valid_org_codes = ["ORG01", "ORG02"] - return check_org_code in valid_org_codes - - def check_correct_numeric_signage( val: Union[float, int, Decimal], expected_sign: str = "+/-" ) -> Optional[bool]: diff --git a/zensical.toml b/zensical.toml index 0141b6df..fc47343e 100644 --- a/zensical.toml +++ b/zensical.toml @@ -44,28 +44,33 @@ nav = [ "advanced_guidance/index.md", {"DVE Package Documentation" = [ "advanced_guidance/package_documentation/index.md", - {"Pipeline" = "advanced_guidance/package_documentation/pipeline.md"}, {"Auditing" = "advanced_guidance/package_documentation/auditing.md"}, - {"Data Contract" = [ - {"Readers" = "advanced_guidance/package_documentation/readers.md"}, - {"Domain Types" = "advanced_guidance/package_documentation/domain_types.md"}, - ]}, {"Business Rules" = [ + "advanced_guidance/package_documentation/business_rules/index.md", {"Rules" = [ - {"Operations" = "advanced_guidance/package_documentation/operations.md"}, + {"Operations" = "advanced_guidance/package_documentation/business_rules/operations.md"}, ]}, {"Refdata" = [ - {"Refdata Types" = "advanced_guidance/package_documentation/refence_data_types.md"}, - {"Refdata Loaders" = "advanced_guidance/package_documentation/refdata_loaders.md"}, + {"Refdata Types" = "advanced_guidance/package_documentation/business_rules/reference_data_types.md"}, + {"Refdata Loaders" = "advanced_guidance/package_documentation/business_rules/refdata_loaders.md"}, ]}, - {"Entity Relationships" = [ - {"Entity Hierarchy" = "advanced_guidance/package_documentation/entity_hierarchy.md"} - ]} + {"User Defined Functions (UDFs)" = "advanced_guidance/package_documentation/business_rules/udfs.md"}, ]}, + {"Data Contract" = [ + "advanced_guidance/package_documentation/data_contract/index.md", + {"Readers" = "advanced_guidance/package_documentation/data_contract/readers.md"}, + {"Domain Types" = "advanced_guidance/package_documentation/data_contract/domain_types.md"}, + ]}, + {"Entity Relationships" = "advanced_guidance/package_documentation/entity_hierarchy.md"}, {"Feedback" = [ - {"Feedback Messages" = "advanced_guidance/package_documentation/feedback_messages.md"}, + "advanced_guidance/package_documentation/feedback/index.md", + {"Feedback Messages" = "advanced_guidance/package_documentation/feedback/feedback_messages.md"}, + ]}, + {"Other" = [ + "advanced_guidance/package_documentation/other/index.md", + {"Core Models" = "advanced_guidance/package_documentation/other/models.md"}, ]}, - {"Models" = "advanced_guidance/package_documentation/models.md"}, + {"Pipeline" = "advanced_guidance/package_documentation/pipeline.md"}, ]}, {"DVE Developer Guidance" = [ {"Implementing a new backend" = "advanced_guidance/new_backend.md"},