From c7cdc0f03147ba29e9403ac309ecbe01c44921aa Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:00:21 +0100 Subject: [PATCH 1/4] fix: adjust regex check for timezone offsets, allowing : to be included --- src/dve/core_engine/backends/utilities.py | 5 ++--- .../test_duckdb/test_duckdb_helpers.py | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py index af2f7573..d8b93d1c 100644 --- a/src/dve/core_engine/backends/utilities.py +++ b/src/dve/core_engine/backends/utilities.py @@ -40,11 +40,11 @@ "H": r"[0-9]{2}", "M": r"[0-9]{2}", "S": r"[0-9]{2}", - "z": r"(\+|\-)?[0-9]+(\.[0-9]*)?", + "z": r"(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?", "Z": r"[A-Z]{0,3}", } -REGEXP_NEED_ESCAPE_CHARS: tuple[str, str, str] = ("+", "-", ".") +REGEXP_NEED_ESCAPE_CHARS: tuple[str, str, str, str] = ("+", "-", ".") """Helper to map python date format to regexp expression. Not exhaustive, but aims to cover all foreseen use cases.""" @@ -54,7 +54,6 @@ def datetime_format_to_regex(format_str: str) -> str: Helper function to convert python datetime formats to regexp string checks for casting purposes. """ - tokens = list(format_str.replace("%", "")) for idx, tkn in enumerate(tokens): diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py index 0e81cf46..6004dcb6 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py @@ -224,8 +224,8 @@ def test_duckdb_rel_to_dictionaries(temp_ddb_conn: DuckDBPyConnection, ("date_test", datetime.date,"TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"date_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"date_test\"), '%Y-%m-%d') ELSE NULL END as DATE)"), ("timestamp_test", datetime.datetime, "TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"timestamp_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"timestamp_test\"), '%Y-%m-%dT%H:%M:%S') ELSE NULL END as TIMESTAMP)"), ("list_int_field", list[int], "TRY_CAST(LIST_TRANSFORM(\"list_int_field\", x -> TRIM(\"x\")) as BIGINT[])"), - ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"), - ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")]) + ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"), + ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")]) def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_statement): assert get_duckdb_cast_statement_from_annotation(field_name, field_type) == cast_statement @@ -270,6 +270,7 @@ def test_relation_is_empty(temp_ddb_conn: DuckDBPyConnection): @pytest.mark.parametrize("date_format, expected_regex", [("%Y-%m-%d", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$"), ("%Y-%m-%dT%H:%M:%S", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"), ("%m/%d/%Y", r"^[0-9]{2}/[0-9]{2}/[0-9]{4}$"), - ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$")]) + ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$")]) + def test_datetime_format_to_regex(date_format, expected_regex): assert datetime_format_to_regex(date_format) == expected_regex \ No newline at end of file From c4ef964b2fbccc1becca9794631b45ff358da5ba Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:06:17 +0100 Subject: [PATCH 2/4] style: resolve linting and static typing issues --- .../backends/implementations/duckdb/rules.py | 18 ++++++------------ src/dve/core_engine/backends/utilities.py | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/dve/core_engine/backends/implementations/duckdb/rules.py b/src/dve/core_engine/backends/implementations/duckdb/rules.py index 2dd6a048..187a880d 100644 --- a/src/dve/core_engine/backends/implementations/duckdb/rules.py +++ b/src/dve/core_engine/backends/implementations/duckdb/rules.py @@ -372,18 +372,12 @@ def join_header(self, entities: DuckDBEntities, *, config: HeaderJoin) -> Messag ), ) - joined_rel = ( - source_rel - .cross(target_rel) - .select( - StarExpression( - exclude=[f"{config.target_name}.{c}" for c in target_rel.columns] - ), - FunctionExpression( - "struct_pack", - *[ColumnExpression(f"{config.target_name}.{c}") for c in target_rel.columns], - ).alias(config.header_column_name) - ) + joined_rel = source_rel.cross(target_rel).select( + StarExpression(exclude=[f"{config.target_name}.{c}" for c in target_rel.columns]), + FunctionExpression( + "struct_pack", + *[ColumnExpression(f"{config.target_name}.{c}") for c in target_rel.columns], + ).alias(config.header_column_name), ) entities[config.new_entity_name or config.entity_name] = joined_rel diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py index d8b93d1c..8c9333d6 100644 --- a/src/dve/core_engine/backends/utilities.py +++ b/src/dve/core_engine/backends/utilities.py @@ -44,7 +44,7 @@ "Z": r"[A-Z]{0,3}", } -REGEXP_NEED_ESCAPE_CHARS: tuple[str, str, str, str] = ("+", "-", ".") +REGEXP_NEED_ESCAPE_CHARS: tuple[str, str, str] = ("+", "-", ".") """Helper to map python date format to regexp expression. Not exhaustive, but aims to cover all foreseen use cases.""" From 7b3aa0c71ad6c652e4b8c82d9e3f949be2c11ff2 Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:37:30 +0100 Subject: [PATCH 3/4] test: add new tests for datetime casting --- .../implementations/spark/spark_helpers.py | 2 +- src/dve/core_engine/backends/utilities.py | 2 +- .../test_duckdb/test_duckdb_helpers.py | 28 +++++++++++++++---- .../test_spark/test_spark_helpers.py | 18 +++++++++++- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py index f3ce03f4..5041095b 100644 --- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py +++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py @@ -112,7 +112,7 @@ def __post_init__(self): "%H": "HH", "%M": "mm", "%S": "ss", - "%z": "XX", + "%z": "XXX", "%Z": "z", } """A mapping of python to java datetime component formats. Not exhaustive but will hopefully support diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py index 8c9333d6..7cb625d4 100644 --- a/src/dve/core_engine/backends/utilities.py +++ b/src/dve/core_engine/backends/utilities.py @@ -40,7 +40,7 @@ "H": r"[0-9]{2}", "M": r"[0-9]{2}", "S": r"[0-9]{2}", - "z": r"(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?", + "z": r"[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?", "Z": r"[A-Z]{0,3}", } diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py index 07c46ad4..5ea18b6e 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py @@ -26,7 +26,7 @@ ) from dve.core_engine.backends.utilities import datetime_format_to_regex from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME -from dve.metadata_parser.domain_types import formatteddatetime +from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime @pytest.fixture def casting_test_table(temp_ddb_conn): @@ -225,8 +225,8 @@ def test_duckdb_rel_to_dictionaries(temp_ddb_conn: DuckDBPyConnection, ("date_test", datetime.date,"TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"date_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"date_test\"), '%Y-%m-%d') ELSE NULL END as DATE)"), ("timestamp_test", datetime.datetime, "TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"timestamp_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"timestamp_test\"), '%Y-%m-%dT%H:%M:%S') ELSE NULL END as TIMESTAMP)"), ("list_int_field", list[int], "TRY_CAST(LIST_TRANSFORM(\"list_int_field\", x -> TRIM(\"x\")) as BIGINT[])"), - ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"), - ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")]) + ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"), + ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")]) def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_statement): assert get_duckdb_cast_statement_from_annotation(field_name, field_type) == cast_statement @@ -283,7 +283,25 @@ def test_relation_is_empty(temp_ddb_conn: DuckDBPyConnection): @pytest.mark.parametrize("date_format, expected_regex", [("%Y-%m-%d", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$"), ("%Y-%m-%dT%H:%M:%S", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"), ("%m/%d/%Y", r"^[0-9]{2}/[0-9]{2}/[0-9]{4}$"), - ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(:[0-9]{2})?(\.[0-9]*)?$")]) + ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$")]) def test_datetime_format_to_regex(date_format, expected_regex): - assert datetime_format_to_regex(date_format) == expected_regex \ No newline at end of file + assert datetime_format_to_regex(date_format) == expected_regex + + +@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", datetime.date(2020,6,11)), + ("15/11/2018", "%d/%m/%Y", datetime.date(2018,11,15)), + ("27/14/2027", "%d/%m/%Y", None), + ("2014-05-27", "%d/%m/%y", None), + ("2020-07-15T13:24:54+0100", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 7, 15, 13, 24, 54)), + ("2020-07-15T13:24:54+01:00", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 7, 15, 13, 24, 54))]) +def test_datetime_casting(temp_ddb_conn: DuckDBPyConnection, datetime_str, datetime_format, expected_value): + _, con = temp_ddb_conn + if isinstance(expected_value, datetime.datetime): + type_ = formatteddatetime(date_format=datetime_format) + else: + type_ = conformatteddate(date_format=datetime_format) + cast_statement = get_duckdb_cast_statement_from_annotation("test", type_) + rel = con.sql(f"SELECT '{datetime_str}' as test") + check = rel.project(cast_statement).fetchone()[0] + assert check == expected_value \ No newline at end of file diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py index 34ecb6ca..3d355852 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py @@ -33,7 +33,7 @@ df_is_empty ) from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME -from dve.metadata_parser.domain_types import conformatteddate +from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime from .....fixtures import spark # pylint: disable=unused-import @@ -380,6 +380,22 @@ def test_df_is_empty(spark): df5 = df4.union(spark.createDataFrame([(4, None, 8)], schema=_schema_w_ri)) assert not df_is_empty(df5) + +@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", dt.date(2020,6,11)), + ("15/11/2018", "%d/%m/%Y", dt.date(2018,11,15)), + ("27/14/2027", "%d/%m/%Y", None), + ("2014-05-27", "%d/%m/%y", None), + ("2020-07-15T13:24:54+01:00", "%Y-%m-%dT%H:%M:%S%z", dt.datetime(2020, 7, 15, 13, 24, 54))]) +def test_datetime_casting(spark, datetime_str, datetime_format, expected_value): + + if isinstance(expected_value, dt.datetime): + type_ = formatteddatetime(date_format=datetime_format) + else: + type_ = conformatteddate(date_format=datetime_format) + cast_statement = get_spark_cast_statement_from_annotation("test", type_) + df = spark.createDataFrame([(datetime_str,)], schema=StructType([StructField("test", StringType())])) + check = df.select(cast_statement.alias("test")).first().test + assert check == expected_value From b624dd211f47f5a1a8d599a0bd4bd62c8707a6e4 Mon Sep 17 00:00:00 2001 From: stevenhsd <56357022+stevenhsd@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:52:24 +0100 Subject: [PATCH 4/4] test: fix failing tests --- .../test_implementations/test_duckdb/test_duckdb_helpers.py | 4 ++-- .../test_implementations/test_spark/test_spark_helpers.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py index 5ea18b6e..58dd423a 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py @@ -293,8 +293,8 @@ def test_datetime_format_to_regex(date_format, expected_regex): ("15/11/2018", "%d/%m/%Y", datetime.date(2018,11,15)), ("27/14/2027", "%d/%m/%Y", None), ("2014-05-27", "%d/%m/%y", None), - ("2020-07-15T13:24:54+0100", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 7, 15, 13, 24, 54)), - ("2020-07-15T13:24:54+01:00", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 7, 15, 13, 24, 54))]) + ("2020-03-15T13:24:54+0000", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54)), + ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54))]) def test_datetime_casting(temp_ddb_conn: DuckDBPyConnection, datetime_str, datetime_format, expected_value): _, con = temp_ddb_conn if isinstance(expected_value, datetime.datetime): diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py index 3d355852..9f7c9282 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py @@ -385,7 +385,7 @@ def test_df_is_empty(spark): ("15/11/2018", "%d/%m/%Y", dt.date(2018,11,15)), ("27/14/2027", "%d/%m/%Y", None), ("2014-05-27", "%d/%m/%y", None), - ("2020-07-15T13:24:54+01:00", "%Y-%m-%dT%H:%M:%S%z", dt.datetime(2020, 7, 15, 13, 24, 54))]) + ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", dt.datetime(2020, 3, 15, 13, 24, 54))]) def test_datetime_casting(spark, datetime_str, datetime_format, expected_value): if isinstance(expected_value, dt.datetime):