From e56fec403d913043b0a81425fe7b14db08f87a9a Mon Sep 17 00:00:00 2001 From: mday-io Date: Thu, 10 Sep 2026 14:47:07 -0400 Subject: [PATCH 1/2] feat: resolve tuple and array physical property table references Signed-off-by: mday-io --- docs/integrations/engines/starrocks.md | 14 +++- sqlmesh/core/model/definition.py | 10 ++- tests/core/engine_adapter/test_starrocks.py | 90 +++++++++++++++++++++ 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/docs/integrations/engines/starrocks.md b/docs/integrations/engines/starrocks.md index a5314b87b6..4e517cff45 100644 --- a/docs/integrations/engines/starrocks.md +++ b/docs/integrations/engines/starrocks.md @@ -517,7 +517,7 @@ SELECT user_id, COUNT(*) AS event_count FROM user_events GROUP BY user_id; * `excluded_trigger_tables`: base tables whose data changes should **not** automatically trigger a refresh of this MV. * `excluded_refresh_tables`: base tables that should **not** be scanned when the MV refreshes. -Both properties accept a single table reference or a comma-separated list of table references. +Both properties accept a single table reference, a comma-separated string of table references, or a tuple or array of table references. StarRocks requires the **physical** base table name for these properties, not the logical view name that SQLMesh normally exposes. SQLMesh handles this automatically: when a reference matches a managed SQLMesh model, the logical name is resolved to its physical table name before the `CREATE MATERIALIZED VIEW` statement is issued. References that do not match any managed model are passed through unchanged. @@ -538,7 +538,17 @@ MODEL ( SELECT order_id, SUM(amount) AS total FROM mydb.orders GROUP BY order_id; ``` -A single reference can be written as a bare identifier (`mydb.orders`) or as a quoted string. Multiple references must be provided as a quoted, comma-separated string (`'mydb.orders,mydb.order_items'`). +A single reference can be written as a bare identifier (`mydb.orders`) or as a quoted string. Multiple references can be provided as a quoted, comma-separated string (`'mydb.orders,mydb.order_items'`), a tuple, or an array. Tuple and array elements can be bare identifiers, quoted strings, or a mix of both: + +```sql +physical_properties ( + refresh_scheme = 'ASYNC', + excluded_trigger_tables = (mydb.orders, 'mydb.order_items'), + excluded_refresh_tables = ['mydb.orders', mydb.order_items] +) +``` + +SQLMesh resolves each reference individually and passes the result to StarRocks as a comma-separated string of table names, regardless of the input form. **Other properties:** diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index e8e122dece..a99174ad90 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2834,13 +2834,19 @@ def _resolve_model_refs_to_physical_tables( ) -> exp.Literal: """Resolve managed-model references in a property value to their physical table names. - The value is a single table reference or a comma-separated list of them. Each reference that - matches a managed model (via ``table_mapping``) is swapped for its physical ``db.table`` name; + The value is a single table reference, a comma-separated string, or a tuple/array of + table references. Each reference that matches a managed model (via ``table_mapping``) is + swapped for its physical ``db.table`` name; anything else (e.g. a raw source) is kept as written. Returns a single string literal so the property renders just like a hand-written value. """ if isinstance(value, exp.Literal) and value.is_string: refs = value.this.split(",") + elif isinstance(value, (exp.Tuple, exp.Array)): + refs = [ + ref.this if isinstance(ref, exp.Literal) and ref.is_string else ref.sql(dialect=dialect) + for ref in value.expressions + ] else: refs = [value.sql(dialect=dialect)] diff --git a/tests/core/engine_adapter/test_starrocks.py b/tests/core/engine_adapter/test_starrocks.py index db0b1cc4ae..2a4a611d4f 100644 --- a/tests/core/engine_adapter/test_starrocks.py +++ b/tests/core/engine_adapter/test_starrocks.py @@ -2180,6 +2180,96 @@ def _build_mv_with_excluded_tables( calls = to_sql_calls(adapter) return calls[-1] + @pytest.mark.parametrize("key", ["excluded_trigger_tables", "excluded_refresh_tables"]) + @pytest.mark.parametrize("deployable", [True, False]) + @pytest.mark.parametrize( + "value, refs", + [ + ("starrocks.upstream", ["managed"]), + ("'starrocks.upstream'", ["managed"]), + ("' starrocks.upstream, ,external.raw, '", ["managed", "external.raw"]), + ("(starrocks.upstream, external.raw)", ["managed", "external.raw"]), + ("('starrocks.upstream', 'external.raw')", ["managed", "external.raw"]), + ("(starrocks.upstream, 'external.raw')", ["managed", "external.raw"]), + ("[starrocks.upstream, external.raw]", ["managed", "external.raw"]), + ("['starrocks.upstream', 'external.raw']", ["managed", "external.raw"]), + ("[starrocks.upstream, 'external.raw']", ["managed", "external.raw"]), + ( + "['external.raw', starrocks.upstream, 'starrocks.upstream']", + ["external.raw", "managed", "managed"], + ), + ("[' ', starrocks.upstream, '']", ["managed"]), + ("('', ' ')", []), + ("[]", []), + ("[starrocks.upstream]", ["managed"]), + ("['starrocks.upstream']", ["managed"]), + ("''", []), + ], + ) + def test_reference_syntax( + self, + make_mocked_engine_adapter: t.Callable[..., EngineAdapter], + key: str, + deployable: bool, + value: str, + refs: t.List[str], + ) -> None: + snapshot = self._make_snapshot( + _load_sql_model( + "MODEL (name starrocks.upstream, kind FULL, dialect starrocks); SELECT 1 AS a" + ) + ) + model = _load_sql_model( + f""" + MODEL ( + name starrocks.mv, + kind VIEW (materialized true), + dialect starrocks, + physical_properties ( + refresh_scheme = ASYNC, + {key} = {value}, + untouched = ('x', 'y') + ) + ); + SELECT a FROM starrocks.upstream; + """ + ) + original = {k: v.copy() for k, v in model.physical_properties.items()} + snapshots = {snapshot.name: snapshot} + index = ( + DeployabilityIndex.all_deployable() + if deployable + else DeployabilityIndex.none_deployable() + ) + adapter = make_mocked_engine_adapter(StarRocksEngineAdapter) + rendered = model.render_physical_properties( + snapshots=snapshots, engine_adapter=adapter, deployability_index=index + ) + physical = exp.table_name( + exp.to_table(snapshot.table_name(is_deployable=deployable)), identify=False + ) + expected = ",".join(physical if ref == "managed" else ref for ref in refs) + assert rendered[key] == exp.Literal.string(expected) + assert rendered["untouched"] == original["untouched"] + assert model.physical_properties == original + + # Adapters without opted-in property keys keep the original values. + assert model.render_physical_properties( + snapshots=snapshots, + engine_adapter=make_mocked_engine_adapter(DuckDBEngineAdapter), + deployability_index=index, + ) == model.render_physical_properties(snapshots=snapshots, deployability_index=index) + + adapter.create_view( + model.name, + model.render_query(), + replace=False, + materialized=True, + target_columns_to_types={"a": exp.DataType.build("INT")}, + view_properties=rendered, + ) + assert f"'{key}'='{expected}'" in to_sql_calls(adapter)[-1] + def test_single_managed_model_ref_is_resolved_to_physical_name( self, make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter], From a97d710ed2bf83f11d01919863752908c15b3122 Mon Sep 17 00:00:00 2001 From: mday-io Date: Fri, 16 Oct 2026 15:41:50 -0400 Subject: [PATCH 2/2] docs: clarify physical property reference syntax Signed-off-by: mday-io --- docs/integrations/engines/starrocks.md | 15 ++++++--------- sqlmesh/core/model/definition.py | 9 +++------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/integrations/engines/starrocks.md b/docs/integrations/engines/starrocks.md index 4e517cff45..97e96bf610 100644 --- a/docs/integrations/engines/starrocks.md +++ b/docs/integrations/engines/starrocks.md @@ -517,9 +517,9 @@ SELECT user_id, COUNT(*) AS event_count FROM user_events GROUP BY user_id; * `excluded_trigger_tables`: base tables whose data changes should **not** automatically trigger a refresh of this MV. * `excluded_refresh_tables`: base tables that should **not** be scanned when the MV refreshes. -Both properties accept a single table reference, a comma-separated string of table references, or a tuple or array of table references. +Both properties accept a single reference, a comma-separated string, a tuple, or an array. References can be bare identifiers or quoted strings. -StarRocks requires the **physical** base table name for these properties, not the logical view name that SQLMesh normally exposes. SQLMesh handles this automatically: when a reference matches a managed SQLMesh model, the logical name is resolved to its physical table name before the `CREATE MATERIALIZED VIEW` statement is issued. References that do not match any managed model are passed through unchanged. +StarRocks requires physical base table names in these properties. SQLMesh replaces references to managed models with physical table names and passes other references through unchanged. It sends the resulting names to StarRocks as a comma-separated string. ```sql MODEL ( @@ -529,7 +529,6 @@ MODEL ( ), physical_properties ( refresh_scheme = 'ASYNC', - -- SQLMesh resolves mydb.orders and mydb.order_items to their physical table names excluded_trigger_tables = 'mydb.orders,mydb.order_items', excluded_refresh_tables = mydb.orders ) @@ -538,18 +537,16 @@ MODEL ( SELECT order_id, SUM(amount) AS total FROM mydb.orders GROUP BY order_id; ``` -A single reference can be written as a bare identifier (`mydb.orders`) or as a quoted string. Multiple references can be provided as a quoted, comma-separated string (`'mydb.orders,mydb.order_items'`), a tuple, or an array. Tuple and array elements can be bare identifiers, quoted strings, or a mix of both: +Tuple and array elements can mix bare identifiers and quoted strings: ```sql physical_properties ( - refresh_scheme = 'ASYNC', - excluded_trigger_tables = (mydb.orders, 'mydb.order_items'), - excluded_refresh_tables = ['mydb.orders', mydb.order_items] + refresh_scheme = 'ASYNC', + excluded_trigger_tables = (mydb.orders, 'mydb.order_items'), + excluded_refresh_tables = ['mydb.orders', mydb.order_items] ) ``` -SQLMesh resolves each reference individually and passes the result to StarRocks as a comma-separated string of table names, regardless of the input form. - **Other properties:** You can specify `partitioning`, `distribution`, `order by` and `properties` the same as normal table properties. But notice that only supported MV properties are useful, Refer to StarRocks' doc for MV creation. diff --git a/sqlmesh/core/model/definition.py b/sqlmesh/core/model/definition.py index a99174ad90..684641e57c 100644 --- a/sqlmesh/core/model/definition.py +++ b/sqlmesh/core/model/definition.py @@ -2832,13 +2832,10 @@ def _split_sql_model_statements( def _resolve_model_refs_to_physical_tables( value: exp.Expr, table_mapping: t.Dict[str, str], dialect: DialectType ) -> exp.Literal: - """Resolve managed-model references in a property value to their physical table names. + """Resolve managed-model references in a property value to physical table names. - The value is a single table reference, a comma-separated string, or a tuple/array of - table references. Each reference that matches a managed model (via ``table_mapping``) is - swapped for its physical ``db.table`` name; - anything else (e.g. a raw source) is kept as written. Returns a single string literal so the - property renders just like a hand-written value. + Unmatched references keep their table names. The function returns the names as one + comma-separated string literal. """ if isinstance(value, exp.Literal) and value.is_string: refs = value.this.split(",")