Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions docs/integrations/engines/starrocks.md
Original file line number Diff line number Diff line change
Expand Up @@ -517,9 +517,9 @@ SELECT user_id, COUNT(*) AS event_count FROM user_events GROUP BY user_id;
* `excluded_trigger_tables`: base tables whose data changes should **not** automatically trigger a refresh of this MV.
* `excluded_refresh_tables`: base tables that should **not** be scanned when the MV refreshes.

Both properties accept a single table reference or a comma-separated list of table references.
Both properties accept a single reference, a comma-separated string, a tuple, or an array. References can be bare identifiers or quoted strings.

StarRocks requires the **physical** base table name for these properties, not the logical view name that SQLMesh normally exposes. SQLMesh handles this automatically: when a reference matches a managed SQLMesh model, the logical name is resolved to its physical table name before the `CREATE MATERIALIZED VIEW` statement is issued. References that do not match any managed model are passed through unchanged.
StarRocks requires physical base table names in these properties. SQLMesh replaces references to managed models with physical table names and passes other references through unchanged. It sends the resulting names to StarRocks as a comma-separated string.

```sql
MODEL (
Expand All @@ -529,7 +529,6 @@ MODEL (
),
physical_properties (
refresh_scheme = 'ASYNC',
-- SQLMesh resolves mydb.orders and mydb.order_items to their physical table names
excluded_trigger_tables = 'mydb.orders,mydb.order_items',
excluded_refresh_tables = mydb.orders
)
Expand All @@ -538,7 +537,15 @@ MODEL (
SELECT order_id, SUM(amount) AS total FROM mydb.orders GROUP BY order_id;
```

A single reference can be written as a bare identifier (`mydb.orders`) or as a quoted string. Multiple references must be provided as a quoted, comma-separated string (`'mydb.orders,mydb.order_items'`).
Tuple and array elements can mix bare identifiers and quoted strings:

```sql
physical_properties (
refresh_scheme = 'ASYNC',
excluded_trigger_tables = (mydb.orders, 'mydb.order_items'),
excluded_refresh_tables = ['mydb.orders', mydb.order_items]
)
```

**Other properties:**

Expand Down
13 changes: 8 additions & 5 deletions sqlmesh/core/model/definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -2832,15 +2832,18 @@ def _split_sql_model_statements(
def _resolve_model_refs_to_physical_tables(
value: exp.Expr, table_mapping: t.Dict[str, str], dialect: DialectType
) -> exp.Literal:
"""Resolve managed-model references in a property value to their physical table names.
"""Resolve managed-model references in a property value to physical table names.

The value is a single table reference or a comma-separated list of them. Each reference that
matches a managed model (via ``table_mapping``) is swapped for its physical ``db.table`` name;
anything else (e.g. a raw source) is kept as written. Returns a single string literal so the
property renders just like a hand-written value.
Unmatched references keep their table names. The function returns the names as one
comma-separated string literal.
"""
if isinstance(value, exp.Literal) and value.is_string:
refs = value.this.split(",")
elif isinstance(value, (exp.Tuple, exp.Array)):
refs = [
ref.this if isinstance(ref, exp.Literal) and ref.is_string else ref.sql(dialect=dialect)
for ref in value.expressions
]
else:
refs = [value.sql(dialect=dialect)]

Expand Down
90 changes: 90 additions & 0 deletions tests/core/engine_adapter/test_starrocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2180,6 +2180,96 @@ def _build_mv_with_excluded_tables(
calls = to_sql_calls(adapter)
return calls[-1]

@pytest.mark.parametrize("key", ["excluded_trigger_tables", "excluded_refresh_tables"])
@pytest.mark.parametrize("deployable", [True, False])
@pytest.mark.parametrize(
"value, refs",
[
("starrocks.upstream", ["managed"]),
("'starrocks.upstream'", ["managed"]),
("' starrocks.upstream, ,external.raw, '", ["managed", "external.raw"]),
("(starrocks.upstream, external.raw)", ["managed", "external.raw"]),
("('starrocks.upstream', 'external.raw')", ["managed", "external.raw"]),
("(starrocks.upstream, 'external.raw')", ["managed", "external.raw"]),
("[starrocks.upstream, external.raw]", ["managed", "external.raw"]),
("['starrocks.upstream', 'external.raw']", ["managed", "external.raw"]),
("[starrocks.upstream, 'external.raw']", ["managed", "external.raw"]),
(
"['external.raw', starrocks.upstream, 'starrocks.upstream']",
["external.raw", "managed", "managed"],
),
("[' ', starrocks.upstream, '']", ["managed"]),
("('', ' ')", []),
("[]", []),
("[starrocks.upstream]", ["managed"]),
("['starrocks.upstream']", ["managed"]),
("''", []),
],
)
def test_reference_syntax(
self,
make_mocked_engine_adapter: t.Callable[..., EngineAdapter],
key: str,
deployable: bool,
value: str,
refs: t.List[str],
) -> None:
snapshot = self._make_snapshot(
_load_sql_model(
"MODEL (name starrocks.upstream, kind FULL, dialect starrocks); SELECT 1 AS a"
)
)
model = _load_sql_model(
f"""
MODEL (
name starrocks.mv,
kind VIEW (materialized true),
dialect starrocks,
physical_properties (
refresh_scheme = ASYNC,
{key} = {value},
untouched = ('x', 'y')
)
);
SELECT a FROM starrocks.upstream;
"""
)
original = {k: v.copy() for k, v in model.physical_properties.items()}
snapshots = {snapshot.name: snapshot}
index = (
DeployabilityIndex.all_deployable()
if deployable
else DeployabilityIndex.none_deployable()
)
adapter = make_mocked_engine_adapter(StarRocksEngineAdapter)
rendered = model.render_physical_properties(
snapshots=snapshots, engine_adapter=adapter, deployability_index=index
)
physical = exp.table_name(
exp.to_table(snapshot.table_name(is_deployable=deployable)), identify=False
)
expected = ",".join(physical if ref == "managed" else ref for ref in refs)
assert rendered[key] == exp.Literal.string(expected)
assert rendered["untouched"] == original["untouched"]
assert model.physical_properties == original

# Adapters without opted-in property keys keep the original values.
assert model.render_physical_properties(
snapshots=snapshots,
engine_adapter=make_mocked_engine_adapter(DuckDBEngineAdapter),
deployability_index=index,
) == model.render_physical_properties(snapshots=snapshots, deployability_index=index)

adapter.create_view(
model.name,
model.render_query(),
replace=False,
materialized=True,
target_columns_to_types={"a": exp.DataType.build("INT")},
view_properties=rendered,
)
assert f"'{key}'='{expected}'" in to_sql_calls(adapter)[-1]

def test_single_managed_model_ref_is_resolved_to_physical_name(
self,
make_mocked_engine_adapter: t.Callable[..., StarRocksEngineAdapter],
Expand Down
Loading