From 1672c3db9c097374970d8af0b29e9675b6626bd1 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Wed, 11 Nov 2026 20:07:07 -0800 Subject: [PATCH 1/3] Add configurable table-backed view definitions --- CMakeLists.txt | 1 + README.md | 52 ++ include/yardstick_ffi.h | 5 + src/definition_store.cpp | 896 ++++++++++++++++++++++++++ src/include/definition_store.hpp | 76 +++ src/yardstick_extension.cpp | 149 ++++- src/yardstick_parser_ffi.cpp | 5 +- test/python/test_definition_store.py | 168 +++++ test/sql/definition_store.test | 309 +++++++++ test/sql/native_definition_store.test | 67 ++ yardstick-rs/src/ffi.rs | 24 + yardstick-rs/src/sql/measures.rs | 76 ++- 12 files changed, 1820 insertions(+), 8 deletions(-) create mode 100644 src/definition_store.cpp create mode 100644 src/include/definition_store.hpp create mode 100644 test/python/test_definition_store.py create mode 100644 test/sql/definition_store.test create mode 100644 test/sql/native_definition_store.test diff --git a/CMakeLists.txt b/CMakeLists.txt index 334c8a1..27b82d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -154,6 +154,7 @@ endif() include_directories(${CMAKE_CURRENT_SOURCE_DIR}/yardstick-rs/include) set(EXTENSION_SOURCES + src/definition_store.cpp src/yardstick_extension.cpp src/yardstick_parser_ffi.cpp src/frontend_peg.cpp diff --git a/README.md b/README.md index 8d3a9a4..4b93314 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,58 @@ SELECT FROM view_name; ``` +### Choosing Where Definitions Are Stored + +Set a fully qualified table name to keep semantic view definitions in any writable +DuckDB attachment: + +```sql +-- Attach your database or lake using its normal DuckDB extension first. +SET yardstick_definition_table = 'lake.semantic.yardstick_views'; + +CREATE VIEW sales_v AS +SELECT region, SUM(amount) AS MEASURE revenue +FROM lake.sales.orders; + +SELECT region, AGGREGATE(revenue) FROM sales_v; +``` + +The setting selects the **attachment, schema, and table**. The schema must already +exist; Yardstick creates the definitions table on the first permanent measure-view +declaration. It uses four `VARCHAR` columns: `view_catalog`, `view_schema`, +`view_name`, and `definition_sql`. + +The selected table is the sole source of truth. `CREATE VIEW ... AS MEASURE` +inserts a definition, `CREATE OR REPLACE VIEW` updates it, and `DROP VIEW` removes +it. Replacing a managed view with an ordinary view definition keeps that definition +in the selected table. These writes use ordinary attachment table operations in +the caller's transaction, so `ROLLBACK` also rolls back definition changes. Source +data is never copied or materialized. + +In a later session, attach the same databases and set the same +`yardstick_definition_table`; queries resolve the stored definitions automatically. +The setting is session-local. Reattach source catalogs under the same names: +Yardstick qualifies table references when saving definitions so later `USE` or +`search_path` changes do not redirect them. + +Configured definitions are resolved during binding, without creating duplicate +native view entries. Inspect the selected table to list them; they do not appear +in `duckdb_views()`. Ordinary unmanaged DuckDB views keep their usual behavior. + +For a completely session-local store, use the temporary catalog: + +```sql +SET yardstick_definition_table = 'temp.main.yardstick_views'; +``` + +Queries that use stored definitions read the table during binding, using the +caller's attachment transaction and DuckDB's existing file/catalog caches. +Yardstick does not maintain a separate durable copy or a TTL cache. Simple queries +that directly read native tables skip the definitions scan. Prepared semantic +queries rebind so replacements and drops are visible on subsequent executions. + +`RESET yardstick_definition_table` restores the existing in-memory catalog behavior. + On DuckDB 1.5+, queries containing `AGGREGATE()` are automatically intercepted by the extension's parser override, so no special prefix is needed. On older versions, the `SEMANTIC` prefix is still supported as a fallback. The native frontend also supports measures directly in `PIVOT`: diff --git a/include/yardstick_ffi.h b/include/yardstick_ffi.h index b473c78..aa8703e 100644 --- a/include/yardstick_ffi.h +++ b/include/yardstick_ffi.h @@ -19,6 +19,11 @@ extern "C" { /* Consult the current measure namespace without treating placeholder columns as dimensions. */ bool yardstick_is_measure_column(const char* view_name, const char* column_name); +/* Query-local namespace used by configured definition tables. */ +void yardstick_push_measure_view_namespace(void); +void yardstick_pop_measure_view_namespace(void); +void yardstick_rename_measure_view(const char* old_name, const char* new_name); + /* ============================================================================= * AT Modifier Types (matches ContextModifier enum in Rust) * ============================================================================= */ diff --git a/src/definition_store.cpp b/src/definition_store.cpp new file mode 100644 index 0000000..15890b8 --- /dev/null +++ b/src/definition_store.cpp @@ -0,0 +1,896 @@ +#include "definition_store.hpp" +#include "frontend_peg.hpp" +#include "yardstick_ffi.h" +#include "yardstick_parser_compat.hpp" + +#include "duckdb/catalog/catalog.hpp" +#include "duckdb/catalog/catalog_entry/table_catalog_entry.hpp" +#include "duckdb/catalog/catalog_entry/schema_catalog_entry.hpp" +#include "duckdb/catalog/catalog_search_path.hpp" +#include "duckdb/catalog/entry_lookup_info.hpp" +#include "duckdb/common/string_util.hpp" +#include "duckdb/execution/executor.hpp" +#include "duckdb/execution/physical_plan_generator.hpp" +#include "duckdb/function/replacement_scan.hpp" +#include "duckdb/main/client_data.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/database_manager.hpp" +#include "duckdb/main/query_profiler.hpp" +#include "duckdb/main/client_context_state.hpp" +#include "duckdb/parser/parsed_data/create_table_info.hpp" +#include "duckdb/parser/parsed_data/create_view_info.hpp" +#include "duckdb/parser/parsed_data/drop_info.hpp" +#include "duckdb/parser/parsed_expression_iterator.hpp" +#include "duckdb/parser/expression/subquery_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" +#include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/statement/create_statement.hpp" +#include "duckdb/parser/statement/drop_statement.hpp" +#include "duckdb/parser/statement/explain_statement.hpp" +#include "duckdb/parser/statement/select_statement.hpp" +#include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/subqueryref.hpp" +#include "duckdb/planner/binder.hpp" +#include "duckdb/planner/operator/logical_extension_operator.hpp" +#include "duckdb/planner/operator/logical_explain.hpp" +#include "duckdb/planner/planner_extension.hpp" +#if !YARDSTICK_GRAMMAR_EXTENSION +#include "duckdb/planner/operator/logical_create.hpp" +#include "duckdb/planner/operator/logical_simple.hpp" +#endif + +#include +#include +#include +#include +#include +#include + +extern "C" { +YardstickCreateViewResult yardstick_process_create_view(const char *sql); +void yardstick_free_create_view_result(YardstickCreateViewResult result); +YardstickAggregateResult yardstick_expand_aggregate(const char *sql); +void yardstick_free_aggregate_result(YardstickAggregateResult result); +void yardstick_free(char *ptr); +} + +namespace duckdb { +namespace { +constexpr const char *DEFINITION_SETTING = "yardstick_definition_table"; +std::mutex routing_lock; +std::unordered_map routing_sessions; +thread_local idx_t parsing_bypass = 0; +using StoredIdentifier = std::decay().alias)>::type; + +struct ParsingBypass { + ParsingBypass() { parsing_bypass++; } + ~ParsingBypass() { parsing_bypass--; } +}; + +string QuoteIdentifier(const string &name) { + return "\"" + StringUtil::Replace(name, "\"", "\"\"") + "\""; +} +string QuoteString(const string &value) { + return "'" + StringUtil::Replace(value, "'", "''") + "'"; +} + +template string NameString(const T &name) { +#if YARDSTICK_NEW_EXPR_API + return name.GetIdentifierName(); +#else + return name; +#endif +} + +struct ViewName { + string catalog; + string schema; + string name; + + string SQL() const { + return QuoteIdentifier(catalog) + "." + QuoteIdentifier(schema) + "." + QuoteIdentifier(name); + } + bool Equals(const ViewName &other) const { + return StringUtil::CIEquals(catalog, other.catalog) && StringUtil::CIEquals(schema, other.schema) && + StringUtil::CIEquals(name, other.name); + } +}; + +ViewName GetName(const CreateInfo &info) { +#if YARDSTICK_NEW_EXPR_API + auto &name = info.GetQualifiedName(); + return {NameString(name.Catalog()), NameString(name.Schema()), NameString(name.Name())}; +#else + auto &view = info.Cast(); + return {view.catalog, view.schema, view.view_name}; +#endif +} + +ViewName GetName(const BaseTableRef &ref) { +#if YARDSTICK_NEW_EXPR_API + auto &name = ref.GetQualifiedName(); + return {NameString(name.Catalog()), NameString(name.Schema()), NameString(name.Name())}; +#else + return {ref.catalog_name, ref.schema_name, ref.table_name}; +#endif +} + +ViewName GetName(const DropInfo &info) { +#if YARDSTICK_NEW_EXPR_API + auto &name = info.GetQualifiedName(); + return {NameString(name.Catalog()), NameString(name.Schema()), NameString(name.Name())}; +#else + return {info.catalog, info.schema, info.name}; +#endif +} + +QueryNode &SubqueryNode(SubqueryExpression &expression) { +#if YARDSTICK_NEW_EXPR_API + return *expression.Subquery()->node; +#else + return *expression.subquery->node; +#endif +} + +optional_ptr FindNativeEntry(ClientContext &context, const ViewName &name) { +#if YARDSTICK_NEW_EXPR_API + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, + QualifiedName(Identifier(name.catalog), Identifier(name.schema), Identifier(name.name))); + return Catalog::GetEntry(context, lookup, OnEntryNotFound::RETURN_NULL); +#else + EntryLookupInfo lookup(CatalogType::TABLE_ENTRY, name.name); + return Catalog::GetEntry(context, name.catalog, name.schema, lookup, OnEntryNotFound::RETURN_NULL); +#endif +} + +string SearchCatalog(const CatalogSearchEntry &entry) { +#if YARDSTICK_NEW_EXPR_API + return NameString(entry.GetCatalog()); +#else + return entry.catalog; +#endif +} + +string SearchSchema(const CatalogSearchEntry &entry) { +#if YARDSTICK_NEW_EXPR_API + return NameString(entry.GetSchema()); +#else + return entry.schema; +#endif +} + +vector> Parse(const string &sql, const ParserOptions &options) { + ParsingBypass bypass; + auto internal_options = options; +#if YARDSTICK_GRAMMAR_EXTENSION + if (auto current = CurrentNativeYardstickParserOptions()) internal_options = *current; +#endif + internal_options.extensions = nullptr; + Parser parser(internal_options); + parser.ParseQuery(sql); + return std::move(parser.statements); +} + +string DefinitionTable(const ClientContext &context) { + Value value; + if (!context.TryGetCurrentSetting(DEFINITION_SETTING, value) || value.IsNull()) return ""; + return value.GetValue(); +} + +ParserOptions DefinitionParserOptions(ClientContext &context) { +#if YARDSTICK_GRAMMAR_EXTENSION + return YardstickContextParserOptions(context); +#else + return context.GetParserOptions(); +#endif +} + +// This SELECT runs in the caller's transaction, including its uncommitted +// definition changes. A second Connection would read a different snapshot. +class DefinitionRowsSink : public PhysicalOperator { +public: + DefinitionRowsSink(PhysicalPlan &plan, PhysicalOperator &scan, vector> *rows) + : PhysicalOperator(plan, PhysicalOperatorType::EXTENSION, {LogicalType::BOOLEAN}, 1), rows(rows) { + children.push_back(scan); + } + vector> *rows; + bool IsSink() const override { return true; } + bool IsSource() const override { return true; } + SinkResultType Sink(ExecutionContext &, DataChunk &chunk, OperatorSinkInput &) const override { + if (!rows) return SinkResultType::NEED_MORE_INPUT; + for (idx_t row = 0; row < chunk.size(); row++) { + vector values; + for (idx_t col = 0; col < chunk.ColumnCount(); col++) values.push_back(chunk.GetValue(col, row)); + rows->push_back(std::move(values)); + } + return SinkResultType::NEED_MORE_INPUT; + } + SourceResultType GetDataInternal(ExecutionContext &, DataChunk &, OperatorSourceInput &) const override { + return SourceResultType::FINISHED; + } +}; + +// Keep CREATE/REPLACE/DROP's empty success result instead of exposing the +// implementation's INSERT/UPDATE/DELETE row count to callers. +class LogicalDefinitionWrite : public LogicalExtensionOperator { +public: + explicit LogicalDefinitionWrite(unique_ptr write) { + children.push_back(std::move(write)); + } + string GetExtensionName() const override { return "yardstick"; } + bool SupportSerialization() const override { return false; } + vector GetColumnBindings() override { +#if YARDSTICK_NEW_EXPR_API + return {ColumnBinding(TableIndex(0), ProjectionIndex(0))}; +#else + return {ColumnBinding(0, 0)}; +#endif + } + void ResolveTypes() override { types = {LogicalType::BOOLEAN}; } + PhysicalOperator &CreatePlan(ClientContext &, PhysicalPlanGenerator &planner) override { + auto &write = planner.CreatePlan(*children[0]); + return planner.Make(write, nullptr); + } +}; + +struct DefinitionProfilerScope { + explicit DefinitionProfilerScope(ClientContext &context) + : data(ClientData::Get(context)), previous(data.profiler) { + data.profiler = make_shared_ptr(context); + } + ~DefinitionProfilerScope() { data.profiler = std::move(previous); } + ClientData &data; + shared_ptr previous; +}; + +vector> ReadDefinitions(ClientContext &context, const string &table) { + auto statements = Parse("SELECT view_catalog, view_schema, view_name, definition_sql FROM " + table, + YardstickBuiltinParserOptions()); + auto &select = statements[0]->Cast().node->Cast(); + if (!FindNativeEntry(context, GetName(select.from_table->Cast()))) return {}; + DefinitionProfilerScope profiler(context); + auto binder = Binder::CreateBinder(context); + auto bound = binder->Bind(*statements[0]); + PhysicalPlanGenerator generator(context); + auto plan = generator.Plan(std::move(bound.plan)); + vector> rows; + auto &sink = plan->Make(plan->Root(), &rows); + { + Executor executor(context); + try { + executor.Initialize(sink); + while (!executor.ExecutionIsFinished()) { + executor.ExecuteTask(); + if (!executor.ExecutionIsFinished()) executor.WaitForTask(); + } + if (executor.HasError()) executor.ThrowException(); + } catch (...) { + executor.CancelTasks(); + throw; + } + // Completed pipelines can still have worker tasks releasing their + // state. Drain them before destroying this executor and its plan. + executor.CancelTasks(); + } + return rows; +} + +struct Definition { + ViewName name; + string sql; + string key; + string clean_sql; +}; + +class DefinitionScope; +thread_local DefinitionScope *active_definitions = nullptr; + +class DefinitionScope { +public: + DefinitionScope(ClientContext &context, const string &table, const ParserOptions &options) + : context(context), options(options), previous(active_definitions) { + auto rows = ReadDefinitions(context, table); + for (auto &row : rows) { + for (auto &value : row) { + if (value.IsNull() || value.type().id() != LogicalTypeId::VARCHAR) { + throw InvalidInputException("Yardstick definitions require non-NULL VARCHAR columns"); + } + } + Definition definition {{row[0].GetValue(), row[1].GetValue(), row[2].GetValue()}, + row[3].GetValue(), "__yardstick_definition_" + std::to_string(definitions.size()), ""}; + for (auto &existing : definitions) { + if (definition.name.Equals(existing.name)) { + throw InvalidInputException("Duplicate Yardstick definition for %s in %s", definition.name.SQL(), table); + } + } + definitions.push_back(std::move(definition)); + } + yardstick_push_measure_view_namespace(); + active_definitions = this; + } + ~DefinitionScope() { + active_definitions = previous; + yardstick_pop_measure_view_namespace(); + } + + ClientContext &context; + ParserOptions options; + vector definitions; + DefinitionScope *previous; + + optional_ptr Find(const ViewName &input) { + for (auto &definition : definitions) { + if (definition.key == input.name && input.catalog.empty() && input.schema.empty()) return &definition; + } + auto &path = *ClientData::Get(context).catalog_search_path; + auto catalog = input.catalog; + auto schema = input.schema; + if (!schema.empty() && catalog.empty()) { +#if YARDSTICK_NEW_EXPR_API + Identifier catalog_id, schema_id(schema); + Binder::BindSchemaOrCatalog(context, catalog_id, schema_id); + catalog = NameString(catalog_id); + schema = NameString(schema_id); +#else + Binder::BindSchemaOrCatalog(context, catalog, schema); +#endif + } + vector candidates; + if (!catalog.empty() && !schema.empty()) { + candidates.push_back({catalog, schema, input.name}); + } else if (!catalog.empty()) { + candidates.push_back({catalog, NameString(path.GetDefaultSchema(StoredIdentifier(catalog))), input.name}); + } else if (!schema.empty()) { + for (auto &entry : path.GetCatalogsForSchema(StoredIdentifier(schema))) candidates.push_back({NameString(entry), schema, input.name}); + if (candidates.empty()) candidates.push_back({NameString(DatabaseManager::GetDefaultDatabase(context)), schema, input.name}); + } else { + auto &entry = path.GetDefault(); + candidates.push_back({SearchCatalog(entry), SearchSchema(entry), input.name}); + for (auto &entry : path.GetSetPaths()) { + candidates.push_back({SearchCatalog(entry), SearchSchema(entry), input.name}); + } + } + for (auto &candidate : candidates) { + if (candidate.catalog.empty()) candidate.catalog = NameString(DatabaseManager::GetDefaultDatabase(context)); + if (candidate.schema.empty()) candidate.schema = DEFAULT_SCHEMA; + for (auto &definition : definitions) { + if (definition.name.Equals(candidate)) return &definition; + } + } + return nullptr; + } + + void Hydrate() { + for (auto &definition : definitions) { + auto result = yardstick_process_create_view(definition.sql.c_str()); + string error = result.error ? result.error : ""; + if (result.clean_sql) definition.clean_sql = result.clean_sql; + if (result.view_name) yardstick_rename_measure_view(result.view_name, definition.key.c_str()); + yardstick_free_create_view_result(result); + if (!error.empty()) throw BinderException("Invalid stored Yardstick definition: %s", error); + } + } +}; + +// Some earlier parser extensions (notably the 1.5 CLI's autocomplete parser) +// accept ordinary SQL before Yardstick's parser override is reached. Track +// only routing and rebind requirements here; TLS-bearing namespaces must stay +// lexical so Prepare/ExtractPlan and cross-thread clients cannot retain them. +class DefinitionSessionState : public ClientContextState { +public: + const void *parser_registry = nullptr; + bool used_binding = false; + + ~DefinitionSessionState() override { SetRouting(nullptr); } + + void SetRouting(const void *registry) { + if (parser_registry == registry) return; + std::lock_guard lock(routing_lock); + if (parser_registry && --routing_sessions[parser_registry] == 0) routing_sessions.erase(parser_registry); + parser_registry = registry; + if (parser_registry) routing_sessions[parser_registry]++; + } + + void QueryEnd() override { used_binding = false; } +}; + +unique_ptr ReplaceDefinition(ClientContext &context, ReplacementScanInput &input, + optional_ptr) { + if (DefinitionTable(context).empty()) return nullptr; + unique_ptr local_definitions; + if (!active_definitions || &active_definitions->context != &context) { + local_definitions = make_uniq(context, DefinitionTable(context), DefinitionParserOptions(context)); + local_definitions->Hydrate(); + } + auto definition = active_definitions->Find({input.catalog_name, input.schema_name, input.table_name}); + if (!definition) return nullptr; + context.registered_state->GetOrCreate("yardstick_definition_binding")->used_binding = true; + auto statements = Parse(definition->clean_sql, YardstickBuiltinParserOptions()); + auto &create = statements[0]->Cast().info->Cast(); + auto result = make_uniq(std::move(create.query)); + if (!create.aliases.empty()) { + // Replacement scans overwrite their outer column aliases with the + // caller's list. Keep the declaration's header on an inner relation. + result->column_name_alias = create.aliases; + result->alias = "__yardstick_view_body"; + auto select = make_uniq(); + select->select_list.push_back(make_uniq()); + select->from_table = std::move(result); + auto query = make_uniq(); + query->node = std::move(select); + result = make_uniq(std::move(query)); + } + result->alias = StoredIdentifier(definition->name.name); + return std::move(result); +} + +void SetDefinitionTable(ClientContext &context, SetScope scope, Value &value) { + if (scope == SetScope::GLOBAL) throw InvalidInputException("yardstick_definition_table is a session setting"); + auto state = context.registered_state->GetOrCreate("yardstick_definition_binding"); + if (value.IsNull() || value.GetValue().empty()) { + state->SetRouting(nullptr); + return; + } + auto sql = "CREATE TABLE " + value.GetValue() + " (view_catalog VARCHAR)"; + auto statements = Parse(sql, YardstickBuiltinParserOptions()); + if (statements.size() != 1 || statements[0]->type != StatementType::CREATE_STATEMENT || + statements[0]->Cast().info->type != CatalogType::TABLE_ENTRY) { + throw InvalidInputException("yardstick_definition_table must be a catalog.schema.table name"); + } + auto &info = statements[0]->Cast().info->Cast(); +#if YARDSTICK_NEW_EXPR_API + auto &name = info.GetQualifiedName(); + if (name.Path().size() != 3) throw InvalidInputException("yardstick_definition_table must be catalog.schema.table"); + value = Value(QuoteIdentifier(NameString(name.Catalog())) + "." + QuoteIdentifier(NameString(name.Schema())) + "." + + QuoteIdentifier(NameString(name.Name()))); +#else + if (info.catalog.empty() || info.schema.empty()) throw InvalidInputException("yardstick_definition_table must be catalog.schema.table"); + value = Value(QuoteIdentifier(info.catalog) + "." + QuoteIdentifier(info.schema) + "." + QuoteIdentifier(info.table)); +#endif + state->SetRouting(DefinitionParserOptions(context).extensions.get()); +} + +BoundStatement BindSQL(ClientContext &context, Binder &parent, const string &sql, const ParserOptions &options) { + auto statements = Parse(sql, options); + if (statements.size() != 1) throw BinderException("Expected one Yardstick statement"); + auto binder = Binder::CreateBinder(context, &parent); + return binder->Bind(*statements[0]); +} + +BoundStatement BindDefinitionWrite(ClientContext &context, Binder &parent, const string &sql) { + auto bound = BindSQL(context, parent, sql, YardstickBuiltinParserOptions()); + bound.plan = make_uniq(std::move(bound.plan)); + bound.names = {"Success"}; + bound.types = {LogicalType::BOOLEAN}; + parent.GetStatementProperties().return_type = StatementReturnType::NOTHING; + return bound; +} + +string KeyPredicate(const ViewName &name) { + return "lower(view_catalog) = lower(" + QuoteString(name.catalog) + ") AND lower(view_schema) = lower(" + + QuoteString(name.schema) + ") AND lower(view_name) = lower(" + QuoteString(name.name) + ")"; +} + +string MaskMeasureDeclarations(const string &sql) { + auto result = sql; + auto tokens = Parser::Tokenize(sql); + string previous; + for (idx_t index = 0; index < tokens.size(); index++) { + auto &token = tokens[index]; + if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT) continue; + auto end = token.start; + while (end < sql.size() && (std::isalnum(static_cast(sql[end])) || sql[end] == '_')) end++; + auto word = StringUtil::Upper(sql.substr(token.start, end - token.start)); + auto next = index + 1; + while (next < tokens.size() && tokens[next].type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT) next++; + if (previous == "AS" && word == "MEASURE" && next < tokens.size() && + tokens[next].type == SimplifiedTokenType::SIMPLIFIED_TOKEN_IDENTIFIER) { + std::fill(result.begin() + token.start, result.begin() + end, ' '); + } + previous = std::move(word); + } + return result; +} + +string FirstKeyword(const string &sql) { + for (auto &token : Parser::Tokenize(sql)) { + if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT) continue; + auto end = token.start; + while (end < sql.size() && std::isalpha(static_cast(sql[end]))) end++; + return StringUtil::Upper(sql.substr(token.start, end - token.start)); + } + return ""; +} + +idx_t ExplainBodyOffset(const string &sql) { + idx_t depth = 0; + for (auto &token : Parser::Tokenize(sql)) { + if (token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT || + token.type == SimplifiedTokenType::SIMPLIFIED_TOKEN_STRING_CONSTANT) continue; + if (sql[token.start] == '(') { depth++; continue; } + if (sql[token.start] == ')') { if (depth) depth--; continue; } + if (depth) continue; + auto end = token.start; + while (end < sql.size() && std::isalpha(static_cast(sql[end]))) end++; + auto keyword = StringUtil::Upper(sql.substr(token.start, end - token.start)); + for (auto body : {"SELECT", "WITH", "VALUES", "CREATE", "DROP", "INSERT", "UPDATE", "DELETE", "COPY", + "EXECUTE", "CALL"}) { + if (keyword == body) return token.start; + } + } + throw ParserException("Expected a statement after EXPLAIN"); +} + +bool NeedsDefinitionLookup(ClientContext &context, const string &sql, const ParserOptions &options) { + vector> statements; + try { + statements = Parse(sql, options); + } catch (const ParserException &) { + return true; // Compatibility AT syntax is parsed after lowering. + } + if (statements.size() != 1 || statements[0]->type != StatementType::SELECT_STATEMENT) return true; + bool needs_definitions = false; + std::function &)> visit_expression; + std::function visit_ref; + visit_expression = [&](unique_ptr &expression) { + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + ParsedExpressionIterator::EnumerateQueryNodeChildren( + SubqueryNode(expression->Cast()), visit_expression, visit_ref); + } + ParsedExpressionIterator::EnumerateChildren(*expression, visit_expression); + }; + visit_ref = [&](TableRef &ref) { + if (needs_definitions || ref.type != TableReferenceType::BASE_TABLE) return; + auto entry = FindNativeEntry(context, GetName(ref.Cast())); + needs_definitions = !entry || entry->type == CatalogType::VIEW_ENTRY; + }; + ParsedExpressionIterator::EnumerateQueryNodeChildren(*statements[0]->Cast().node, + visit_expression, visit_ref); + return needs_definitions; +} + +// Persist resolved source names, so USE/search_path changes in a later session +// cannot redirect a stored measure to an unrelated table. Keep declaration SQL +// intact, including AS MEASURE, comments, and header column aliases. +string QualifyDefinitionSources(ClientContext &context, DefinitionScope &definitions, const string &sql, + SelectStatement &query, bool temporary_store) { + vector>> replacements; + std::set cte_names; + std::function collect_ctes; + std::function &)> collect_expression; + collect_expression = [&](unique_ptr &expression) { + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + collect_ctes(SubqueryNode(expression->Cast())); + } + ParsedExpressionIterator::EnumerateChildren(*expression, collect_expression); + }; + collect_ctes = [&](QueryNode &node) { + for (auto &entry : node.cte_map.map) { + cte_names.insert(StringUtil::Lower(NameString(entry.first))); +#if YARDSTICK_NEW_EXPR_API + collect_ctes(*entry.second->query_node); +#else + collect_ctes(*entry.second->query->node); +#endif + } + ParsedExpressionIterator::EnumerateQueryNodeChildren(node, collect_expression); + }; + collect_ctes(*query.node); + auto identifier_end = [&](idx_t start) { + if (sql[start] == '"') { + auto end = start + 1; + while (end < sql.size()) { + if (sql[end++] != '"') continue; + if (end < sql.size() && sql[end] == '"') { end++; continue; } + break; + } + return end; + } + auto end = start; + while (end < sql.size()) { + auto c = static_cast(sql[end]); + if (!std::isalnum(c) && c != '_' && c != '$' && c < 128) break; + end++; + } + return end; + }; + auto tokens = Parser::Tokenize(sql); + std::function &)> visit_expression; + std::function visit_ref; + visit_expression = [&](unique_ptr &expression) { + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + ParsedExpressionIterator::EnumerateQueryNodeChildren( + SubqueryNode(expression->Cast()), visit_expression, visit_ref); + } + ParsedExpressionIterator::EnumerateChildren(*expression, visit_expression); + }; + visit_ref = [&](TableRef &ref) { + if (ref.type != TableReferenceType::BASE_TABLE || !ref.query_location.IsValid()) return; + auto input = GetName(ref.Cast()); + if (input.catalog.empty() && input.schema.empty() && cte_names.count(StringUtil::Lower(input.name))) return; + ViewName resolved; + if (auto entry = FindNativeEntry(context, input)) { + if (entry->temporary && !temporary_store) { + throw BinderException("A persistent Yardstick definition cannot reference temporary table %s", input.name); + } + resolved = {NameString(entry->ParentCatalog().GetName()), NameString(entry->ParentSchema().name), NameString(entry->name)}; + } else if (auto definition = definitions.Find(input)) { + resolved = definition->name; + } else { + return; // Leave replacement scans (including file paths) to DuckDB. + } +#if YARDSTICK_NEW_EXPR_API + idx_t start = ref.query_location.Start(); +#else + idx_t start = ref.query_location.GetIndex(); +#endif + auto end = identifier_end(start); + // Tokenization supplies exact dot/name boundaries even when comments + // or whitespace separate qualified-name components. + idx_t components = 1 + !input.schema.empty() + !input.catalog.empty(); + auto next = std::find_if(tokens.begin(), tokens.end(), [&](const SimplifiedToken &token) { return token.start >= end; }); + while (components > 1 && next != tokens.end()) { + if (next->type == SimplifiedTokenType::SIMPLIFIED_TOKEN_COMMENT) { next++; continue; } + if (sql[next->start] == '.') { next++; continue; } + end = identifier_end(next->start); + components--; + next++; + } + replacements.push_back({start, {end, resolved.SQL()}}); + }; + ParsedExpressionIterator::EnumerateQueryNodeChildren(*query.node, visit_expression, visit_ref); + auto result = sql; + std::sort(replacements.begin(), replacements.end(), [](const auto &a, const auto &b) { return a.first > b.first; }); + for (auto &replacement : replacements) { + result.replace(replacement.first, replacement.second.first - replacement.first, replacement.second.second); + } + return result; +} +} // namespace + +bool RouteYardstickDefinitionStatements(const string &query, const ParserOptions &options) { + if (parsing_bypass) return false; + if (StringUtil::Contains(StringUtil::Lower(query), DEFINITION_SETTING)) return true; +#if YARDSTICK_GRAMMAR_EXTENSION + return options.context && !DefinitionTable(*options.context).empty(); +#else + std::lock_guard lock(routing_lock); + return routing_sessions.count(options.extensions.get()) != 0; +#endif +} +bool YardstickDefinitionParsingBypassed() { return parsing_bypass != 0; } +bool YardstickDefinitionStoreConfigured(const ClientContext &context) { return !DefinitionTable(context).empty(); } + +unique_ptr YardstickDefinitionQueryRef(ClientContext &context, const string &sql, + const ParserOptions &options, ParserExtensionInfo *parser_info) { + if (DefinitionTable(context).empty()) return nullptr; + ParsingBypass bypass; +#if YARDSTICK_GRAMMAR_EXTENSION + NativeYardstickBindScope bind_scope(context); + NativeYardstickParseScope parse_scope(parser_info, options); +#endif + unique_ptr local_definitions; + if (!active_definitions || &active_definitions->context != &context) { + local_definitions = make_uniq(context, DefinitionTable(context), options); + local_definitions->Hydrate(); + } + auto temporary = EnterYardstickTemporaryDefinitions(context); + auto expanded = yardstick_expand_aggregate(sql.c_str()); + string error = expanded.error ? expanded.error : ""; + string query = expanded.expanded_sql ? expanded.expanded_sql : sql; + string warnings = expanded.warnings ? expanded.warnings : ""; + yardstick_free_aggregate_result(expanded); + if (!error.empty()) throw BinderException(error); + HandleAggregateWarnings(context, warnings); + auto statements = Parse(query, options); + if (statements.size() != 1 || statements[0]->type != StatementType::SELECT_STATEMENT) { + throw BinderException("yardstick() requires one SELECT statement"); + } + return make_uniq(unique_ptr_cast(std::move(statements[0]))); +} + +string YardstickStoredRelationName(const BaseTableRef &ref) { + if (!active_definitions) return ""; + auto name = GetName(ref); + auto definition = active_definitions->Find(name); + if (!definition || FindNativeEntry(active_definitions->context, name)) return ""; + return definition ? definition->key : ""; +} + +void RegisterYardstickDefinitionStore(DBConfig &config) { + config.AddExtensionOption(DEFINITION_SETTING, + "Fully qualified table holding Yardstick view definitions; empty uses the legacy in-memory catalog", + LogicalType::VARCHAR, Value(""), SetDefinitionTable); + config.replacement_scans.emplace_back(ReplaceDefinition); + PlannerExtension planner; + planner.post_bind_function = [](PlannerExtensionInput &input, BoundStatement &statement) { + auto state = input.context.registered_state->Get("yardstick_definition_binding"); + if (state && state->used_binding) { + input.binder.GetStatementProperties().always_require_rebind = true; + state->used_binding = false; + } +#if !YARDSTICK_GRAMMAR_EXTENSION + if (parsing_bypass || DefinitionTable(input.context).empty()) return; + input.binder.GetStatementProperties().always_require_rebind = true; + string sql; + ViewName name; + auto target = statement.plan.get(); + optional_ptr explanation; + if (target->type == LogicalOperatorType::LOGICAL_EXPLAIN) { + explanation = target->Cast(); + target = explanation->children[0].get(); + } + if (target->type == LogicalOperatorType::LOGICAL_CREATE_VIEW) { + auto &view = target->Cast().info->Cast(); + if (view.temporary) return; + name = GetName(view); + sql = view.ToString(); + } else if (target->type == LogicalOperatorType::LOGICAL_DROP) { + auto &drop = target->Cast().info->Cast(); + if (drop.type != CatalogType::VIEW_ENTRY) return; + name = GetName(drop); + sql = drop.ToString(); + } else { + return; + } + DefinitionScope definitions(input.context, DefinitionTable(input.context), input.context.GetParserOptions()); + if (!definitions.Find(name)) return; + definitions.Hydrate(); + // The native binder planned a catalog write before discovering the + // managed definition. Replace that write with the store's ordinary DML. + input.binder.GetStatementProperties().modified_databases.clear(); + auto rewritten = BindYardstickDefinitionStatement(input.context, input.binder, + YardstickDefinitionParseData(sql, input.context.GetParserOptions()), nullptr); + if (explanation) { + explanation->children[0] = std::move(rewritten.plan); + explanation->logical_plan_unopt = explanation->children[0]->ToString(explanation->explain_format); + input.binder.GetStatementProperties().return_type = StatementReturnType::QUERY_RESULT; + } else { + statement = std::move(rewritten); + } +#endif + }; + PlannerExtension::Register(config, std::move(planner)); +} + +BoundStatement BindYardstickDefinitionStatement(ClientContext &context, Binder &parent, + const YardstickDefinitionParseData &data, + ParserExtensionInfo *parser_info) { + ParsingBypass bypass; + auto table = DefinitionTable(context); + if (table.empty()) { + if (data.initialize) return BindSQL(context, parent, "SELECT true WHERE false", YardstickBuiltinParserOptions()); + return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); + } +#if YARDSTICK_GRAMMAR_EXTENSION + NativeYardstickBindScope bind_scope(context); + NativeYardstickParseScope parse_scope(parser_info, data.options); +#endif + if (data.initialize) { + auto create = StringUtil::StartsWith(StringUtil::Lower(table), "\"temp\".") + ? "CREATE TEMP TABLE IF NOT EXISTS " : "CREATE TABLE IF NOT EXISTS "; + return BindSQL(context, parent, create + table + + " (view_catalog VARCHAR, view_schema VARCHAR, view_name VARCHAR, definition_sql VARCHAR)", + YardstickBuiltinParserOptions()); + } + if (FirstKeyword(data.sql) == "EXPLAIN") { + auto body = ExplainBodyOffset(data.sql); + auto parsed = Parse(data.sql.substr(0, body) + " SELECT true", YardstickBuiltinParserOptions()); + auto &explanation = parsed[0]->Cast(); + auto child = BindYardstickDefinitionStatement(context, parent, + YardstickDefinitionParseData(data.sql.substr(body), data.options), parser_info); +#if YARDSTICK_NEW_EXPR_API + auto plan = make_uniq(std::move(child.plan), explanation.explain_type, explanation.format); + plan->logical_plan_unopt = plan->children[0]->ToString(context, explanation.format); + if (explanation.explain_type == ExplainType::EXPLAIN_SQL) { + plan->sql_output_names = std::move(child.names); + plan->allow_unsupported_sql = explanation.allow_unsupported_sql; + } +#else + auto plan = make_uniq(std::move(child.plan), explanation.explain_type, explanation.explain_format); + plan->logical_plan_unopt = plan->children[0]->ToString(explanation.explain_format); +#endif + child.plan = std::move(plan); + child.names = {"explain_key", "explain_value"}; + child.types = {LogicalType::VARCHAR, LogicalType::VARCHAR}; + parent.GetStatementProperties().return_type = StatementReturnType::QUERY_RESULT; + return child; + } + if (!NeedsDefinitionLookup(context, data.sql, data.options)) { + return BindSQL(context, parent, data.sql, data.options); + } + parent.GetStatementProperties().always_require_rebind = true; + unique_ptr local_definitions; + if (!active_definitions || &active_definitions->context != &context) { + local_definitions = make_uniq(context, table, data.options); + local_definitions->Hydrate(); + } + auto &definitions = *active_definitions; + auto temporary_definitions = EnterYardstickTemporaryDefinitions(context); + string error; + // Context modifiers need semantic lowering before the builtin parser can + // accept them. Only view DDL needs inspection before that lowering. + auto keyword = FirstKeyword(data.sql); + bool inspect_ddl = keyword == "CREATE" || keyword == "DROP"; + if (!inspect_ddl) { + auto expanded = yardstick_expand_aggregate(data.sql.c_str()); + error = expanded.error ? expanded.error : ""; + string expanded_sql = expanded.expanded_sql ? expanded.expanded_sql : data.sql; + string warnings = expanded.warnings ? expanded.warnings : ""; + yardstick_free_aggregate_result(expanded); + if (!error.empty()) throw BinderException(error); + HandleAggregateWarnings(context, warnings); + return BindSQL(context, parent, expanded_sql, data.options); + } + auto statements = Parse(MaskMeasureDeclarations(data.sql), data.options); + if (statements.size() != 1) throw BinderException("Expected one Yardstick statement"); + auto &statement = *statements[0]; + if (statement.type == StatementType::CREATE_STATEMENT && + statement.Cast().info->type == CatalogType::VIEW_ENTRY) { + auto &view = statement.Cast().info->Cast(); + if (view.temporary) return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); + auto validator = Binder::CreateBinder(context); + validator->SearchSchema(view); + auto name = GetName(view); + auto existing = definitions.Find(name); + auto native = FindNativeEntry(context, name); + if ((existing || native) && view.on_conflict == OnCreateConflict::IGNORE_ON_CONFLICT) { + return BindDefinitionWrite(context, parent, "SELECT true WHERE false"); + } + if (native) throw CatalogException("%s already exists outside the configured Yardstick definitions table", name.SQL()); + if (existing && view.on_conflict != OnCreateConflict::REPLACE_ON_CONFLICT) { + throw CatalogException("Yardstick view %s already exists", name.SQL()); + } + bool temporary_store = StringUtil::StartsWith(StringUtil::Lower(table), "\"temp\"."); + auto definition_sql = QualifyDefinitionSources(context, definitions, data.sql, *view.query, temporary_store); + auto lowered = yardstick_process_create_view(definition_sql.c_str()); + error = lowered.error ? lowered.error : ""; + bool is_measure = lowered.is_measure_view; + yardstick_free_create_view_result(lowered); + if (!error.empty()) throw BinderException(error); + if (!is_measure && !existing) return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); + // Validate the actual expressions, not the NULL placeholders emitted + // for decomposable measures. Ordinary aggregates require the implicit + // GROUP BY that AS MEASURE supplies. + if (is_measure && view.query->node->type == QueryNodeType::SELECT_NODE) { + auto &select = view.query->node->Cast(); + bool has_window = false; + std::function &)> inspect_expression; + inspect_expression = [&](unique_ptr &expression) { + has_window |= expression->GetExpressionClass() == ExpressionClass::WINDOW; + ParsedExpressionIterator::EnumerateChildren(*expression, inspect_expression); + }; + for (auto &expression : select.select_list) inspect_expression(expression); + if (select.groups.group_expressions.empty() && !has_window) { + select.aggregate_handling = AggregateHandling::FORCE_AGGREGATES; + } + } + validator->BindCreateViewInfo(view); + auto sql = existing + ? "UPDATE " + table + " SET definition_sql = " + QuoteString(definition_sql) + " WHERE " + KeyPredicate(name) + : "INSERT INTO " + table + " (view_catalog, view_schema, view_name, definition_sql) VALUES (" + + QuoteString(name.catalog) + ", " + QuoteString(name.schema) + ", " + QuoteString(name.name) + ", " + QuoteString(definition_sql) + ")"; + return BindDefinitionWrite(context, parent, sql); + } + if (statement.type == StatementType::DROP_STATEMENT) { + auto &drop = *statement.Cast().info; + if (drop.type == CatalogType::VIEW_ENTRY) { + auto definition = definitions.Find(GetName(drop)); + if (definition) { + return BindDefinitionWrite(context, parent, + "DELETE FROM " + table + " WHERE " + KeyPredicate(definition->name)); + } + return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); + } + } + auto expanded = yardstick_expand_aggregate(data.sql.c_str()); + error = expanded.error ? expanded.error : ""; + string expanded_sql = expanded.expanded_sql ? expanded.expanded_sql : data.sql; + string warnings = expanded.warnings ? expanded.warnings : ""; + yardstick_free_aggregate_result(expanded); + if (!error.empty()) throw BinderException(error); + HandleAggregateWarnings(context, warnings); + return BindSQL(context, parent, expanded_sql, data.options); +} +} // namespace duckdb diff --git a/src/include/definition_store.hpp b/src/include/definition_store.hpp new file mode 100644 index 0000000..94a7881 --- /dev/null +++ b/src/include/definition_store.hpp @@ -0,0 +1,76 @@ +#pragma once + +#include "yardstick_compat.hpp" +#include "duckdb/parser/parser.hpp" +#include "duckdb/parser/parser_extension.hpp" +#include "duckdb/parser/statement/extension_statement.hpp" +#include "duckdb/planner/bound_statement.hpp" + +namespace duckdb { +class BaseTableRef; +class Binder; +class ClientContext; +class TableRef; +struct DBConfig; + +// DuckDB's ExtensionStatement::Copy drops SQLStatement metadata. Prepared +// statements need that metadata to validate parameters before rebinding. +class YardstickDefinitionStatement : public ExtensionStatement { +public: + YardstickDefinitionStatement(ParserExtension extension, unique_ptr data) + : ExtensionStatement(std::move(extension), std::move(data)) {} + + unique_ptr Copy() const override { + auto result = make_uniq(extension, parse_data->Copy()); + result->query = query; + result->stmt_location = stmt_location; + result->named_param_map = named_param_map; +#if YARDSTICK_NEW_EXPR_API + result->has_anonymous_parameters = has_anonymous_parameters; +#else + result->stmt_length = stmt_length; +#endif + return std::move(result); + } +}; + +// The parser has no ClientContext. Once a session selects a store, defer its +// statements until binding, where the setting and transaction are available. +bool RouteYardstickDefinitionStatements(const string &query, const ParserOptions &options); +bool YardstickDefinitionParsingBypassed(); +bool YardstickDefinitionStoreConfigured(const ClientContext &context); +unique_ptr YardstickDefinitionQueryRef(ClientContext &context, const string &sql, + const ParserOptions &options, ParserExtensionInfo *parser_info); +void HandleAggregateWarnings(ClientContext &context, const string &warnings); + +struct YardstickDefinitionParseData : ParserExtensionParseData { + string sql; + ParserOptions options; + bool initialize; + + YardstickDefinitionParseData(string sql, ParserOptions options, bool initialize = false) + : sql(std::move(sql)), options(std::move(options)), initialize(initialize) {} + unique_ptr Copy() const override { + return make_uniq_base(sql, options, initialize); + } + string ToString() const override { return sql; } +}; + +void RegisterYardstickDefinitionStore(DBConfig &config); +BoundStatement BindYardstickDefinitionStatement(ClientContext &context, Binder &binder, + const YardstickDefinitionParseData &data, + ParserExtensionInfo *parser_info); +// Preserve legacy behavior when another session enabled parser routing but +// this session has no configured store. +BoundStatement BindYardstickUnstoredStatement(ClientContext &context, Binder &binder, + const string &sql, const ParserOptions &options); + +// Supply a unique query-local measure key, retaining the SQL table alias. +string YardstickStoredRelationName(const BaseTableRef &ref); + +class YardstickTemporaryDefinitionScope { +public: + virtual ~YardstickTemporaryDefinitionScope() = default; +}; +unique_ptr EnterYardstickTemporaryDefinitions(ClientContext &context); +} // namespace duckdb diff --git a/src/yardstick_extension.cpp b/src/yardstick_extension.cpp index ed5490d..7b47f53 100644 --- a/src/yardstick_extension.cpp +++ b/src/yardstick_extension.cpp @@ -8,6 +8,9 @@ #include "yardstick_parser_extension.hpp" #include "frontend_peg.hpp" #include "aggregate_state.hpp" +#include "definition_store.hpp" +#include "duckdb/parser/statement/multi_statement.hpp" +#include "duckdb/parser/statement/prepare_statement.hpp" #include "duckdb/parser/parser.hpp" #include "duckdb/parser/parser_extension.hpp" #include "duckdb/parser/parsed_data/create_table_function_info.hpp" @@ -155,7 +158,7 @@ static std::string RewritePercentileWithinGroup(const std::string &sql) { //============================================================================= static std::string AggregateWarnings(YardstickAggregateResult &result); -static void HandleAggregateWarnings(ClientContext &context, const string &warnings); +void HandleAggregateWarnings(ClientContext &context, const string &warnings); // DuckDB 2.0 uses Identifier for result and table-function column names. using YardstickColumnName = std::decay().ColumnName(0))>::type; @@ -1104,7 +1107,7 @@ static bool WarningsAsErrors(ClientContext &context) { return setting.GetValue(); } -static void HandleAggregateWarnings(ClientContext &context, const string &warnings) { +void HandleAggregateWarnings(ClientContext &context, const string &warnings) { if (warnings.empty()) { return; } @@ -2059,6 +2062,79 @@ static vector> DeferMeasureColumnListBatch(const string ParserOverrideResult yardstick_parser_override(ParserExtensionInfo *info, const std::string &query, ParserOptions &options) { + if (RouteYardstickDefinitionStatements(query, options)) { +#if YARDSTICK_GRAMMAR_EXTENSION + NativeYardstickParseScope definition_parse_scope(info, options); +#endif + vector> statements; + auto defer = [&](const string &sql, bool initialize = false) { + auto statement = make_uniq(YardstickParserExtension(), + make_uniq_base(sql, options, initialize)); + auto syntax_options = options; +#if YARDSTICK_GRAMMAR_EXTENSION + syntax_options = *CurrentNativeYardstickParserOptions(); +#endif + syntax_options.extensions = nullptr; + try { + Parser syntax(syntax_options); + syntax.ParseQuery(sql); + if (syntax.statements.size() == 1) { + statement->named_param_map = syntax.statements[0]->named_param_map; +#if YARDSTICK_GRAMMAR_EXTENSION + statement->has_anonymous_parameters = syntax.statements[0]->has_anonymous_parameters; +#endif + } + } catch (const ParserException &) { + // Compatibility syntax is parsed after Yardstick lowering. + } + return statement; + }; + for (auto &sql : SplitSqlStatements(query)) { + StringUtil::Trim(sql); + if (sql.empty()) continue; + string definition_body; + if (StartsWithSemantic(sql, definition_body)) sql = std::move(definition_body); + auto pos = SkipWhitespaceAndComments(sql, 0); + auto prepare_pos = pos; + if (ConsumeKeyword(sql, prepare_pos, "PREPARE")) { + auto as_pos = FindTopLevelKeyword(sql, "AS"); + if (as_pos == string::npos) throw ParserException("Expected AS in PREPARE"); + Parser parser(YardstickBuiltinParserOptions()); + parser.ParseQuery(sql.substr(0, as_pos + 2) + " SELECT true"); + auto &prepare = parser.statements[0]->Cast(); + prepare.statement = defer(sql.substr(as_pos + 2)); + statements.push_back(std::move(parser.statements[0])); + continue; + } + // Keep engine-visible session/transaction statements intact. The + // setting can be selected earlier in this very statement batch. + bool engine_statement = false; + for (auto keyword : {"SET", "RESET", "BEGIN", "START", "COMMIT", "ROLLBACK", "ABORT", "ATTACH", + "DETACH", "LOAD", "INSTALL", "PRAGMA", "EXECUTE"}) { + auto next = pos; + if (ConsumeKeyword(sql, next, keyword)) { + engine_statement = true; + break; + } + } + if (engine_statement) { + Parser parser(YardstickBuiltinParserOptions()); + parser.ParseQuery(sql); + for (auto &statement : parser.statements) statements.push_back(std::move(statement)); + continue; + } + auto statement = defer(sql); + if (StartsWithCreateViewStatement(sql) && !IsTemporaryCreateViewStatement(sql)) { + auto multi = make_uniq(); + multi->statements.push_back(defer(sql, true)); + multi->statements.push_back(std::move(statement)); + statements.push_back(std::move(multi)); + } else { + statements.push_back(std::move(statement)); + } + } + return ParserOverrideResult(std::move(statements)); + } #if YARDSTICK_GRAMMAR_EXTENSION NativeYardstickParseScope native_scope(info, options); #endif @@ -2387,7 +2463,7 @@ class DeferredMeasureCatalogState : public ClientContextState { // Temporary entries belong to the originating session. A thread-local overlay // exposes them to the lowerer without modifying another binder's catalog. -class DeferredTemporaryMeasureScope { +class DeferredTemporaryMeasureScope : public YardstickTemporaryDefinitionScope { public: explicit DeferredTemporaryMeasureScope(DeferredMeasureCatalogState &state) { yardstick_push_measure_view_overlay(); @@ -2558,6 +2634,12 @@ static BoundStatement BindDeferredMeasureStatement(ClientContext &context, Binde } static unique_ptr DeferredMeasureSelectBindReplace(ClientContext &context, TableFunctionBindInput &input) { + if (YardstickDefinitionStoreConfigured(context)) { + auto &info = input.info->Cast(); + input.binder->GetStatementProperties().always_require_rebind = true; + return YardstickDefinitionQueryRef(context, input.inputs[0].GetValue(), + YardstickContextParserOptions(context), info.parser_info.get()); + } auto &info = input.info->Cast(); NativeYardstickBindScope bind_scope(context); NativeYardstickParseScope parse_scope(info.parser_info.get(), YardstickContextParserOptions(context)); @@ -2594,6 +2676,29 @@ static unique_ptr DeferredMeasureSelectBindReplace(ClientContext &cont } #endif +unique_ptr EnterYardstickTemporaryDefinitions(ClientContext &context) { +#if YARDSTICK_GRAMMAR_EXTENSION + auto state = context.registered_state->GetOrCreate("yardstick_deferred_catalog"); + return make_uniq(*state); +#else + return nullptr; +#endif +} + +BoundStatement BindYardstickUnstoredStatement(ClientContext &context, Binder &parent, + const string &sql, const ParserOptions &options) { +#if YARDSTICK_GRAMMAR_EXTENSION + auto deferred_options = options; + if (auto current = CurrentNativeYardstickParserOptions()) deferred_options = *current; + return BindDeferredMeasureStatement(context, parent, YardstickDeferredParseData(sql, deferred_options)); +#else + Parser parser(options); + parser.ParseQuery(sql); + auto binder = Binder::CreateBinder(context, &parent); + return binder->Bind(*parser.statements[0]); +#endif +} + ParserExtensionPlanResult yardstick_plan(ParserExtensionInfo *, ClientContext &context, unique_ptr parse_data) { @@ -2605,6 +2710,17 @@ ParserExtensionPlanResult yardstick_plan(ParserExtensionInfo *, BoundStatement yardstick_bind(ClientContext &context, Binder &binder, OperatorExtensionInfo *info, SQLStatement &statement) { +#if !YARDSTICK_GRAMMAR_EXTENSION + if (statement.type != StatementType::EXTENSION_STATEMENT && + !YardstickDefinitionParsingBypassed() && YardstickDefinitionStoreConfigured(context)) { + auto sql = statement.query.empty() ? context.GetCurrentQuery() : statement.query; + if (statement.stmt_length > 0 && statement.stmt_location < sql.size()) { + sql = sql.substr(statement.stmt_location, statement.stmt_length); + } + return BindYardstickDefinitionStatement(context, binder, + YardstickDefinitionParseData(sql, context.GetParserOptions()), nullptr); + } +#endif switch (statement.type) { case StatementType::EXTENSION_STATEMENT: { auto &ext_statement = dynamic_cast(statement); @@ -2613,6 +2729,15 @@ BoundStatement yardstick_bind(ClientContext &context, Binder &binder, auto lookup = context.registered_state->Get("yardstick"); if (lookup) { auto state = (YardstickState *)lookup.get(); + if (auto *definition = dynamic_cast(state->parse_data.get())) { +#if YARDSTICK_GRAMMAR_EXTENSION + auto *operator_info = dynamic_cast(info); + return BindYardstickDefinitionStatement(context, binder, *definition, + operator_info ? operator_info->parser_info.get() : nullptr); +#else + return BindYardstickDefinitionStatement(context, binder, *definition, nullptr); +#endif + } #if YARDSTICK_GRAMMAR_EXTENSION if (auto *deferred = dynamic_cast(state->parse_data.get())) { return BindDeferredMeasureStatement(context, binder, *deferred); @@ -2684,6 +2809,21 @@ BoundStatement yardstick_bind(ClientContext &context, Binder &binder, // EXTENSION LOADING //============================================================================= +static unique_ptr StoredQueryBindReplace(ClientContext &context, TableFunctionBindInput &input) { + if (!YardstickDefinitionStoreConfigured(context)) return nullptr; + if (input.inputs.size() > 1) HandleAggregateWarnings(context, input.inputs[1].GetValue()); + if (input.binder) input.binder->GetStatementProperties().always_require_rebind = true; + ParserExtensionInfo *parser_info = nullptr; + auto options = YardstickBuiltinParserOptions(); +#if YARDSTICK_GRAMMAR_EXTENSION + parser_info = input.info->Cast().parser_info.get(); + options = YardstickContextParserOptions(context); +#else + options = context.GetParserOptions(); +#endif + return YardstickDefinitionQueryRef(context, input.inputs[0].GetValue(), options, parser_info); +} + static void RegisterQueryFunction(ExtensionLoader &loader, TableFunction function, vector parameter_names, const string &description, const string &example) { @@ -2741,6 +2881,7 @@ static void LoadInternal(ExtensionLoader &loader) { auto &db = loader.GetDatabaseInstance(); auto &config = DBConfig::GetConfig(db); + RegisterYardstickDefinitionStore(config); // Enable parser_override so yardstick intercepts queries before DuckDB's native parser. // FALLBACK mode: if our override doesn't handle the query, DuckDB's parser takes over. @@ -2767,6 +2908,7 @@ static void LoadInternal(ExtensionLoader &loader) { // Register table function for AGGREGATE() expansion TableFunction query_func("yardstick", {LogicalType::VARCHAR}, YardstickQueryFunction, YardstickQueryBind); + query_func.bind_replace = StoredQueryBindReplace; #if YARDSTICK_GRAMMAR_EXTENSION query_func.function_info = make_shared_ptr(parser.parser_info); #endif @@ -2775,6 +2917,7 @@ static void LoadInternal(ExtensionLoader &loader) { "SELECT * FROM yardstick('SELECT 1 AS value');"); TableFunction query_func_with_warnings("yardstick", {LogicalType::VARCHAR, LogicalType::VARCHAR}, YardstickQueryFunction, YardstickQueryBind); + query_func_with_warnings.bind_replace = StoredQueryBindReplace; #if YARDSTICK_GRAMMAR_EXTENSION query_func_with_warnings.function_info = make_shared_ptr(parser.parser_info); #endif diff --git a/src/yardstick_parser_ffi.cpp b/src/yardstick_parser_ffi.cpp index 740927c..e4bce16 100644 --- a/src/yardstick_parser_ffi.cpp +++ b/src/yardstick_parser_ffi.cpp @@ -13,6 +13,7 @@ #include "frontend_peg.hpp" #include "aggregate_decorations.hpp" #include "measure_windows.hpp" +#include "definition_store.hpp" #if YARDSTICK_GRAMMAR_EXTENSION #include "duckdb/parser/peg/compiled_grammar.hpp" #include "duckdb/parser/statement/insert_statement.hpp" @@ -1006,8 +1007,10 @@ static void CollectTablesFromTableRef(TableRef* ref, std::vector(ref); YardstickTableRef t {}; - t.table_name = safe_strdup(YsBaseTableName(*base)); + auto stored_name = duckdb::YardstickStoredRelationName(*base); + t.table_name = safe_strdup(stored_name.empty() ? YsBaseTableName(*base) : stored_name); t.alias = base->alias.empty() ? nullptr : safe_strdup(YsName(base->alias)); + if (!stored_name.empty() && !t.alias) t.alias = safe_strdup(YsBaseTableName(*base)); t.is_subquery = false; #if YARDSTICK_GRAMMAR_EXTENSION t.schema_qualified = !base->GetQualifiedName().Schema().empty(); diff --git a/test/python/test_definition_store.py b/test/python/test_definition_store.py new file mode 100644 index 0000000..5c58653 --- /dev/null +++ b/test/python/test_definition_store.py @@ -0,0 +1,168 @@ +"""Cross-process definition-store tests for either supported DuckDB CLI. + +Run with YARDSTICK_DUCKDB_CLI and YARDSTICK_EXTENSION set to built artifacts: + uv run test/python/test_definition_store.py +Set YARDSTICK_DUCKLAKE_EXTENSION to also exercise a real DuckLake attachment. +""" + +import csv +import io +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +def sql_string(value): + return "'" + str(value).replace("'", "''") + "'" + + +class DefinitionStoreTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cli = os.environ.get("YARDSTICK_DUCKDB_CLI") + extension = os.environ.get("YARDSTICK_EXTENSION") + if not cli or not extension: + raise unittest.SkipTest("Set YARDSTICK_DUCKDB_CLI and YARDSTICK_EXTENSION") + cls.cli = str(Path(cli).resolve()) + cls.extension = str(Path(extension).resolve()) + + def setUp(self): + self.scratch = tempfile.TemporaryDirectory(prefix="yardstick-definition-tests-") + self.addCleanup(self.scratch.cleanup) + self.store = Path(self.scratch.name) / "definitions.duckdb" + + def run_sql(self, sql, *, readonly=False, ducklake=False, expect_error=False): + args = [self.cli, "-unsigned", "-batch", "-csv", "-noheader", "-init", os.devnull] + args += ["-cmd", f"LOAD {sql_string(self.extension)}"] + if ducklake: + extension = os.environ.get("YARDSTICK_DUCKLAKE_EXTENSION") + if not extension: + self.skipTest("Set YARDSTICK_DUCKLAKE_EXTENSION") + args += ["-cmd", f"LOAD {sql_string(Path(extension).resolve())}"] + path = f"ducklake:{self.store}" if ducklake else str(self.store) + attach = f"ATTACH {sql_string(path)} AS store" + if readonly: + attach += " (READ_ONLY)" + args += ["-cmd", attach, "-c", sql] + environment = os.environ.copy() + environment.pop("CODEX_CI", None) # Keep the CLI's ordinary CSV output. + result = subprocess.run(args, text=True, capture_output=True, env=environment, timeout=60) + if expect_error: + self.assertNotEqual(result.returncode, 0, result.stdout) + return result.stderr + self.assertEqual(result.returncode, 0, result.stderr) + return list(csv.reader(io.StringIO(result.stdout))) + + def create_sales_view(self, *, ducklake=False): + self.run_sql( + "create table store.main.sales(region varchar, amount integer);" + "insert into store.main.sales values ('N', 10), ('N', 20), ('S', 40);" + "set yardstick_definition_table='store.main.definitions';" + "create view sales_v as select region, sum(amount) as measure revenue from store.main.sales;", + ducklake=ducklake, + ) + + def test_fresh_process_and_readonly_attachment(self): + self.create_sales_view() + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "select aggregate(revenue) from sales_v;" + "select count(*) from duckdb_views() where view_name='sales_v';", + readonly=True, + ) + self.assertEqual(rows, [["70"], ["0"]]) + + def test_rollback_and_committed_drop(self): + self.create_sales_view() + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "begin;" + "create or replace view sales_v as select region, sum(amount * 2) as measure revenue from store.main.sales;" + "select aggregate(revenue) from sales_v;" + "rollback;" + "select aggregate(revenue) from sales_v;" + "drop view sales_v;" + "select count(*) from store.main.definitions;" + ) + self.assertEqual(rows, [["140"], ["70"], ["0"]]) + error = self.run_sql( + "set yardstick_definition_table='store.main.definitions'; select aggregate(revenue) from sales_v;", + expect_error=True, + ) + self.assertIn("sales_v", error) + + def test_native_queries_do_not_scan_definitions(self): + # A scan would fail on the missing definition columns. Native-table and + # constant queries should not touch this deliberately unrelated table. + rows = self.run_sql( + "create table store.main.unrelated(value integer);" + "insert into store.main.unrelated values (41);" + "set yardstick_definition_table='store.main.unrelated';" + "select value + 1 from store.main.unrelated;" + "select 7;" + ) + self.assertEqual(rows, [["42"], ["7"]]) + + def test_explain_does_not_create_a_second_definition(self): + self.create_sales_view() + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "explain analyze create or replace view sales_v as select 17 as value;" + "select value from sales_v;" + "select count(*) from duckdb_views() where view_name='sales_v';" + ) + self.assertEqual(rows[-2:], [["17"], ["0"]]) + + def test_prepared_parameters_and_replacement(self): + self.create_sales_view() + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "prepare sales_total as select aggregate(revenue) from sales_v where region=$1;" + "execute sales_total('N');" + "create or replace view sales_v as " + "select region, sum(amount * 2) as measure revenue from store.main.sales;" + "execute sales_total('N');" + "drop view sales_v;" + "select count(*) from store.main.definitions;" + ) + self.assertEqual(rows, [["30"], ["60"], ["0"]]) + + def test_source_binding_survives_schema_changes(self): + rows = self.run_sql( + "create schema original; create schema other;" + "create table original.sales as select 7 amount;" + "create table other.sales as select 99 amount;" + "set schema='original';" + "set yardstick_definition_table='store.main.definitions';" + "create view main.sales_v as select sum(amount) as measure total from sales;" + "set schema='other';" + "select aggregate(total) from main.sales_v;" + ) + self.assertEqual(rows, [["7"]]) + rows = self.run_sql( + "create schema other; create table other.sales as select 99 amount;" + "set schema='other'; set yardstick_definition_table='store.main.definitions';" + "select aggregate(total) from main.sales_v;", + expect_error=True, + ) + # The definition must retain its original source, which belonged to + # the previous in-memory database, rather than silently reading 99. + self.assertIn("original", rows) + + def test_ducklake_lifecycle_and_reopen(self): + self.create_sales_view(ducklake=True) + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "select aggregate(revenue) from sales_v;" + "create or replace view sales_v as select region, sum(amount * 2) as measure revenue from store.main.sales;" + "select aggregate(revenue) from sales_v;" + "drop view sales_v; select count(*) from store.main.definitions;", + ducklake=True, + ) + self.assertEqual(rows, [["70"], ["140"], ["0"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/sql/definition_store.test b/test/sql/definition_store.test new file mode 100644 index 0000000..804182a --- /dev/null +++ b/test/sql/definition_store.test @@ -0,0 +1,309 @@ +# name: test/sql/definition_store.test +# description: A configurable attachment table is the sole source of semantic view definitions. +# group: [yardstick] + +require yardstick + +statement ok +ATTACH ':memory:' AS definition_store; + +statement ok +ATTACH ':memory:' AS other_definition_store; + +statement ok +CREATE TABLE definition_sales(region VARCHAR, amount INTEGER); + +statement ok +INSERT INTO definition_sales VALUES ('N', 10), ('N', 20), ('S', 40); + +statement ok +SET yardstick_definition_table='definition_store.main.views'; + +# Selecting an empty store does not stop ordinary SQL from working. +query I +SELECT 42; +---- +42 + +statement ok +CREATE VIEW stored_sales AS SELECT region, SUM(amount) AS MEASURE revenue FROM definition_sales; + +query I +SELECT count(*) FROM definition_store.main.views WHERE view_name='stored_sales'; +---- +1 + +# No second native view definition is written to any catalog. +query I +SELECT count(*) FROM duckdb_views() WHERE view_name='stored_sales'; +---- +0 + +query TI rowsort +SELECT region, AGGREGATE(revenue) FROM stored_sales; +---- +N 30 +S 40 + +query TI rowsort +SELECT region, revenue FROM stored_sales; +---- +N 30 +S 40 + +query I +SELECT AGGREGATE(revenue) AT (ALL) FROM stored_sales; +---- +70 + +statement error +CREATE VIEW stored_sales AS SELECT SUM(amount) AS MEASURE revenue FROM definition_sales; +---- +already exists + +statement ok +CREATE VIEW IF NOT EXISTS stored_sales AS SELECT SUM(missing) AS MEASURE ignored FROM missing_table; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +70 + +statement ok +CREATE OR REPLACE VIEW stored_sales AS SELECT region, SUM(amount * 2) AS MEASURE revenue FROM definition_sales; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +# A prepared query must re-read definition rows after replacements. +statement ok +PREPARE stored_query AS SELECT AGGREGATE(revenue) FROM stored_sales; + +query I +EXECUTE stored_query; +---- +140 + +statement ok +CREATE OR REPLACE VIEW stored_sales AS SELECT region, SUM(amount * 4) AS MEASURE revenue FROM definition_sales; + +query I +EXECUTE stored_query; +---- +280 + +statement ok +CREATE OR REPLACE VIEW stored_sales AS SELECT region, SUM(amount * 2) AS MEASURE revenue FROM definition_sales; + +statement ok +PREPARE stored_parameter AS SELECT AGGREGATE(revenue) FROM stored_sales WHERE region=$1; + +query I +EXECUTE stored_parameter('N'); +---- +60 + +query I +SELECT * FROM yardstick('SELECT AGGREGATE(revenue) FROM stored_sales'); +---- +140 + +statement error +CREATE OR REPLACE VIEW stored_sales AS SELECT region, SUM(missing) AS MEASURE revenue FROM definition_sales; +---- +missing + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +# Metadata DML and reads use the caller's transaction. +statement ok +BEGIN; + +statement ok +CREATE OR REPLACE VIEW stored_sales AS SELECT region, SUM(amount * 3) AS MEASURE revenue FROM definition_sales; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +210 + +statement ok +ROLLBACK; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +statement ok +BEGIN; + +statement ok +DROP VIEW stored_sales; + +query I +SELECT count(*) FROM definition_store.main.views; +---- +0 + +statement ok +ROLLBACK; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +# A different connection resolves the same store without any import call. +statement ok reader +SET yardstick_definition_table='definition_store.main.views'; + +query I reader +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +statement ok +SET yardstick_definition_table='other_definition_store.main.views'; + +statement ok +CREATE VIEW stored_sales AS SELECT SUM(amount * 5) AS MEASURE revenue FROM definition_sales; + +query I +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +350 + +query I reader +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +statement ok +DROP VIEW stored_sales; + +statement error +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +stored_sales + +query I reader +SELECT AGGREGATE(revenue) FROM stored_sales; +---- +140 + +statement ok +SET yardstick_definition_table='definition_store.main.views'; + +statement ok +CREATE SCHEMA definition_a; + +statement ok +CREATE SCHEMA definition_b; + +statement ok +/* Keep the declaration in the selected table even with a leading comment. */ +CREATE VIEW definition_a.same_name AS SELECT SUM(amount) AS MEASURE total FROM definition_sales; + +statement ok +CREATE VIEW definition_b.same_name AS SELECT SUM(amount * 3) AS MEASURE total FROM definition_sales; + +query I +SELECT AGGREGATE(total) FROM definition_a.same_name; +---- +70 + +query I +SELECT AGGREGATE(total) FROM definition_b.same_name; +---- +210 + +statement ok +CREATE VIEW "stored.dot""quote" AS +SELECT region AS area, SUM(amount) AS MEASURE total FROM definition_sales; + +query TI rowsort +SELECT area, AGGREGATE(total) FROM "stored.dot""quote"; +---- +N 30 +S 40 + +statement ok +DROP VIEW "stored.dot""quote"; + +statement ok +DROP VIEW definition_a.same_name; + +statement ok +DROP VIEW definition_b.same_name; + +statement ok +CREATE VIEW stored_derived AS +SELECT region, SUM(amount) AS MEASURE revenue, + COUNT(*) AS MEASURE n, revenue / n AS MEASURE average +FROM definition_sales; + +query TR rowsort +SELECT region, AGGREGATE(average) FROM stored_derived; +---- +N 15.0 +S 40.0 + +statement ok +DROP VIEW stored_derived; + +# Ordinary replacement of a managed semantic view stays in the same store. +statement ok +CREATE OR REPLACE VIEW stored_sales AS SELECT 17 AS value; + +query I +SELECT value FROM stored_sales; +---- +17 + +statement ok +DROP VIEW stored_sales; + +query I +SELECT count(*) FROM definition_store.main.views; +---- +0 + +statement ok +DROP VIEW IF EXISTS stored_sales; + +statement error +SET yardstick_definition_table='main.views'; +---- +catalog.schema.table + +statement ok +RESET yardstick_definition_table; + +# The same table-backed path can be entirely session-local. +statement ok +SET yardstick_definition_table='temp.main.session_views'; + +statement ok +CREATE VIEW session_only AS SELECT SUM(amount) AS MEASURE total FROM definition_sales; + +query I +SELECT AGGREGATE(total) FROM session_only; +---- +70 + +query I +SELECT count(*) FROM temp.main.session_views; +---- +1 + +statement ok +DROP VIEW session_only; + +statement ok +RESET yardstick_definition_table; diff --git a/test/sql/native_definition_store.test b/test/sql/native_definition_store.test new file mode 100644 index 0000000..31cc20a --- /dev/null +++ b/test/sql/native_definition_store.test @@ -0,0 +1,67 @@ +# name: test/sql/native_definition_store.test +# description: Native header aliases, windows and temporary views with table-backed definitions. +# group: [yardstick] + +require-env YARDSTICK_NATIVE_PEG 1 + +require yardstick + +statement ok +ATTACH ':memory:' AS native_definition_store; + +statement ok +CREATE TABLE native_definition_sales(region VARCHAR, amount INTEGER); + +statement ok +INSERT INTO native_definition_sales VALUES ('N', 10), ('N', 20), ('S', 40); + +statement ok +SET yardstick_definition_table='native_definition_store.main.views'; + +statement ok +CREATE VIEW stored_header(area, total) AS +SELECT region, SUM(amount) AS MEASURE original_total FROM native_definition_sales; + +query TI rowsort +SELECT area, AGGREGATE(total) FROM stored_header; +---- +N 30 +S 40 + +statement ok +CREATE TEMP VIEW stored_temporary(r, total) AS +SELECT region, SUM(amount) AS MEASURE revenue FROM native_definition_sales; + +query TI rowsort +SELECT r, AGGREGATE(total) FROM stored_temporary; +---- +N 30 +S 40 + +query I +SELECT count(*) FROM native_definition_store.main.views WHERE view_name='stored_temporary'; +---- +0 + +statement ok +DROP VIEW stored_temporary; + +statement ok +CREATE TEMP VIEW stored_temporary AS SELECT 17 AS total; + +query I +SELECT total FROM stored_temporary; +---- +17 + +statement ok +DROP VIEW stored_temporary; + +statement ok +CREATE VIEW stored_window AS SELECT region, amount, +SUM(amount) OVER () AS MEASURE total FROM native_definition_sales; + +query I +SELECT AGGREGATE(total) AT (ALL) FROM stored_window; +---- +70 diff --git a/yardstick-rs/src/ffi.rs b/yardstick-rs/src/ffi.rs index 2b0800e..cdd623b 100644 --- a/yardstick-rs/src/ffi.rs +++ b/yardstick-rs/src/ffi.rs @@ -202,6 +202,30 @@ pub extern "C" fn yardstick_pop_measure_view_overlay() { pop_measure_view_overlay(); } +#[no_mangle] +pub extern "C" fn yardstick_push_measure_view_namespace() { + crate::sql::measures::push_measure_view_namespace(); +} + +#[no_mangle] +pub extern "C" fn yardstick_pop_measure_view_namespace() { + crate::sql::measures::pop_measure_view_namespace(); +} + +#[no_mangle] +pub extern "C" fn yardstick_rename_measure_view(old_name: *const c_char, new_name: *const c_char) { + if old_name.is_null() || new_name.is_null() { + return; + } + let (Ok(old_name), Ok(new_name)) = ( + unsafe { CStr::from_ptr(old_name) }.to_str(), + unsafe { CStr::from_ptr(new_name) }.to_str(), + ) else { + return; + }; + crate::sql::measures::rename_measure_view(old_name, new_name); +} + #[no_mangle] pub extern "C" fn yardstick_set_measure_view_overlay(name: *const c_char, snapshot: *const c_void) { if name.is_null() { diff --git a/yardstick-rs/src/sql/measures.rs b/yardstick-rs/src/sql/measures.rs index e0ab46e..c68a985 100644 --- a/yardstick-rs/src/sql/measures.rs +++ b/yardstick-rs/src/sql/measures.rs @@ -64,10 +64,30 @@ enum MeasureOverlayEntry { } thread_local! { + // Configured definition stores hydrate only the current binding. Never + // publish these entries to the legacy, process-global namespace. + static MEASURE_VIEW_NAMESPACES: RefCell>> = + const { RefCell::new(Vec::new()) }; static MEASURE_VIEW_OVERLAYS: RefCell>> = const { RefCell::new(Vec::new()) }; } +pub fn push_measure_view_namespace() { + MEASURE_VIEW_NAMESPACES.with(|namespaces| namespaces.borrow_mut().push(HashMap::new())); +} + +pub fn pop_measure_view_namespace() { + MEASURE_VIEW_NAMESPACES.with(|namespaces| { namespaces.borrow_mut().pop(); }); +} + +pub fn rename_measure_view(old_name: &str, new_name: &str) { + if let Some(mut view) = get_measure_view(old_name) { + write_measure_view(old_name, None); + view.view_name = new_name.to_string(); + write_measure_view(new_name, Some(view)); + } +} + pub fn push_measure_view_overlay() { MEASURE_VIEW_OVERLAYS.with(|overlays| overlays.borrow_mut().push(HashMap::new())); } @@ -95,7 +115,8 @@ pub fn bypass_measure_view_overlay(name: &str) { // Return owned data: callers may invoke parser callbacks, including nested // lookups, without holding either the global mutex or a thread-local borrow. fn measure_views_snapshot() -> HashMap { - let permanent = MEASURE_VIEWS.lock().unwrap().clone(); + let permanent = MEASURE_VIEW_NAMESPACES.with(|namespaces| namespaces.borrow().last().cloned()) + .unwrap_or_else(|| MEASURE_VIEWS.lock().unwrap().clone()); if MEASURE_VIEW_OVERLAYS.with(|overlays| overlays.borrow().iter().all(HashMap::is_empty)) { return permanent; } @@ -131,6 +152,18 @@ fn write_measure_view(name: &str, view: Option) { false }); if !scoped { + let namespaced = MEASURE_VIEW_NAMESPACES.with(|namespaces| { + let mut namespaces = namespaces.borrow_mut(); + let Some(views) = namespaces.last_mut() else { return false; }; + remove_measure_view_case_insensitive(views, name); + if let Some(view) = view.clone() { + views.insert(view.view_name.clone(), view); + } + true + }); + if namespaced { + return; + } let mut views = MEASURE_VIEWS.lock().unwrap(); remove_measure_view_case_insensitive(&mut views, name); if let Some(view) = view { @@ -5248,9 +5281,10 @@ fn find_expression_start(sql: &str, end: usize) -> usize { } } - // Find actual start by looking for comma - let byte_pos: usize = chars[..pos].iter().collect::().len(); - sql[..byte_pos].rfind(',').map(|p| p + 1).unwrap_or(0) + // The scan stopped after SELECT or a projection separator. Searching for + // an earlier comma loses the first projection (and can include CREATE VIEW + // in its expression when there is no preceding comma). + chars[..pos].iter().collect::().len() } // ============================================================================= @@ -8994,6 +9028,40 @@ mod tests { } } + #[test] + #[serial] + fn test_definition_namespaces_are_isolated_and_nested() { + let name = "definition_namespace_view"; + restore_measure_view(overlay_test_view(name)); + push_measure_view_namespace(); + assert!(get_measure_view(name).is_none()); + let mut local = overlay_test_view(name); + local.measures[0].expression = "SUM(amount * 2)".to_string(); + restore_measure_view(local); + std::thread::spawn(move || { + assert_eq!(get_measure_view(name).unwrap().measures[0].expression, "SUM(amount)"); + }).join().unwrap(); + push_measure_view_namespace(); + assert!(get_measure_view(name).is_none()); + pop_measure_view_namespace(); + assert_eq!(get_measure_view(name).unwrap().measures[0].expression, "SUM(amount * 2)"); + pop_measure_view_namespace(); + assert_eq!(get_measure_view(name).unwrap().measures[0].expression, "SUM(amount)"); + drop_measure_view(name); + } + + #[test] + #[serial] + fn test_first_projection_measure_preserves_create_header() { + let result = process_create_view( + "CREATE VIEW first_projection AS SELECT SUM(amount) AS MEASURE total FROM source", + ); + assert!(result.error.is_none()); + assert_eq!(result.measures[0].expression, "SUM(amount)"); + assert!(result.clean_sql.starts_with("CREATE VIEW first_projection AS SELECT NULL AS total")); + drop_measure_view("first_projection"); + } + #[test] #[serial] fn test_temporary_overlay_isolates_threads_and_preserves_permanent_updates() { From 123ce7b816df509134c2fa591c8d865ae416427d Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Wed, 11 Nov 2026 20:28:54 -0800 Subject: [PATCH 2/3] Fix definition storage shadowing, scopes, and concurrent writes --- README.md | 17 ++ src/definition_store.cpp | 306 +++++++++++++++++++++------ test/python/test_definition_store.py | 91 +++++++- test/sql/definition_store.test | 117 +++++++++- 4 files changed, 455 insertions(+), 76 deletions(-) diff --git a/README.md b/README.md index 4b93314..7deb0cd 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,9 @@ The setting selects the **attachment, schema, and table**. The schema must alrea exist; Yardstick creates the definitions table on the first permanent measure-view declaration. It uses four `VARCHAR` columns: `view_catalog`, `view_schema`, `view_name`, and `definition_sql`. +One reserved row has empty catalog, schema, and view names and holds the +transaction concurrency guard. To list only view definitions, filter with +`WHERE view_name <> ''`. The selected table is the sole source of truth. `CREATE VIEW ... AS MEASURE` inserts a definition, `CREATE OR REPLACE VIEW` updates it, and `DROP VIEW` removes @@ -165,6 +168,20 @@ in the selected table. These writes use ordinary attachment table operations in the caller's transaction, so `ROLLBACK` also rolls back definition changes. Source data is never copied or materialized. +Definition writes update and verify that guard in the same transaction before +changing a view. Competing writers can receive a transaction conflict and should +retry in a fresh transaction. This works without requiring UNIQUE constraints +from the attachment. Direct external edits to the table bypass this protection. + +An existing definitions table without a guard remains readable. Before writing, +initialize its guard once with exclusive writer access: + +```sql +INSERT INTO lake.semantic.yardstick_views + (view_catalog, view_schema, view_name, definition_sql) +VALUES ('', '', '', 'yardstick_guard_v1:' || uuid()::VARCHAR); +``` + In a later session, attach the same databases and set the same `yardstick_definition_table`; queries resolve the stored definitions automatically. The setting is session-local. Reattach source catalogs under the same names: diff --git a/src/definition_store.cpp b/src/definition_store.cpp index 15890b8..15784d2 100644 --- a/src/definition_store.cpp +++ b/src/definition_store.cpp @@ -9,6 +9,7 @@ #include "duckdb/catalog/catalog_search_path.hpp" #include "duckdb/catalog/entry_lookup_info.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/types/uuid.hpp" #include "duckdb/execution/executor.hpp" #include "duckdb/execution/physical_plan_generator.hpp" #include "duckdb/function/replacement_scan.hpp" @@ -25,11 +26,14 @@ #include "duckdb/parser/expression/subquery_expression.hpp" #include "duckdb/parser/expression/star_expression.hpp" #include "duckdb/parser/query_node/select_node.hpp" +#include "duckdb/parser/query_node/recursive_cte_node.hpp" +#include "duckdb/parser/query_node/set_operation_node.hpp" #include "duckdb/parser/statement/create_statement.hpp" #include "duckdb/parser/statement/drop_statement.hpp" #include "duckdb/parser/statement/explain_statement.hpp" #include "duckdb/parser/statement/select_statement.hpp" #include "duckdb/parser/tableref/basetableref.hpp" +#include "duckdb/parser/tableref/list.hpp" #include "duckdb/parser/tableref/subqueryref.hpp" #include "duckdb/planner/binder.hpp" #include "duckdb/planner/operator/logical_extension_operator.hpp" @@ -58,6 +62,8 @@ void yardstick_free(char *ptr); namespace duckdb { namespace { constexpr const char *DEFINITION_SETTING = "yardstick_definition_table"; +constexpr const char *GUARD_PREDICATE = "view_catalog = '' AND view_schema = '' AND view_name = ''"; +constexpr const char *GUARD_TOKEN_PREFIX = "yardstick_guard_v1:"; std::mutex routing_lock; std::unordered_map routing_sessions; thread_local idx_t parsing_bypass = 0; @@ -211,29 +217,6 @@ class DefinitionRowsSink : public PhysicalOperator { } }; -// Keep CREATE/REPLACE/DROP's empty success result instead of exposing the -// implementation's INSERT/UPDATE/DELETE row count to callers. -class LogicalDefinitionWrite : public LogicalExtensionOperator { -public: - explicit LogicalDefinitionWrite(unique_ptr write) { - children.push_back(std::move(write)); - } - string GetExtensionName() const override { return "yardstick"; } - bool SupportSerialization() const override { return false; } - vector GetColumnBindings() override { -#if YARDSTICK_NEW_EXPR_API - return {ColumnBinding(TableIndex(0), ProjectionIndex(0))}; -#else - return {ColumnBinding(0, 0)}; -#endif - } - void ResolveTypes() override { types = {LogicalType::BOOLEAN}; } - PhysicalOperator &CreatePlan(ClientContext &, PhysicalPlanGenerator &planner) override { - auto &write = planner.CreatePlan(*children[0]); - return planner.Make(write, nullptr); - } -}; - struct DefinitionProfilerScope { explicit DefinitionProfilerScope(ClientContext &context) : data(ClientData::Get(context)), previous(data.profiler) { @@ -244,12 +227,10 @@ struct DefinitionProfilerScope { shared_ptr previous; }; -vector> ReadDefinitions(ClientContext &context, const string &table) { - auto statements = Parse("SELECT view_catalog, view_schema, view_name, definition_sql FROM " + table, - YardstickBuiltinParserOptions()); - auto &select = statements[0]->Cast().node->Cast(); - if (!FindNativeEntry(context, GetName(select.from_table->Cast()))) return {}; +vector> ExecuteDefinitionSQL(ClientContext &context, const string &sql) { + ParsingBypass bypass; DefinitionProfilerScope profiler(context); + auto statements = Parse(sql, YardstickBuiltinParserOptions()); auto binder = Binder::CreateBinder(context); auto bound = binder->Bind(*statements[0]); PhysicalPlanGenerator generator(context); @@ -276,6 +257,86 @@ vector> ReadDefinitions(ClientContext &context, const string &tabl return rows; } +vector> ReadDefinitions(ClientContext &context, const string &table) { + auto sql = "SELECT view_catalog, view_schema, view_name, definition_sql FROM " + table; + auto statements = Parse(sql, YardstickBuiltinParserOptions()); + auto &select = statements[0]->Cast().node->Cast(); + if (!FindNativeEntry(context, GetName(select.from_table->Cast()))) return {}; + return ExecuteDefinitionSQL(context, sql); +} + +struct DefinitionWriteState : public GlobalSourceState { + bool finished = false; +}; + +// The guard and mutation must execute in order within one statement's +// transaction. Separate statements would autocommit between these operations. +class DefinitionWriteSource : public PhysicalOperator { +public: + DefinitionWriteSource(PhysicalPlan &plan, string guard_sql, string mutation_sql, string table, string token) + : PhysicalOperator(plan, PhysicalOperatorType::EXTENSION, {LogicalType::BOOLEAN}, 1), + guard_sql(std::move(guard_sql)), mutation_sql(std::move(mutation_sql)), + table(std::move(table)), token(std::move(token)) {} + + string guard_sql; + string mutation_sql; + string table; + string token; + + bool IsSource() const override { return true; } + unique_ptr GetGlobalSourceState(ClientContext &) const override { + return make_uniq(); + } + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &, OperatorSourceInput &input) const override { + auto &state = input.global_state.Cast(); + if (state.finished) return SourceResultType::FINISHED; + state.finished = true; + ExecuteDefinitionSQL(context.client, guard_sql); + // Rebind after UPDATE: lake scans can capture transaction-local file + // lists at bind time. A prebound scan may miss our new guard row. + auto rows = ExecuteDefinitionSQL(context.client, + "SELECT definition_sql FROM " + table + " WHERE " + GUARD_PREDICATE); + if (rows.size() != 1 || rows[0][0].IsNull() || rows[0][0].GetValue() != token) { + throw TransactionException("Concurrent Yardstick definition change; retry in a fresh transaction"); + } + // Bind again after acquiring the guard so attachments that rewrite + // files on UPDATE do not leave the mutation with stale row identities. + ExecuteDefinitionSQL(context.client, mutation_sql); + return SourceResultType::FINISHED; + } +}; + +// Keep CREATE/REPLACE/DROP's empty success result instead of exposing the +// implementation's INSERT/UPDATE/DELETE row count to callers. +class LogicalDefinitionWrite : public LogicalExtensionOperator { +public: + explicit LogicalDefinitionWrite(unique_ptr write) { + children.push_back(std::move(write)); + } + string guard_sql; + string mutation_sql; + string table; + string token; + + string GetExtensionName() const override { return "yardstick"; } + bool SupportSerialization() const override { return false; } + vector GetColumnBindings() override { +#if YARDSTICK_NEW_EXPR_API + return {ColumnBinding(TableIndex(0), ProjectionIndex(0))}; +#else + return {ColumnBinding(0, 0)}; +#endif + } + void ResolveTypes() override { types = {LogicalType::BOOLEAN}; } + PhysicalOperator &CreatePlan(ClientContext &, PhysicalPlanGenerator &planner) override { + if (!guard_sql.empty()) { + return planner.Make(guard_sql, mutation_sql, table, token); + } + auto &write = planner.CreatePlan(*children[0]); + return planner.Make(write, nullptr); + } +}; + struct Definition { ViewName name; string sql; @@ -297,6 +358,15 @@ class DefinitionScope { throw InvalidInputException("Yardstick definitions require non-NULL VARCHAR columns"); } } + if (row[0].GetValue().empty() && row[1].GetValue().empty() && + row[2].GetValue().empty()) { + auto token = row[3].GetValue(); + if (!guard_token.empty() || !StringUtil::StartsWith(token, GUARD_TOKEN_PREFIX)) { + throw InvalidInputException("Invalid or duplicate Yardstick concurrency guard in %s", table); + } + guard_token = std::move(token); + continue; + } Definition definition {{row[0].GetValue(), row[1].GetValue(), row[2].GetValue()}, row[3].GetValue(), "__yardstick_definition_" + std::to_string(definitions.size()), ""}; for (auto &existing : definitions) { @@ -316,6 +386,7 @@ class DefinitionScope { ClientContext &context; ParserOptions options; + string guard_token; vector definitions; DefinitionScope *previous; @@ -458,9 +529,28 @@ BoundStatement BindSQL(ClientContext &context, Binder &parent, const string &sql return binder->Bind(*statements[0]); } -BoundStatement BindDefinitionWrite(ClientContext &context, Binder &parent, const string &sql) { +BoundStatement BindDefinitionWrite(ClientContext &context, Binder &parent, const string &sql, + const string &table = "", const string &guard_token = "") { auto bound = BindSQL(context, parent, sql, YardstickBuiltinParserOptions()); - bound.plan = make_uniq(std::move(bound.plan)); + auto write = make_uniq(std::move(bound.plan)); + if (!table.empty()) { + if (guard_token.empty()) { + throw InvalidInputException( + "Yardstick definitions table %s has no concurrency guard. With exclusive writer access, run: " + "INSERT INTO %s (view_catalog, view_schema, view_name, definition_sql) " + "VALUES ('', '', '', 'yardstick_guard_v1:' || uuid()::VARCHAR)", table, table); + } + write->table = table; + write->token = GUARD_TOKEN_PREFIX + UUID::ToString(UUID::GenerateRandomUUID()); + write->mutation_sql = sql; + write->guard_sql = "UPDATE " + table + " SET definition_sql = " + QuoteString(write->token) + + " WHERE " + GUARD_PREDICATE + " AND definition_sql = " + QuoteString(guard_token); + auto guard = BindSQL(context, parent, write->guard_sql, YardstickBuiltinParserOptions()); + // Binding both commands here validates them and records their modified + // database in the caller's statement properties before execution starts. + write->children.push_back(std::move(guard.plan)); + } + bound.plan = std::move(write); bound.names = {"Success"}; bound.types = {LogicalType::BOOLEAN}; parent.GetStatementProperties().return_type = StatementReturnType::NOTHING; @@ -550,33 +640,112 @@ bool NeedsDefinitionLookup(ClientContext &context, const string &sql, const Pars return needs_definitions; } +// Walk query boundaries explicitly: the engine's general iterator recursively +// flattens nested queries and CTE bodies, losing lexical CTE visibility. +class DefinitionSourceVisitor { +public: + using CteNames = std::set; + using Callback = std::function; + + explicit DefinitionSourceVisitor(Callback callback) : callback(std::move(callback)) {} + + void Query(QueryNode &node, CteNames visible = {}) { + for (auto &entry : node.cte_map.map) { +#if YARDSTICK_NEW_EXPR_API + Query(*entry.second->query_node, visible); +#else + Query(*entry.second->query->node, visible); +#endif + // A nonrecursive CTE sees preceding CTEs, not itself. The main + // query and later CTEs can see this declaration. + visible.insert(StringUtil::Lower(NameString(entry.first))); + } + switch (node.type) { + case QueryNodeType::SELECT_NODE: { + auto &select = node.Cast(); + for (auto &expression : select.select_list) Expression(expression, visible); + for (auto &expression : select.groups.group_expressions) Expression(expression, visible); + Expression(select.where_clause, visible); + Expression(select.having, visible); + Expression(select.qualify, visible); + if (select.from_table) Table(*select.from_table, visible); + break; + } + case QueryNodeType::SET_OPERATION_NODE: + for (auto &child : node.Cast().children) Query(*child, visible); + break; + case QueryNodeType::RECURSIVE_CTE_NODE: { + auto &recursive = node.Cast(); + Query(*recursive.left, visible); + visible.insert(StringUtil::Lower(NameString(recursive.ctename))); + Query(*recursive.right, visible); + for (auto &expression : recursive.key_targets) Expression(expression, visible); + break; + } + default: + throw NotImplementedException("Unsupported query in Yardstick definition"); + } + ParsedExpressionIterator::EnumerateQueryNodeModifiers(node, + [&](unique_ptr &expression) { Expression(expression, visible); }); + } + +private: + Callback callback; + + void Expression(unique_ptr &expression, const CteNames &visible) { + if (!expression) return; + if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { + Query(SubqueryNode(expression->Cast()), visible); + } + ParsedExpressionIterator::EnumerateChildren(*expression, + [&](unique_ptr &child) { Expression(child, visible); }); + } + + void Table(TableRef &ref, const CteNames &visible) { + switch (ref.type) { + case TableReferenceType::BASE_TABLE: + callback(ref.Cast(), visible); + break; + case TableReferenceType::SUBQUERY: + Query(*ref.Cast().subquery->node, visible); + break; + case TableReferenceType::JOIN: { + auto &join = ref.Cast(); + Table(*join.left, visible); + Table(*join.right, visible); + Expression(join.condition, visible); + break; + } + case TableReferenceType::PIVOT: { + auto &pivot = ref.Cast(); + Table(*pivot.source, visible); + for (auto &aggregate : pivot.aggregates) Expression(aggregate, visible); + break; + } + case TableReferenceType::EXPRESSION_LIST: + for (auto &row : ref.Cast().values) + for (auto &expression : row) Expression(expression, visible); + break; + case TableReferenceType::TABLE_FUNCTION: { + auto &function = ref.Cast(); + Expression(function.function, visible); + if (function.subquery) Query(*function.subquery->node, visible); + break; + } + case TableReferenceType::EMPTY_FROM: + break; + default: + throw NotImplementedException("Unsupported table reference in Yardstick definition"); + } + } +}; + // Persist resolved source names, so USE/search_path changes in a later session // cannot redirect a stored measure to an unrelated table. Keep declaration SQL // intact, including AS MEASURE, comments, and header column aliases. string QualifyDefinitionSources(ClientContext &context, DefinitionScope &definitions, const string &sql, SelectStatement &query, bool temporary_store) { vector>> replacements; - std::set cte_names; - std::function collect_ctes; - std::function &)> collect_expression; - collect_expression = [&](unique_ptr &expression) { - if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { - collect_ctes(SubqueryNode(expression->Cast())); - } - ParsedExpressionIterator::EnumerateChildren(*expression, collect_expression); - }; - collect_ctes = [&](QueryNode &node) { - for (auto &entry : node.cte_map.map) { - cte_names.insert(StringUtil::Lower(NameString(entry.first))); -#if YARDSTICK_NEW_EXPR_API - collect_ctes(*entry.second->query_node); -#else - collect_ctes(*entry.second->query->node); -#endif - } - ParsedExpressionIterator::EnumerateQueryNodeChildren(node, collect_expression); - }; - collect_ctes(*query.node); auto identifier_end = [&](idx_t start) { if (sql[start] == '"') { auto end = start + 1; @@ -596,18 +765,9 @@ string QualifyDefinitionSources(ClientContext &context, DefinitionScope &definit return end; }; auto tokens = Parser::Tokenize(sql); - std::function &)> visit_expression; - std::function visit_ref; - visit_expression = [&](unique_ptr &expression) { - if (expression->GetExpressionClass() == ExpressionClass::SUBQUERY) { - ParsedExpressionIterator::EnumerateQueryNodeChildren( - SubqueryNode(expression->Cast()), visit_expression, visit_ref); - } - ParsedExpressionIterator::EnumerateChildren(*expression, visit_expression); - }; - visit_ref = [&](TableRef &ref) { - if (ref.type != TableReferenceType::BASE_TABLE || !ref.query_location.IsValid()) return; - auto input = GetName(ref.Cast()); + DefinitionSourceVisitor visitor([&](BaseTableRef &ref, const DefinitionSourceVisitor::CteNames &cte_names) { + if (!ref.query_location.IsValid()) return; + auto input = GetName(ref); if (input.catalog.empty() && input.schema.empty() && cte_names.count(StringUtil::Lower(input.name))) return; ViewName resolved; if (auto entry = FindNativeEntry(context, input)) { @@ -638,8 +798,8 @@ string QualifyDefinitionSources(ClientContext &context, DefinitionScope &definit next++; } replacements.push_back({start, {end, resolved.SQL()}}); - }; - ParsedExpressionIterator::EnumerateQueryNodeChildren(*query.node, visit_expression, visit_ref); + }); + visitor.Query(*query.node); auto result = sql; std::sort(replacements.begin(), replacements.end(), [](const auto &a, const auto &b) { return a.first > b.first; }); for (auto &replacement : replacements) { @@ -730,6 +890,7 @@ void RegisterYardstickDefinitionStore(DBConfig &config) { auto &drop = target->Cast().info->Cast(); if (drop.type != CatalogType::VIEW_ENTRY) return; name = GetName(drop); + if (FindNativeEntry(input.context, name)) return; sql = drop.ToString(); } else { return; @@ -771,7 +932,8 @@ BoundStatement BindYardstickDefinitionStatement(ClientContext &context, Binder & auto create = StringUtil::StartsWith(StringUtil::Lower(table), "\"temp\".") ? "CREATE TEMP TABLE IF NOT EXISTS " : "CREATE TABLE IF NOT EXISTS "; return BindSQL(context, parent, create + table + - " (view_catalog VARCHAR, view_schema VARCHAR, view_name VARCHAR, definition_sql VARCHAR)", + " AS SELECT ''::VARCHAR AS view_catalog, ''::VARCHAR AS view_schema, ''::VARCHAR AS view_name, " + "'yardstick_guard_v1:' || uuid()::VARCHAR AS definition_sql", YardstickBuiltinParserOptions()); } if (FirstKeyword(data.sql) == "EXPLAIN") { @@ -871,15 +1033,21 @@ BoundStatement BindYardstickDefinitionStatement(ClientContext &context, Binder & ? "UPDATE " + table + " SET definition_sql = " + QuoteString(definition_sql) + " WHERE " + KeyPredicate(name) : "INSERT INTO " + table + " (view_catalog, view_schema, view_name, definition_sql) VALUES (" + QuoteString(name.catalog) + ", " + QuoteString(name.schema) + ", " + QuoteString(name.name) + ", " + QuoteString(definition_sql) + ")"; - return BindDefinitionWrite(context, parent, sql); + return BindDefinitionWrite(context, parent, sql, table, definitions.guard_token); } if (statement.type == StatementType::DROP_STATEMENT) { auto &drop = *statement.Cast().info; if (drop.type == CatalogType::VIEW_ENTRY) { - auto definition = definitions.Find(GetName(drop)); + auto name = GetName(drop); + // A native temporary view can shadow a stored permanent view. + // Match DuckDB's target precedence before deleting a durable row. + if (FindNativeEntry(context, name)) { + return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); + } + auto definition = definitions.Find(name); if (definition) { return BindDefinitionWrite(context, parent, - "DELETE FROM " + table + " WHERE " + KeyPredicate(definition->name)); + "DELETE FROM " + table + " WHERE " + KeyPredicate(definition->name), table, definitions.guard_token); } return BindYardstickUnstoredStatement(context, parent, data.sql, data.options); } diff --git a/test/python/test_definition_store.py b/test/python/test_definition_store.py index 5c58653..3c5a362 100644 --- a/test/python/test_definition_store.py +++ b/test/python/test_definition_store.py @@ -84,7 +84,7 @@ def test_rollback_and_committed_drop(self): "rollback;" "select aggregate(revenue) from sales_v;" "drop view sales_v;" - "select count(*) from store.main.definitions;" + "select count(*) from store.main.definitions where view_name<>'';" ) self.assertEqual(rows, [["140"], ["70"], ["0"]]) error = self.run_sql( @@ -125,10 +125,89 @@ def test_prepared_parameters_and_replacement(self): "select region, sum(amount * 2) as measure revenue from store.main.sales;" "execute sales_total('N');" "drop view sales_v;" - "select count(*) from store.main.definitions;" + "select count(*) from store.main.definitions where view_name<>'';" ) self.assertEqual(rows, [["30"], ["60"], ["0"]]) + def test_drop_respects_temporary_view_shadowing(self): + self.create_sales_view() + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "create temp view sales_v as select 1 as value;" + "drop view sales_v;" + "select count(*) from duckdb_views() where temporary and view_name='sales_v';" + "select count(*) from store.main.definitions where view_name<>'';" + "select aggregate(revenue) from sales_v;" + "create temp view sales_v as select 2 as value;" + "drop view memory.main.sales_v;" + "select value from sales_v;" + "select count(*) from store.main.definitions where view_name<>'';" + "drop view temp.main.sales_v;" + ) + self.assertEqual(rows, [["0"], ["1"], ["70"], ["2"], ["0"]]) + + def test_nested_ctes_do_not_redirect_stored_sources(self): + self.run_sql( + "create schema store.original; create schema store.other;" + "create table store.original.sales as select 7 amount;" + "create table store.other.sales as select 99 amount;" + "set schema='store.original';" + "set yardstick_definition_table='store.main.definitions';" + "create view store.main.nested_cte as " + "select sum(amount) as measure total from sales " + "where exists (with sales as (select 1) select * from sales);" + "create view store.main.derived_cte as " + "select sum(amount) as measure total from sales, " + "(with sales as (select 1 marker) select * from sales) nested where nested.marker=1;" + "create view store.main.self_named_cte as " + "with sales as (select amount from sales), copy as (select * from sales) " + "select sum(amount) as measure total from copy;" + ) + rows = self.run_sql( + "set schema='store.other';" + "set yardstick_definition_table='store.main.definitions';" + "select aggregate(total) from store.main.nested_cte;" + "select aggregate(total) from store.main.derived_cte;" + "select aggregate(total) from store.main.self_named_cte;" + ) + self.assertEqual(rows, [["7"], ["7"], ["7"]]) + + def test_existing_table_guard_initialization_preserves_definitions(self): + self.create_sales_view() + rows = self.run_sql( + "delete from store.main.definitions where view_name='';" + "set yardstick_definition_table='store.main.definitions';" + "select aggregate(revenue) from sales_v;" + ) + self.assertEqual(rows, [["70"]]) + error = self.run_sql( + "set yardstick_definition_table='store.main.definitions'; drop view sales_v;", + expect_error=True, + ) + self.assertIn("exclusive writer access", error) + rows = self.run_sql( + "insert into store.main.definitions values ('', '', '', 'yardstick_guard_v1:' || uuid()::varchar);" + "set yardstick_definition_table='store.main.definitions';" + "create or replace view sales_v as " + "select region, sum(amount * 2) as measure revenue from store.main.sales;" + "select aggregate(revenue) from sales_v;" + ) + self.assertEqual(rows, [["140"]]) + + def test_invalid_or_duplicate_guard_is_rejected(self): + self.create_sales_view() + self.run_sql("update store.main.definitions set definition_sql='invalid' where view_name='';") + query = "set yardstick_definition_table='store.main.definitions'; select aggregate(revenue) from sales_v;" + error = self.run_sql(query, expect_error=True) + self.assertIn("Invalid or duplicate Yardstick concurrency guard", error) + self.run_sql( + "update store.main.definitions set definition_sql='yardstick_guard_v1:' || uuid()::varchar " + "where view_name='';" + "insert into store.main.definitions values ('', '', '', 'yardstick_guard_v1:' || uuid()::varchar);" + ) + error = self.run_sql(query, expect_error=True) + self.assertIn("Invalid or duplicate Yardstick concurrency guard", error) + def test_source_binding_survives_schema_changes(self): rows = self.run_sql( "create schema original; create schema other;" @@ -156,12 +235,16 @@ def test_ducklake_lifecycle_and_reopen(self): rows = self.run_sql( "set yardstick_definition_table='store.main.definitions';" "select aggregate(revenue) from sales_v;" + "begin;" + "create or replace view sales_v as select region, sum(amount * 3) as measure revenue from store.main.sales;" + "select aggregate(revenue) from sales_v;" + "rollback; select aggregate(revenue) from sales_v;" "create or replace view sales_v as select region, sum(amount * 2) as measure revenue from store.main.sales;" "select aggregate(revenue) from sales_v;" - "drop view sales_v; select count(*) from store.main.definitions;", + "drop view sales_v; select count(*) from store.main.definitions where view_name<>'';", ducklake=True, ) - self.assertEqual(rows, [["70"], ["140"], ["0"]]) + self.assertEqual(rows, [["70"], ["210"], ["70"], ["140"], ["0"]]) if __name__ == "__main__": diff --git a/test/sql/definition_store.test b/test/sql/definition_store.test index 804182a..10a0090 100644 --- a/test/sql/definition_store.test +++ b/test/sql/definition_store.test @@ -147,7 +147,7 @@ statement ok DROP VIEW stored_sales; query I -SELECT count(*) FROM definition_store.main.views; +SELECT count(*) FROM definition_store.main.views WHERE view_name<>''; ---- 0 @@ -270,7 +270,7 @@ statement ok DROP VIEW stored_sales; query I -SELECT count(*) FROM definition_store.main.views; +SELECT count(*) FROM definition_store.main.views WHERE view_name<>''; ---- 0 @@ -285,6 +285,117 @@ catalog.schema.table statement ok RESET yardstick_definition_table; +# Competing sessions must not commit duplicate keys, including case variants. +statement ok +SET threads=1; + +statement ok +SET yardstick_definition_table='definition_store.main.guarded_views'; + +statement ok +CREATE VIEW guard_seed AS SELECT SUM(amount) AS MEASURE total FROM definition_sales; + +statement ok guard_one +SET yardstick_definition_table='definition_store.main.guarded_views'; + +statement ok guard_two +SET yardstick_definition_table='definition_store.main.guarded_views'; + +statement ok guard_one +BEGIN; + +query I guard_one +SELECT count(*) FROM definition_store.main.guarded_views WHERE view_name<>''; +---- +1 + +statement ok guard_two +BEGIN; + +query I guard_two +SELECT count(*) FROM definition_store.main.guarded_views WHERE view_name<>''; +---- +1 + +statement ok guard_one +CREATE VIEW GuardRace AS SELECT SUM(amount) AS MEASURE total FROM definition_sales; + +statement error guard_two +CREATE VIEW guardrace AS SELECT SUM(amount * 2) AS MEASURE total FROM definition_sales; +---- +Conflict + +statement ok guard_one +COMMIT; + +statement ok guard_two +ROLLBACK; + +query I +SELECT count(*) FROM definition_store.main.guarded_views WHERE lower(view_name)='guardrace'; +---- +1 + +query I +SELECT AGGREGATE(total) FROM guardrace; +---- +70 + +# An old snapshot cannot insert a duplicate after the winning writer commits. +statement ok guard_two +BEGIN; + +query I guard_two +SELECT count(*) FROM definition_store.main.guarded_views WHERE view_name<>''; +---- +2 + +statement ok +CREATE VIEW GuardStale AS SELECT SUM(amount) AS MEASURE total FROM definition_sales; + +statement error guard_two +CREATE VIEW guardstale AS SELECT SUM(amount * 3) AS MEASURE total FROM definition_sales; +---- +Conflict + +statement ok guard_two +ROLLBACK; + +query I +SELECT AGGREGATE(total) FROM guard_seed; +---- +70 + +# Guard changes also roll back, allowing subsequent mutations to succeed. +statement ok +BEGIN; + +statement ok +CREATE OR REPLACE VIEW guard_seed AS SELECT SUM(amount * 2) AS MEASURE total FROM definition_sales; + +statement ok +DROP VIEW guard_seed; + +statement ok +CREATE VIEW guard_seed AS SELECT SUM(amount * 3) AS MEASURE total FROM definition_sales; + +statement ok +ROLLBACK; + +statement ok +CREATE OR REPLACE VIEW guard_seed AS SELECT SUM(amount * 4) AS MEASURE total FROM definition_sales; + +query I +SELECT AGGREGATE(total) FROM guard_seed; +---- +280 + +statement ok +RESET threads; + +statement ok +RESET yardstick_definition_table; + # The same table-backed path can be entirely session-local. statement ok SET yardstick_definition_table='temp.main.session_views'; @@ -298,7 +409,7 @@ SELECT AGGREGATE(total) FROM session_only; 70 query I -SELECT count(*) FROM temp.main.session_views; +SELECT count(*) FROM temp.main.session_views WHERE view_name<>''; ---- 1 From 419df871f741fa1c44e468ea2c881505612aa8a2 Mon Sep 17 00:00:00 2001 From: Nico Ritschel Date: Wed, 11 Nov 2026 20:43:46 -0800 Subject: [PATCH 3/3] Parse commented and qualified legacy view names --- test/python/test_definition_store.py | 17 ++++++ yardstick-rs/src/sql/measures.rs | 78 +++++++++++++++++----------- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/test/python/test_definition_store.py b/test/python/test_definition_store.py index 3c5a362..aaf8fec 100644 --- a/test/python/test_definition_store.py +++ b/test/python/test_definition_store.py @@ -172,6 +172,23 @@ def test_nested_ctes_do_not_redirect_stored_sources(self): ) self.assertEqual(rows, [["7"], ["7"], ["7"]]) + def test_commented_definitions_with_same_name_in_different_schemas(self): + self.run_sql( + "create schema store.first_schema; create schema store.second_schema;" + "create table store.main.sales as select 7 amount;" + "set yardstick_definition_table='store.main.definitions';" + "/* Keep the declaration comment. */ create view store.first_schema.same_name as " + "select sum(amount) as measure total from store.main.sales;" + "create view store.second_schema.same_name as " + "select sum(amount * 3) as measure total from store.main.sales;" + ) + rows = self.run_sql( + "set yardstick_definition_table='store.main.definitions';" + "select aggregate(total) from store.first_schema.same_name;" + "select aggregate(total) from store.second_schema.same_name;" + ) + self.assertEqual(rows, [["7"], ["21"]]) + def test_existing_table_guard_initialization_preserves_definitions(self): self.create_sales_view() rows = self.run_sql( diff --git a/yardstick-rs/src/sql/measures.rs b/yardstick-rs/src/sql/measures.rs index c68a985..5d88bc9 100644 --- a/yardstick-rs/src/sql/measures.rs +++ b/yardstick-rs/src/sql/measures.rs @@ -1767,38 +1767,44 @@ pub fn extract_all_aggregate_calls(sql: &str) -> Vec<(String, usize, usize)> { // Nom Parsers - CREATE VIEW // ============================================================================= -/// Parse CREATE [OR REPLACE] VIEW name -fn create_view_header(input: &str) -> IResult<&str, &str> { - let (input, _) = multispace0(input)?; - let (input, _) = tag_no_case("CREATE")(input)?; - let (input, _) = multispace1(input)?; - - // Optional OR REPLACE - let (input, _) = opt(tuple(( - tag_no_case("OR"), - multispace1, - tag_no_case("REPLACE"), - multispace1, - )))(input)?; - - // Optional TEMP/TEMPORARY - let (input, _) = opt(tuple(( - alt((tag_no_case("TEMPORARY"), tag_no_case("TEMP"))), - multispace1, - )))(input)?; - - let (input, _) = tag_no_case("VIEW")(input)?; - let (input, _) = multispace1(input)?; - let (input, name) = identifier(input)?; - - Ok((input, name)) -} - /// Extract view name from CREATE VIEW statement pub fn extract_view_name(sql: &str) -> Option { - create_view_header(sql) - .ok() - .map(|(_, name)| name.to_string()) + let upper = sql.to_ascii_uppercase(); + let mut idx = skip_ws_and_comments(sql, 0); + if !matches_keyword_at(&upper, idx, "CREATE") { + return None; + } + idx = skip_ws_and_comments(sql, idx + "CREATE".len()); + if matches_keyword_at(&upper, idx, "OR") { + idx = skip_ws_and_comments(sql, idx + "OR".len()); + if !matches_keyword_at(&upper, idx, "REPLACE") { + return None; + } + idx = skip_ws_and_comments(sql, idx + "REPLACE".len()); + } + for keyword in ["TEMPORARY", "TEMP"] { + if matches_keyword_at(&upper, idx, keyword) { + idx = skip_ws_and_comments(sql, idx + keyword.len()); + break; + } + } + if !matches_keyword_at(&upper, idx, "VIEW") { + return None; + } + idx = skip_ws_and_comments(sql, idx + "VIEW".len()); + if matches_keyword_at(&upper, idx, "IF") { + idx = skip_ws_and_comments(sql, idx + "IF".len()); + if !matches_keyword_at(&upper, idx, "NOT") { + return None; + } + idx = skip_ws_and_comments(sql, idx + "NOT".len()); + if !matches_keyword_at(&upper, idx, "EXISTS") { + return None; + } + idx = skip_ws_and_comments(sql, idx + "EXISTS".len()); + } + let end = parse_qualified_name_span(sql, idx)?; + extract_last_qualified_identifier(&sql[idx..end]) } /// Extract view name from DROP VIEW statement @@ -10826,6 +10832,18 @@ FROM orders"#; extract_view_name("CREATE OR REPLACE TEMPORARY VIEW temp_bar AS SELECT 1"), Some("temp_bar".to_string()) ); + assert_eq!( + extract_view_name("/* declaration */ CREATE VIEW catalog.schema.foo AS SELECT 1"), + Some("foo".to_string()) + ); + assert_eq!( + extract_view_name("-- declaration\nCREATE VIEW IF /* gap */ NOT EXISTS \"schema\".\"foo.bar\" (x) AS SELECT 1"), + Some("foo.bar".to_string()) + ); + assert_eq!( + extract_view_name("CREATE /* gap */ OR REPLACE VIEW \"foo\"\"bar\" AS SELECT 1"), + Some("foo\"bar".to_string()) + ); } #[test]