diff --git a/.github/workflows/check_cluster_schema.yml b/.github/workflows/check_cluster_schema.yml new file mode 100644 index 00000000..17c6ac96 --- /dev/null +++ b/.github/workflows/check_cluster_schema.yml @@ -0,0 +1,16 @@ +# Checks that scripts/cluster-migration/schema.sql and its migrations give the same tables +on: + pull_request: + paths: + - "scripts/cluster-migration/schema.sql" + - "scripts/cluster-migration/migrations/**" + - "scripts/cluster-migration/test/**" + - ".github/workflows/check_cluster_schema.yml" + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Compare schema.sql with the migrations + run: scripts/cluster-migration/test/check.sh diff --git a/scripts/cluster-migration/migrations/000_baseline.sql b/scripts/cluster-migration/migrations/000_baseline.sql new file mode 100644 index 00000000..48cdf331 --- /dev/null +++ b/scripts/cluster-migration/migrations/000_baseline.sql @@ -0,0 +1,610 @@ +-- The tables and views of the ooni database, as production has them (dumped from every replica on +-- 2026-10-20 after asnmeta was made replicated again on data1, OOPRIV-181). Create the database on every +-- node, then load this file into it: +-- clickhouse-client -q "CREATE DATABASE IF NOT EXISTS ooni ON CLUSTER oonidata_cluster" +-- clickhouse-client --database ooni --multiquery < schema.sql +-- Any database name works, e.g. for CI: ON CLUSTER creates each table in the client's database on every +-- node, and the Keeper paths include {database}, so a second database on the same cluster doesn't share +-- production's replicas. +-- Where the replicas differ, this is the definition of the replica that holds the data: +-- event_detector_cusums, labeling_frames: data1 (the only replica with rows, and with their newer columns) +-- fastpath: data1 and data3 (data2 alone has deduplicate_merge_projection_mode = 'rebuild') +-- event_detector_cusums, event_detector_v2_changepoints, labeling_frames and changepoint_label are not +-- replicated: ON CLUSTER creates an empty local table on each node, as production has them. +-- Not included: the updaters' *_tmp tables (recreated on every run), msmt_feedback_old (superseded by +-- msmt_feedback), session_expunge (unused since backend #972) and the leftover inner table of the dropped +-- counters_test_list view. + +CREATE TABLE IF NOT EXISTS analysis_web_measurement ON CLUSTER oonidata_cluster +( + `domain` String, + `input` String, + `test_name` String, + `probe_asn` UInt32, + `probe_as_org_name` String, + `probe_cc` String, + `resolver_asn` UInt32, + `resolver_as_cc` String, + `network_type` String, + `measurement_start_time` DateTime64(3, 'UTC'), + `measurement_uid` String, + `ooni_run_link_id` String, + `top_probe_analysis` Nullable(String), + `top_dns_failure` Nullable(String), + `top_tcp_failure` Nullable(String), + `top_tls_failure` Nullable(String), + `dns_blocked` Float32, + `dns_down` Float32, + `dns_ok` Float32, + `tcp_blocked` Float32, + `tcp_down` Float32, + `tcp_ok` Float32, + `tls_blocked` Float32, + `tls_down` Float32, + `tls_ok` Float32, + `top_dns_rule_id` LowCardinality(String), + `top_tcp_rule_id` LowCardinality(String), + `top_tls_rule_id` LowCardinality(String), + `probe_id` FixedString(64), + `analysis_rules_version` UInt16, + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2, + INDEX probe_cc_idx probe_cc TYPE minmax GRANULARITY 1, + INDEX probe_asn_idx probe_asn TYPE minmax GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/analysis_web_measurement/{shard}', '{replica}') +PARTITION BY substring(measurement_uid, 1, 6) +PRIMARY KEY measurement_uid +ORDER BY (measurement_uid, measurement_start_time, probe_cc, probe_asn, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS asnmeta ON CLUSTER oonidata_cluster +( + `asn` UInt32, + `org_name` String, + `cc` String, + `changed` Date, + `aut_name` String, + `source` String +) +ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/asnmeta', '{replica}') +ORDER BY (asn, changed) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS changepoint_label ON CLUSTER oonidata_cluster +( + -- Ground truth, set by a person. Represents the network state for + -- the (probe_cc, probe_asn, resolver_asn, domain) series of the + -- changepoint it labels + `id` UUID, + -- UUID in event_detector_v2_changepoints (foreign key) + `changepoint_id` UUID, + `created_at` DateTime64(3, 'UTC') DEFAULT now64(), + `author` String, + -- blocked, ok or undecided + `verdict` LowCardinality(String), + -- Additional context the author might share + `notes` String, + `last_ok_time` Nullable(DateTime64(3, 'UTC')), + `first_block_time` Nullable(DateTime64(3, 'UTC')), + `last_block_time` Nullable(DateTime64(3, 'UTC')), + `first_ok_time` Nullable(DateTime64(3, 'UTC')) +) +ENGINE = MergeTree +ORDER BY (changepoint_id, created_at, id) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS citizenlab ON CLUSTER oonidata_cluster +( + `domain` String, + `url` String, + `cc` FixedString(32), + `category_code` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/citizenlab', '{replica}') +ORDER BY (domain, url, cc, category_code) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS event_detector_changepoints ON CLUSTER oonidata_cluster +( + `probe_asn` UInt32, + `probe_cc` String, + `domain` String, + `ts` DateTime64(3, 'UTC'), + `count_isp_resolver` Nullable(UInt32), + `count_other_resolver` Nullable(UInt32), + `count` Nullable(UInt32), + `dns_isp_blocked` Nullable(Float32), + `dns_other_blocked` Nullable(Float32), + `tcp_blocked` Nullable(Float32), + `tls_blocked` Nullable(Float32), + `change_dir` Nullable(Int8), + `s_pos` Nullable(Float32), + `s_neg` Nullable(Float32), + `current_state` String, + `h` Nullable(Float32), + `block_type` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/event_detector_changepoints/{shard}', '{replica}') +ORDER BY (probe_asn, probe_cc, ts, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS event_detector_cusums ON CLUSTER oonidata_cluster +( + `probe_asn` UInt32, + `probe_cc` String, + `domain` String, + `ts` DateTime64(3, 'UTC'), + `dns_isp_blocked_current_state` String DEFAULT 'ok', + `dns_isp_blocked_s_pos` Nullable(Float64), + `dns_isp_blocked_s_neg` Nullable(Float64), + `dns_other_blocked_current_state` String DEFAULT 'ok', + `dns_other_blocked_s_pos` Nullable(Float64), + `dns_other_blocked_s_neg` Nullable(Float64), + `tcp_blocked_current_state` String DEFAULT 'ok', + `tcp_blocked_s_pos` Nullable(Float64), + `tcp_blocked_s_neg` Nullable(Float64), + `tls_blocked_current_state` String DEFAULT 'ok', + `tls_blocked_s_pos` Nullable(Float64), + `tls_blocked_s_neg` Nullable(Float64), + `dns_isp_blocked_last_change` Int8 DEFAULT 0, + `dns_isp_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `dns_other_blocked_last_change` Int8 DEFAULT 0, + `dns_other_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `tcp_blocked_last_change` Int8 DEFAULT 0, + `tcp_blocked_last_ts` Nullable(DateTime64(3, 'UTC')), + `tls_blocked_last_change` Int8 DEFAULT 0, + `tls_blocked_last_ts` Nullable(DateTime64(3, 'UTC')) +) +ENGINE = ReplacingMergeTree(ts) +ORDER BY (probe_asn, probe_cc, domain) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS event_detector_v2_changepoints ON CLUSTER oonidata_cluster +( + `uuid` UUID, + `domain` String, + `probe_cc` String, + `probe_asn` UInt32, + `resolver_asn` UInt32, + `layer` LowCardinality(String), + `ts_hour` DateTime64(3, 'UTC'), + `s_neg` Float64, + `s_pos` Float64, + `h` Float64, + `state` LowCardinality(String), + -- JSON-encoded parameters the detector was run with + `run_parameters` String, + `created_at` DateTime64(3, 'UTC') DEFAULT now64() +) +ENGINE = MergeTree +ORDER BY (ts_hour, domain, layer, probe_cc, probe_asn, resolver_asn, state, uuid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS fastpath ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `report_id` String, + `input` String, + `probe_cc` LowCardinality(String), + `probe_asn` Int32, + `test_name` LowCardinality(String), + `test_start_time` DateTime, + `measurement_start_time` DateTime, + `filename` String, + `scores` String, + `platform` String, + `anomaly` LowCardinality(String), + `confirmed` LowCardinality(String), + `msm_failure` LowCardinality(String), + `domain` String, + `software_name` String, + `software_version` String, + `control_failure` String, + `blocking_general` Float32, + `is_ssl_expected` Int8, + `page_len` Int32, + `page_len_ratio` Float32, + `server_cc` String, + `server_asn` Int8, + `server_as_name` String, + `update_time` DateTime64(3) MATERIALIZED now64(), + `test_version` String, + `architecture` String, + `engine_name` LowCardinality(String), + `engine_version` String, + `test_runtime` Float32, + `blocking_type` String, + `test_helper_address` LowCardinality(String), + `test_helper_type` LowCardinality(String), + `ooni_run_link_id` Nullable(UInt64), + `is_verified` LowCardinality(String) DEFAULT 'u', + INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1, + INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8, + INDEX input_bloom_idx input TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/fastpath/{shard}', '{replica}', update_time) +ORDER BY (measurement_start_time, report_id, input, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS faulty_measurements ON CLUSTER oonidata_cluster +( + `ts` DateTime64(3, 'UTC') DEFAULT now64(), + `type` String, + `uid` UUID DEFAULT generateUUIDv4(), + -- geoip lookup result for the probe IP + `probe_cc` String, + `probe_asn` UInt32, + -- JSON-encoded details about the anomaly + `details` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/faulty_measurements/{shard}', '{replica}') +ORDER BY (ts, type, probe_cc, probe_asn, uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS fingerprints_dns ON CLUSTER oonidata_cluster +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + +CREATE TABLE IF NOT EXISTS fingerprints_http ON CLUSTER oonidata_cluster +( + `name` String, + `scope` Enum8('nat' = 1, 'isp' = 2, 'prod' = 3, 'inst' = 4, 'vbw' = 5, 'fp' = 6, 'injb' = 7, 'prov' = 8), + `other_names` String, + `location_found` String, + `pattern_type` Enum8('full' = 1, 'prefix' = 2, 'contains' = 3, 'regexp' = 4), + `pattern` String, + `confidence_no_fp` UInt8, + `expected_countries` String, + `source` String, + `exp_url` String, + `notes` String +) +ENGINE = EmbeddedRocksDB +PRIMARY KEY name; + +CREATE TABLE IF NOT EXISTS jsonl ON CLUSTER oonidata_cluster +( + `report_id` String, + `input` String, + `s3path` String, + `linenum` Int32, + `measurement_uid` String, + `date` Date, + `source` String, + `update_time` DateTime64(3) MATERIALIZED now64() +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/jsonl/{shard}', '{replica}', update_time) +ORDER BY (report_id, input, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS labeling_frames ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `blocked_fastpath` Bool, + `blocked_analysis` Bool, + `stratum` LowCardinality(String), + `test_name` LowCardinality(String), + `day` DateTime('UTC'), + `fastpath_query` LowCardinality(String), + `analysis_query` LowCardinality(String), + `failed_fastpath` Bool +) +ENGINE = MergeTree +ORDER BY (stratum, measurement_uid) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS msmt_feedback ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `account_id` String, + `status` String, + `update_time` DateTime64(3) DEFAULT now64(), + `comment` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/msmt_feedback_new/{shard}', '{replica}') +ORDER BY (measurement_uid, account_id, update_time) +SETTINGS index_granularity = 4; + +CREATE TABLE IF NOT EXISTS obs_http_middlebox ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `probe_asn` UInt32, + `probe_cc` String, + `probe_as_org_name` String, + `probe_as_cc` String, + `probe_as_name` String, + `network_type` String, + `platform` String, + `origin` String, + `engine_name` String, + `engine_version` String, + `architecture` String, + `resolver_ip` String, + `resolver_asn` UInt32, + `resolver_cc` String, + `resolver_as_org_name` String, + `resolver_as_cc` String, + `resolver_is_scrubbed` UInt8, + `resolver_asn_probe` UInt32, + `resolver_as_org_name_probe` String, + `created_at` Nullable(DateTime64(3, 'UTC')), + `hirl_sent_0` Nullable(String), + `hirl_sent_1` Nullable(String), + `hirl_sent_2` Nullable(String), + `hirl_sent_3` Nullable(String), + `hirl_sent_4` Nullable(String), + `hirl_received_0` Nullable(String), + `hirl_received_1` Nullable(String), + `hirl_received_2` Nullable(String), + `hirl_received_3` Nullable(String), + `hirl_received_4` Nullable(String), + `hirl_failure` Nullable(String), + `hirl_success` Nullable(UInt8), + `hfm_diff` Nullable(String), + `hfm_failure` Nullable(String), + `hfm_success` Nullable(UInt8), + `probe_id` FixedString(64) +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_http_middlebox/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS obs_openvpn ON CLUSTER oonidata_cluster +( + `anomaly` Int8, + `bootstrap_time` Float32, + `confirmed` Int8, + `error` String, + `failure` String, + `input` String, + `last_handshake_transaction_id` Int32, + `measurement_start_time` DateTime, + `measurement_uid` String, + `minivpn_version` String, + `obfs4_version` String, + `obfuscation` String, + `platform` String, + `probe_asn` Int32, + `probe_cc` String, + `probe_network_name` String, + `provider` String, + `remote` String, + `report_id` String, + `resolver_asn` Int32, + `resolver_ip` String, + `resolver_network_name` String, + `software_name` String, + `software_version` String, + `success` Int8, + `success_handshake` Int8, + `success_icmp` Int8, + `success_urlgrab` Int8, + `tcp_connect_status_success` Int8, + `test_runtime` Float32, + `test_start_time` DateTime, + `transport` String +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_openvpn/{shard}', '{replica}') +ORDER BY (measurement_start_time, report_id, input) +SETTINGS index_granularity = 8; + +CREATE TABLE IF NOT EXISTS obs_web ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `probe_asn` UInt32, + `probe_cc` String, + `probe_as_org_name` String, + `probe_as_cc` String, + `probe_as_name` String, + `network_type` String, + `platform` String, + `origin` String, + `engine_name` String, + `engine_version` String, + `architecture` String, + `resolver_ip` String, + `resolver_asn` UInt32, + `resolver_cc` String, + `resolver_as_org_name` String, + `resolver_as_cc` String, + `resolver_is_scrubbed` UInt8, + `resolver_asn_probe` UInt32, + `resolver_as_org_name_probe` String, + `created_at` Nullable(DateTime('UTC')), + `target_id` Nullable(String), + `hostname` Nullable(String), + `transaction_id` Nullable(UInt16), + `ip` Nullable(String), + `port` Nullable(UInt16), + `ip_asn` Nullable(UInt32), + `ip_as_org_name` Nullable(String), + `ip_as_cc` Nullable(String), + `ip_cc` Nullable(String), + `ip_is_bogon` Nullable(UInt8), + `dns_query_type` Nullable(String), + `dns_failure` Nullable(String), + `dns_engine` Nullable(String), + `dns_engine_resolver_address` Nullable(String), + `dns_answer_type` Nullable(String), + `dns_answer` Nullable(String), + `dns_answer_asn` Nullable(UInt32), + `dns_answer_as_org_name` Nullable(String), + `dns_t` Nullable(Float64), + `tcp_failure` Nullable(String), + `tcp_success` Nullable(UInt8), + `tcp_t` Nullable(Float64), + `tls_failure` Nullable(String), + `tls_server_name` Nullable(String), + `tls_outer_server_name` Nullable(String), + `tls_echconfig` Nullable(String), + `tls_version` Nullable(String), + `tls_cipher_suite` Nullable(String), + `tls_is_certificate_valid` Nullable(UInt8), + `tls_end_entity_certificate_fingerprint` Nullable(String), + `tls_end_entity_certificate_subject` Nullable(String), + `tls_end_entity_certificate_subject_common_name` Nullable(String), + `tls_end_entity_certificate_issuer` Nullable(String), + `tls_end_entity_certificate_issuer_common_name` Nullable(String), + `tls_end_entity_certificate_san_list` Array(String), + `tls_end_entity_certificate_not_valid_after` Nullable(DateTime64(3, 'UTC')), + `tls_end_entity_certificate_not_valid_before` Nullable(DateTime64(3, 'UTC')), + `tls_certificate_chain_length` Nullable(UInt16), + `tls_certificate_chain_fingerprints` Array(String), + `tls_handshake_read_count` Nullable(UInt16), + `tls_handshake_write_count` Nullable(UInt16), + `tls_handshake_read_bytes` Nullable(UInt32), + `tls_handshake_write_bytes` Nullable(UInt32), + `tls_handshake_last_operation` Nullable(String), + `tls_handshake_time` Nullable(Float64), + `tls_t` Nullable(Float64), + `http_request_url` Nullable(String), + `http_network` Nullable(String), + `http_alpn` Nullable(String), + `http_failure` Nullable(String), + `http_request_body_length` Nullable(UInt32), + `http_request_method` Nullable(String), + `http_runtime` Nullable(Float64), + `http_response_body_length` Nullable(Int32), + `http_response_body_is_truncated` Nullable(UInt8), + `http_response_body_sha1` Nullable(String), + `http_response_status_code` Nullable(UInt16), + `http_response_header_location` Nullable(String), + `http_response_header_server` Nullable(String), + `http_request_redirect_from` Nullable(String), + `http_request_body_is_truncated` Nullable(UInt8), + `http_t` Nullable(Float64), + `probe_analysis` Nullable(String), + `probe_id` FixedString(64), + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2, + INDEX probe_cc_idx probe_cc TYPE minmax GRANULARITY 1, + INDEX probe_asn_idx probe_asn TYPE minmax GRANULARITY 1 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_web_repl/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time, probe_cc, probe_asn) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS obs_web_ctrl ON CLUSTER oonidata_cluster +( + `measurement_uid` String, + `observation_idx` UInt16, + `input` Nullable(String), + `report_id` String, + `ooni_run_link_id` String DEFAULT '', + `measurement_start_time` DateTime64(3, 'UTC'), + `software_name` String, + `software_version` String, + `test_name` String, + `test_version` String, + `bucket_date` String, + `hostname` String, + `created_at` Nullable(DateTime64(3, 'UTC')), + `ip` String, + `port` Nullable(UInt16), + `ip_asn` Nullable(UInt32), + `ip_as_org_name` Nullable(String), + `ip_as_cc` Nullable(String), + `ip_cc` Nullable(String), + `ip_is_bogon` Nullable(UInt8), + `dns_failure` Nullable(String), + `dns_success` Nullable(UInt8), + `tcp_failure` Nullable(String), + `tcp_success` Nullable(UInt8), + `tls_failure` Nullable(String), + `tls_success` Nullable(UInt8), + `tls_server_name` Nullable(String), + `http_request_url` Nullable(String), + `http_failure` Nullable(String), + `http_success` Nullable(UInt8), + `http_response_body_length` Nullable(Int32), + INDEX measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2 +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{cluster}/tables/{database}/obs_web_ctrl/{shard}', '{replica}') +PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) +PRIMARY KEY (measurement_uid, observation_idx) +ORDER BY (measurement_uid, observation_idx, measurement_start_time, hostname) +SETTINGS index_granularity = 8192; + +CREATE TABLE IF NOT EXISTS test_groups ON CLUSTER oonidata_cluster +( + `test_name` String, + `test_group` String +) +ENGINE = Join(ANY, LEFT, test_name); + +CREATE TABLE IF NOT EXISTS url_priorities ON CLUSTER oonidata_cluster +( + `sign` Int8, + `category_code` String, + `cc` String, + `domain` String, + `url` String, + `priority` Int32 +) +ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{database}/url_priorities/{shard}', '{replica}', sign) +ORDER BY (category_code, cc, domain, url, priority) +SETTINGS index_granularity = 1024; + +-- Its rows are in a hidden table, .inner_id.. The Keeper path keeps production's name +-- for it, so a replica added later joins production's table whatever UUID its view gets. +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/.inner_id.6a72ceec-7e0e-4b65-a3ab-da2809202114/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input) +SETTINGS index_granularity = 8192 +AS SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + week, + probe_cc, + probe_asn, + input; diff --git a/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql b/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql new file mode 100644 index 00000000..57f8a487 --- /dev/null +++ b/scripts/cluster-migration/migrations/001_counters_asn_test_list_target.sql @@ -0,0 +1,73 @@ +-- Give counters_asn_test_list an explicit target table, counters_asn_test_list_data, instead of the +-- hidden .inner_id. table it made itself, and add week to its sorting key. The hidden table is +-- sorted by (probe_cc, probe_asn, input) only, so merges add up the weeks of each key and keep one +-- arbitrary week: the reader, ooniprobe's test-list/urls (common/prio.py), sums the current and previous +-- week and misses every count whose merged row says an older week. The new table keeps one row per +-- week, can be backed up and restored by name, and lets the view be dropped or changed without losing +-- its rows. +-- In production: not a mutation. The old rows are wrong, so they aren't copied: the two weeks the +-- reader uses are counted again from fastpath, from rows inserted before the new view existed (the +-- view counts the rest). Unlike the view, this also counts uploads that came more than 8 days after +-- their measurement. Queries on counters_asn_test_list fail between the DROP and the RENAME, for about +-- a second; test-list/urls returns the corrected counts from then on. + +CREATE TABLE IF NOT EXISTS counters_asn_test_list_data ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/counters_asn_test_list_data/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input, week) +SETTINGS index_granularity = 8192; + +-- Count fastpath's new rows from now on, alongside the old view +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list_new ON CLUSTER oonidata_cluster TO counters_asn_test_list_data +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +AS SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time < now()) AND (measurement_start_time > (now() - toIntervalDay(8))) AND (test_name = 'web_connectivity') +GROUP BY + week, + probe_cc, + probe_asn, + input; + +-- Count the two weeks the reader sums, from the rows the new view didn't see +INSERT INTO counters_asn_test_list_data SELECT + toStartOfWeek(measurement_start_time) AS week, + probe_cc, + probe_asn, + input, + count() AS msmt_cnt +FROM fastpath +INNER JOIN citizenlab ON fastpath.input = citizenlab.url +WHERE (measurement_start_time >= toStartOfWeek(now() - toIntervalWeek(1))) AND (test_name = 'web_connectivity') AND (update_time < ( + SELECT metadata_modification_time + FROM system.tables + WHERE (database = currentDatabase()) AND (name = 'counters_asn_test_list_new') +)) +GROUP BY + week, + probe_cc, + probe_asn, + input; + +-- The old view and its hidden table. They exist only on data1 and data3, hence IF EXISTS. +DROP TABLE IF EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster SYNC; + +RENAME TABLE counters_asn_test_list_new TO counters_asn_test_list ON CLUSTER oonidata_cluster; diff --git a/scripts/cluster-migration/migrations/README.md b/scripts/cluster-migration/migrations/README.md new file mode 100644 index 00000000..c0093add --- /dev/null +++ b/scripts/cluster-migration/migrations/README.md @@ -0,0 +1,33 @@ +# ClickHouse schema migrations + +`../schema.sql` is the schema as it is now: read it to see a table, and load it to build a new +cluster. A change to an existing cluster's schema goes in a numbered file here, applied once, in +order, on one node (its DDL uses `ON CLUSTER`): + + clickhouse-client --database ooni --multiquery < migrations/NNN_description.sql + +- Every schema change updates `schema.sql` and adds a migration. `../test/check.sh`, which CI runs, + fails unless `schema.sql` gives the same tables as all the migrations in order. +- `000_baseline.sql` is `schema.sql` as of ooni/devops#521. Don't edit it, or a migration that has + been applied: fix forward with a new one. +- Start each file with what it does to production: whether it rewrites data (a mutation, e.g. + `MODIFY COLUMN` or `MATERIALIZE`), how long it took on a copy, and what readers see meanwhile. +- Say in the PR when it was applied to production. + +## Rebuilding counters_asn_test_list + +Nothing rebuilds the view's counts by itself: it only counts new fastpath inserts. Restore +`counters_asn_test_list_data` from a backup before the view (a view without its table makes every +fastpath insert fail). If the table is lost with no backup, recreate it from `schema.sql` and count the +two weeks that test-list/urls reads again from fastpath, from the rows inserted before the table +existed (the view counts the rest): + +```sql +INSERT INTO ooni.counters_asn_test_list_data +SELECT toStartOfWeek(measurement_start_time) AS week, probe_cc, probe_asn, input, count() AS msmt_cnt +FROM ooni.fastpath INNER JOIN ooni.citizenlab ON fastpath.input = citizenlab.url +WHERE measurement_start_time >= toStartOfWeek(now() - INTERVAL 1 WEEK) AND test_name = 'web_connectivity' + AND update_time < (SELECT metadata_modification_time FROM system.tables + WHERE database = 'ooni' AND name = 'counters_asn_test_list_data') +GROUP BY week, probe_cc, probe_asn, input +``` diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql index 48cdf331..402a0dbc 100644 --- a/scripts/cluster-migration/schema.sql +++ b/scripts/cluster-migration/schema.sql @@ -5,7 +5,7 @@ -- clickhouse-client --database ooni --multiquery < schema.sql -- Any database name works, e.g. for CI: ON CLUSTER creates each table in the client's database on every -- node, and the Keeper paths include {database}, so a second database on the same cluster doesn't share --- production's replicas. +-- production's replicas. A change to an existing cluster also goes in migrations/: see its README. -- Where the replicas differ, this is the definition of the replica that holds the data: -- event_detector_cusums, labeling_frames: data1 (the only replica with rows, and with their newer columns) -- fastpath: data1 and data3 (data2 alone has deduplicate_merge_projection_mode = 'rebuild') @@ -581,9 +581,19 @@ ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{database}/ ORDER BY (category_code, cc, domain, url, priority) SETTINGS index_granularity = 1024; --- Its rows are in a hidden table, .inner_id.. The Keeper path keeps production's name --- for it, so a replica added later joins production's table whatever UUID its view gets. -CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster +CREATE TABLE IF NOT EXISTS counters_asn_test_list_data ON CLUSTER oonidata_cluster +( + `week` DateTime, + `probe_cc` String, + `probe_asn` UInt64, + `input` String, + `msmt_cnt` UInt64 +) +ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/counters_asn_test_list_data/{shard}', '{replica}') +ORDER BY (probe_cc, probe_asn, input, week) +SETTINGS index_granularity = 8192; + +CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidata_cluster TO counters_asn_test_list_data ( `week` DateTime, `probe_cc` String, @@ -591,9 +601,6 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS counters_asn_test_list ON CLUSTER oonidat `input` String, `msmt_cnt` UInt64 ) -ENGINE = ReplicatedSummingMergeTree('/clickhouse/{cluster}/tables/{database}/.inner_id.6a72ceec-7e0e-4b65-a3ab-da2809202114/{shard}', '{replica}') -ORDER BY (probe_cc, probe_asn, input) -SETTINGS index_granularity = 8192 AS SELECT toStartOfWeek(measurement_start_time) AS week, probe_cc, diff --git a/scripts/cluster-migration/test/check.sh b/scripts/cluster-migration/test/check.sh new file mode 100755 index 00000000..2b0ab45b --- /dev/null +++ b/scripts/cluster-migration/test/check.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Checks that schema.sql and the migrations agree: loads schema.sql into one database and every +# migrations/NNN_*.sql, in order, into another, then compares each table's definition and Keeper path. +# Needs docker compose; set COMPOSE="podman compose" to use podman. +set -euo pipefail +cd "$(dirname "$0")" +compose=${COMPOSE:-docker compose} +q() { $compose exec -T clickhouse clickhouse-client -u admin --password admin "$@"; } +work=$(mktemp -d) +trap '$compose down -v >/dev/null 2>&1; rm -rf "$work"' EXIT + +$compose up -d +for _ in $(seq 60); do + q -q "SELECT 1 FROM system.zookeeper WHERE path = '/'" /dev/null 2>&1 && break + sleep 1 +done + +for db in from_schema from_migrations; do + q -q "CREATE DATABASE $db ON CLUSTER oonidata_cluster FORMAT Null" /dev/null +for f in ../migrations/[0-9][0-9][0-9]_*.sql; do + echo "applying $(basename "$f")" + q --database from_migrations --multiquery <"$f" >/dev/null +done + +# each table's definition and Keeper path, with the database name and UUIDs blanked +snapshot() { + q -q " + SELECT t.name, + replaceAll(replaceAll(replaceRegexpAll(t.create_table_query, '[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}', ''), + '$1.', '.'), '/$1/', '//'), + replaceAll(r.zookeeper_path, '/$1/', '//') + FROM system.tables AS t + LEFT JOIN system.replicas AS r ON r.database = t.database AND r.table = t.name + WHERE t.database = '$1' + ORDER BY t.name + FORMAT TSV" "$work/schema.tsv" +snapshot from_migrations >"$work/migrations.tsv" +if ! diff -u "$work/schema.tsv" "$work/migrations.tsv"; then + echo "schema.sql and the migrations disagree: the diff above is schema.sql (-) against the migrations (+)" + exit 1 +fi +echo "schema.sql and the migrations agree on all $(wc -l <"$work/schema.tsv") tables" diff --git a/scripts/cluster-migration/test/clickhouse.xml b/scripts/cluster-migration/test/clickhouse.xml new file mode 100644 index 00000000..f393db4b --- /dev/null +++ b/scripts/cluster-migration/test/clickhouse.xml @@ -0,0 +1,22 @@ + + + + + + + + + clickhouse9000adminadmin + + + + oonidata_cluster01clickhouse + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + 1clickhouse9234 + + clickhouse9181 + diff --git a/scripts/cluster-migration/test/compose.yml b/scripts/cluster-migration/test/compose.yml new file mode 100644 index 00000000..5ebd8d0f --- /dev/null +++ b/scripts/cluster-migration/test/compose.yml @@ -0,0 +1,11 @@ +# A one-node oonidata_cluster with Keeper, at production's ClickHouse version, for check.sh +name: cluster-schema-check +services: + clickhouse: + image: docker.io/clickhouse/clickhouse-server:24.8.6.70 + hostname: clickhouse + environment: {CLICKHOUSE_USER: admin, CLICKHOUSE_PASSWORD: admin} + volumes: + - ./clickhouse.xml:/etc/clickhouse-server/config.d/cluster.xml:ro,z + ulimits: {nofile: 262144} + mem_limit: 2g