Clickhouse backup

.github/workflows/test_clickhouse_backup.yml

@@ -0,0 +1,26 @@
+# Tests the ClickHouse backup role, its alerts and their Alertmanager routes
+# (ansible/roles/clickhouse_backup/tests), only when one of them changes
+name: Test ClickHouse backups
+on:
+  pull_request:
+    paths:
+      - "ansible/roles/clickhouse_backup/**"
+      - "ansible/roles/prometheus/files/alert_rules.yml"
+      - "ansible/roles/prometheus_alertmanager/**"
+      - ".github/workflows/test_clickhouse_backup.yml"
+  push:
+    branches: [main]
+    paths:
+      - "ansible/roles/clickhouse_backup/**"
+      - "ansible/roles/prometheus/files/alert_rules.yml"
+      - "ansible/roles/prometheus_alertmanager/**"
+      - ".github/workflows/test_clickhouse_backup.yml"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ansible-core python3-boto3
+      - run: ansible/roles/clickhouse_backup/tests/run.sh

ansible/deploy-clickhouse.yml

@@ -11,3 +11,16 @@
       tags: [node_exporter]
     - role: oonidata_clickhouse
       tags: [clickhouse]
+
+# data3 backs up the replicated tables: they hold the same data on every
+# node, and data3 isn't behind clickhouseproxy and doesn't run Airflow.
+# data1 backs up the plain MergeTree tables whose data only it holds (see
+# clickhouse_backup_data1_tables). The jobs are set in host_vars.
+- name: Deploy ClickHouse backups
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+    - data3.htz-fsn.prod.ooni.nu
+  become: true
+  roles:
+    - role: clickhouse_backup
+      tags: [clickhouse_backup]

ansible/group_vars/clickhouse/vars.yml

@@ -411,6 +411,27 @@ clickhouse_prometheus:
   asynchronous_metrics: true
   status_info: true
 
+# For textfile metrics, e.g. from clickhouse_backup
+node_exporter_textfile_dir: /var/lib/node_exporter/textfile_collector
+
+# clickhouse_backup: Cloudflare R2. The endpoint names the account, so it's
+# kept in SSM with the token's keys.
+clickhouse_backup_s3_endpoint: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_backup_r2_endpoint', profile='oonidevops_user_prod') }}"
+clickhouse_backup_s3_access_key_id: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_backup_r2_access_key_id', profile='oonidevops_user_prod') }}"
+clickhouse_backup_s3_secret_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_backup_r2_secret_access_key', profile='oonidevops_user_prod') }}"
+# Tables with personal or unpublished data, backed up to the private bucket
+clickhouse_backup_private_tables:
+  - msmt_feedback        # account ids and free-text comments
+  - faulty_measurements  # until its contents are reviewed
+  - url_priorities       # until it's decided whether it's published
+# Plain MergeTree tables whose data only data1 holds (2026-10-20). Remove a
+# table from here once it's replicated, and data3 picks it up.
+clickhouse_backup_data1_tables:
+  - labeling_frames
+  - event_detector_cusums
+  - event_detector_v2_changepoints
+  - changepoint_label
+
 prometheus_nginx_proxy_config:
   - location: /metrics/node_exporter
     proxy_pass: http://127.0.0.1:8100/metrics

ansible/host_vars/data1.htz-fsn.prod.ooni.nu

@@ -2,3 +2,10 @@ tls_cert_dir: /var/lib/dehydrated/certs
 ssl_domains:
    - "data1.htz-fsn.prod.ooni.nu"
    - "airflow.prod.ooni.io"
+
+# Only the tables whose data only data1 holds; data3 backs up the rest
+clickhouse_backup_jobs:
+  - name: data1
+    s3_bucket: ooni-clickhouse-backup
+    include_tables: "{{ clickhouse_backup_data1_tables }}"
+    exclude_tables: []

ansible/host_vars/data3.htz-fsn.prod.ooni.nu

@@ -1,2 +1,20 @@
 non_admin_usernames: []
 clickhouse_base_path: /data/clickhouse
+
+# The replicated tables, minus the ones only data1 holds data for. Private
+# tables go to their own bucket: R2 tokens grant access per bucket, and
+# researchers may be given read access to the measurement tables.
+clickhouse_backup_jobs:
+  - name: measurements
+    s3_bucket: ooni-clickhouse-backup
+    include_tables: ["*"]
+    exclude_tables: "{{ clickhouse_backup_private_tables + clickhouse_backup_data1_tables }}"
+  - name: private
+    s3_bucket: ooni-clickhouse-backup-private
+    include_tables: "{{ clickhouse_backup_private_tables }}"
+    exclude_tables: []
+
+# After each run, report the tables on data1-3 that no backup covers;
+# data1's own tables are backed up by data1
+clickhouse_backup_check_cluster: oonidata_cluster
+clickhouse_backup_backed_up_elsewhere: "{{ clickhouse_backup_data1_tables }}"

ansible/roles/clickhouse_backup/defaults/main.yml

@@ -0,0 +1,78 @@
+---
+# Back up the ooni ClickHouse database to S3-compatible object storage
+# (Cloudflare R2) with ClickHouse's native BACKUP. Deploy where the data is:
+# data3 for the replicated tables, data1 for the tables only data1 holds
+# (see deploy-clickhouse.yml and host_vars).
+# Restore and upgrade procedure: docs/Runbooks.md, "ClickHouse backup".
+
+# false stops and disables the timer, e.g. while upgrading the cluster
+clickhouse_backup_enabled: true
+# Weekly, on Sundays at a quiet time (UTC): a full backup of everything takes
+# more than a day. A run backs up every job's tables, one job after the other;
+# a run still going delays the next one instead of overlapping it.
+clickhouse_backup_on_calendar: "Sun *-*-* 02:30:00"
+
+clickhouse_backup_database: ooni
+clickhouse_backup_clickhouse_url: "http://127.0.0.1:8123/"
+clickhouse_backup_clickhouse_user: "{{ clickhouse_admin_user }}"
+clickhouse_backup_clickhouse_password: "{{ clickhouse_admin_password }}"
+
+# Cloudflare R2: the endpoint names the account,
+# https://<account id>.r2.cloudflarestorage.com. One API token with Object
+# Read & Write on every bucket the jobs below use.
+clickhouse_backup_s3_endpoint: ""
+clickhouse_backup_s3_region: auto
+clickhouse_backup_s3_prefix: prod
+clickhouse_backup_s3_access_key_id: ""
+clickhouse_backup_s3_secret_access_key: ""
+
+# Query settings for ClickHouse's uploads. R2 rejects a multipart upload
+# whose parts (all but the last) differ in size, and ClickHouse grows the
+# part size as an upload goes on unless the factor is 1. R2 also allows at
+# most 10,000 parts, so with 128 MiB parts one file can reach 1.25 TiB.
+clickhouse_backup_s3_settings:
+  s3_upload_part_size_multiply_factor: 1
+  s3_min_upload_part_size: 134217728
+  s3_max_upload_part_size: 134217728
+
+# One job per bucket. A job backs up the tables matching include_tables ("*"
+# for every table; an empty list backs up none) minus the excludes below and
+# its own.
+clickhouse_backup_jobs:
+  - name: measurements
+    s3_bucket: ""
+    include_tables: ["*"]
+    exclude_tables: []
+
+# Complete chains kept per table, besides chains pinned with --keep: the
+# current one and the one before, so a month or two of restore points
+clickhouse_backup_keep_chains: 2
+# Bytes/s passed to ClickHouse as max_backup_bandwidth: at most 50 MB/s, 40%
+# of the 1 Gbit/s link, leaving the rest for replication and queries.
+# Against MinIO in testing, 24.8 uploaded at about a third of this value; check
+# the real rate on the first backups (ooni_clickhouse_backup_last_size_bytes /
+# ooni_clickhouse_backup_last_duration_seconds) and raise it if they're too slow.
+clickhouse_backup_max_bandwidth: 50000000
+
+# fnmatch patterns, for every job. Materialized views' .inner tables are
+# always skipped: they're backed up with their view.
+# Tables that are never backed up, for every job, and why
+clickhouse_backup_exclude_tables:
+  - "*_tmp"               # updater staging tables
+  - msmt_feedback_old
+  - fingerprints_dns      # the hourly fingerprints updater rebuilds them
+  - fingerprints_http
+  - session_expunge       # empty, no longer used (backend #972)
+
+# The cluster whose tables are checked after each run: any table on any of its
+# nodes that no job here backs up, that isn't excluded above and that isn't in
+# clickhouse_backup_backed_up_elsewhere is reported (ClickHouseTableNotBackedUp).
+# Set on one host, the one taking most backups; empty checks nothing.
+clickhouse_backup_check_cluster: ""
+# fnmatch patterns of the tables another host backs up
+clickhouse_backup_backed_up_elsewhere: []
+
+# Results for Prometheus, read by node_exporter's textfile collector
+# (node_exporter_textfile_dir in the prometheus_node_exporter role)
+clickhouse_backup_metrics_file: "{{ node_exporter_textfile_dir | default('/var/lib/node_exporter/textfile_collector') }}/ooni_clickhouse_backup.prom"
+clickhouse_backup_state_file: /var/lib/ooni-clickhouse-backup/state.json