diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b359ca0..45dc524 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -76,6 +76,9 @@ jobs: - name: Run tests shell: bash + # Bound the step: a hung extension download once stalled Windows jobs + # indefinitely (tests themselves finish in seconds). + timeout-minutes: 10 run: pytest tests/ -v # The abi3 wheel is identical across Python legs; keep just one for diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d341a7..ab860a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## UNRELEASED +### Added + +- Enabled the crate's `spatial` feature, so `DRAW spatial` map layers now render through the Vega-Lite writer instead of raising "Spatial visualization requires the 'spatial' feature to be enabled". Spatial layers require DuckDB's `spatial` extension at runtime: ggsql issues `LOAD spatial` automatically but never `INSTALL` it, so run `INSTALL spatial` once per machine (e.g. via `DuckDBReader.execute_sql()`; the download is cached under `~/.duckdb/extensions`). Works with WKB geometry columns and the built-in `ggsql:world` dataset, including named map projections such as `PROJECT TO equal_earth`. + ### Changed - Data now crosses the Python↔Rust boundary via the zero-copy Arrow C Data Interface instead of Arrow IPC serialization, reducing memory copies and peak RSS for large results (on a 1M-row × 50-col register+read microbenchmark: ~5.7× faster, ~1 GB lower peak RSS). Set `GGSQL_ARROW_IPC=1` to restore the previous IPC behavior. The minimum pyarrow version is now 15. diff --git a/Cargo.lock b/Cargo.lock index 60ea8de..f699651 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,7 +617,7 @@ dependencies = [ "comfy-table", "fallible-iterator", "fallible-streaming-iterator", - "hashlink", + "hashlink 0.10.0", "libduckdb-sys", "num", "num-integer", @@ -828,6 +828,43 @@ dependencies = [ "slab", ] +[[package]] +name = "geo-types" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" +dependencies = [ + "approx", + "num-traits", + "serde", +] + +[[package]] +name = "geojson" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e26f3c45b36fccc9cf2805e61d4da6bc4bbd5a3a9589b01afa3a40eff703bd79" +dependencies = [ + "log", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "geozero" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5f28f34864745eb2f123c990c6ffd92c1584bd39439b3f27ff2a0f4ea5b309b" +dependencies = [ + "geojson", + "log", + "scroll", + "serde_json", + "thiserror 1.0.69", + "wkt", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -881,10 +918,12 @@ dependencies = [ "csscolorparser", "duckdb", "fontique", + "geozero", "hephaestus", "palette", "rand 0.8.6", "regex", + "rusqlite", "serde", "serde_json", "sprintf", @@ -946,6 +985,15 @@ dependencies = [ "foldhash 0.1.5", ] +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", +] + [[package]] name = "hashbrown" version = "0.17.0" @@ -964,6 +1012,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -1468,6 +1525,17 @@ dependencies = [ "redox_syscall 0.7.4", ] +[[package]] +name = "libsqlite3-sys" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linebender_resource_handle" version = "0.1.1" @@ -2307,6 +2375,32 @@ dependencies = [ "memchr", ] +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror 2.0.18", +] + +[[package]] +name = "rusqlite" +version = "0.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1c93dd1c9683b438c392c492109cb702b8090b2bfc8fed6f6e4eb4523f17af3" +dependencies = [ + "bitflags", + "chrono", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink 0.11.1", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + [[package]] name = "rust_decimal" version = "1.41.0" @@ -2418,6 +2512,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scroll" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c565b551bafbef4157586fa379538366e4385d42082f255bfd96e4fe8519da" + [[package]] name = "seahash" version = "4.1.0" @@ -2561,6 +2661,18 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b2c760607300407ddeaee518acf28c795661b7108c75421303dbefb237d3a36" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3553,6 +3665,18 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wkt" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54f7f1ff4ea4c18936d6cd26a6fd24f0003af37e951a8e0e8b9e9a2d0bd0a46d" +dependencies = [ + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index 4f7c073..fc3a265 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] } arrow = { version = "58", default-features = false, features = ["ipc", "ffi"] } -ggsql = { version = "0.5.2", default-features = false, features = ["duckdb", "vegalite", "builtin-data", "svg", "pdf", "hep"] } +ggsql = { version = "0.5.2", default-features = false, features = ["duckdb", "vegalite", "builtin-data", "svg", "pdf", "hep", "spatial"] } [features] default = [] diff --git a/tests/test_ggsql.py b/tests/test_ggsql.py index a02acfc..3d47051 100644 --- a/tests/test_ggsql.py +++ b/tests/test_ggsql.py @@ -218,6 +218,57 @@ def test_render_multi_layer(self): assert "layer" in spec_dict +class TestSpatial: + """Tests for spatial (map) layers via the VegaLiteWriter.""" + + @pytest.fixture(scope="class", autouse=True) + def install_spatial_extension(self): + """Install DuckDB's spatial extension (the ggsql crate only LOADs it). + + The binary is cached per DuckDB version/platform under + ~/.duckdb/extensions, so this downloads at most once per machine. + """ + reader = ggsql.DuckDBReader("duckdb://memory") + try: + reader.execute_sql("INSTALL spatial") + except Exception as e: + pytest.skip(f"DuckDB spatial extension unavailable (offline?): {e}") + + def test_render_spatial_builtin_world(self): + reader = ggsql.DuckDBReader("duckdb://memory") + spec = reader.execute("VISUALISE FROM ggsql:world DRAW spatial") + writer = ggsql.VegaLiteWriter() + + result = writer.render(spec) + spec_dict = json.loads(result) + marks = [ + layer["mark"]["type"] if isinstance(layer["mark"], dict) else layer["mark"] + for layer in spec_dict["layer"] + ] + assert "geoshape" in marks + + def test_render_spatial_choropleth(self): + reader = ggsql.DuckDBReader("duckdb://memory") + spec = reader.execute( + "VISUALISE FROM ggsql:world DRAW spatial MAPPING population AS fill" + ) + writer = ggsql.VegaLiteWriter() + + result = writer.render(spec) + assert "geoshape" in result + + def test_render_chart_spatial(self): + reader = ggsql.DuckDBReader("duckdb://memory") + spec = reader.execute("VISUALISE FROM ggsql:world DRAW spatial") + writer = ggsql.VegaLiteWriter() + + chart = writer.render_chart(spec) + assert isinstance(chart, altair.TopLevelMixin) + # The chart survives Altair-level round-tripping (this is how the + # spec reaches the browser widget) + assert "projection" in chart.to_dict() + + class TestRenderAltairDataFrameConversion: """Tests for DataFrame handling in render_altair()."""