Main v14.0.0 beta.6

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "3.0.0-beta.2"
+current_version = "14.0.0-beta.6"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)(-(?P<prerelease>(beta|rc))\\.(?P<prerelease_num>\\d+))?"
 serialize = [
     "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}",
@@ -55,6 +55,11 @@ filename = "Cargo.toml"
 search = 'lance-datagen = {{ version = "={current_version}"'
 replace = 'lance-datagen = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-derive = {{ version = "={current_version}"'
+replace = 'lance-derive = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-encoding = {{ version = "={current_version}"'
@@ -75,6 +80,11 @@ filename = "Cargo.toml"
 search = 'lance-index = {{ version = "={current_version}"'
 replace = 'lance-index = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-index-core = {{ version = "={current_version}"'
+replace = 'lance-index-core = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'lance-io = {{ version = "={current_version}"'
@@ -92,13 +102,13 @@ replace = 'lance-namespace = {{ version = "={new_version}"'
 
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
-search = 'lance-namespace-datafusion = {{ version = "={current_version}"'
-replace = 'lance-namespace-datafusion = {{ version = "={new_version}"'
+search = 'lance-namespace-impls = {{ version = "={current_version}"'
+replace = 'lance-namespace-impls = {{ version = "={new_version}"'
 
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
-search = 'lance-namespace-impls = {{ version = "={current_version}"'
-replace = 'lance-namespace-impls = {{ version = "={new_version}"'
+search = 'lance-select = {{ version = "={current_version}"'
+replace = 'lance-select = {{ version = "={new_version}"'
 
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
@@ -115,6 +125,11 @@ filename = "Cargo.toml"
 search = 'lance-testing = {{ version = "={current_version}"'
 replace = 'lance-testing = {{ version = "={new_version}"'
 
+[[tool.bumpversion.files]]
+filename = "Cargo.toml"
+search = 'lance-tokenizer = {{ version = "={current_version}"'
+replace = 'lance-tokenizer = {{ version = "={new_version}"'
+
 [[tool.bumpversion.files]]
 filename = "Cargo.toml"
 search = 'fsst = {{ version = "={current_version}"'
@@ -145,4 +160,3 @@ replace = 'version = "{new_version}"'
 filename = "java/pom.xml"
 search = "<version>{current_version}</version>"
 replace = "<version>{new_version}</version>"
-

.cargo/config.toml

@@ -9,12 +9,27 @@ debug = true
 codegen-units = 16
 lto = "thin"
 
+[profile.release-no-lto]
+inherits = "release"
+debug = true
+lto = false
+# Prioritize compile time when LTO is not relevant to the measurement.
+codegen-units = 16
+
 [profile.bench]
 inherits = "release"
 lto = "thin"
 codegen-units = 16
 
 [target.x86_64-unknown-linux-gnu]
+# The default target is haswell.  This is an old enough target
+# that portability is high but a new enough target that we gain
+# some of the most common SIMD optimizations.
+#
+# On certain paths we use explicit SIMD and runtime dispatch to
+# opt-in to even higher CPU targets.  The target specified here
+# is for all code that is NOT using explicit SIMD (still the large
+# majority of code).
 rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"]
 
 [target.aarch64-apple-darwin]

.coderabbit.yaml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json
+language: en-US
+early_access: false
+reviews:
+  profile: quiet
+  high_level_summary: false
+  poem: false
+  review_status: true
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords:
+      - WIP
+      - Draft
+    drafts: false
+    base_branches:
+      - main

.gitattributes

@@ -1 +1,4 @@
-notebooks/** linguist-vendored
+notebooks/** linguist-vendored
+Cargo.lock linguist-generated
+**/Cargo.lock linguist-generated
+rust/lance-file/test_data/**/*.lance binary

.github/CODEOWNERS

@@ -5,6 +5,7 @@ docs/src/format/file/ @file-spec-team
 
 # table specification
 protos/table.proto @table-spec-team
+protos/fragment_metadata.proto @table-spec-team
 protos/rowids.proto @table-spec-team
 docs/src/format/table/ @table-spec-team
 

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,79 @@
+name: Bug report
+description: Report incorrect, unexpected, or crashing behavior.
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to file a bug report! Please search
+        [existing issues](https://github.com/lance-format/lance/issues) first to
+        avoid filing a duplicate.
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: >-
+        A minimal, self-contained code snippet or sequence of steps that
+        reproduces the problem. The more we can copy-paste and run, the faster
+        we can fix it.
+      placeholder: |
+        1. Write a dataset with `...`
+        2. Scan with filter `...`
+        3. Observe `...`
+      render: python
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What did you expect to happen?
+    validations:
+      required: false
+  - type: input
+    id: lance-version
+    attributes:
+      label: Lance version
+      description: >-
+        Output of `python -c "import lance; print(lance.__version__)"`, or the
+        `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`.
+      placeholder: "e.g. 0.40.0"
+    validations:
+      required: true
+  - type: dropdown
+    id: language
+    attributes:
+      label: Language binding
+      multiple: true
+      options:
+        - Python
+        - Rust
+        - Java
+        - Other / not sure
+    validations:
+      required: true
+  - type: input
+    id: environment
+    attributes:
+      label: Environment
+      description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...).
+      placeholder: "e.g. Ubuntu 22.04, x86_64, S3"
+    validations:
+      required: false
+  - type: textarea
+    id: logs
+    attributes:
+      label: Logs / traceback
+      description: >-
+        Any relevant log output or stack trace. This will be automatically
+        formatted as code, so no need for backticks.
+      render: shell
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Question / usage help
+    url: https://discord.gg/lance
+    about: >-
+      For questions and general discussion, please ask in the Lance Discord
+      rather than opening an issue.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: Feature request
+about: Suggest a new capability or an improvement to an existing one.
+labels: feature
+---
+
+<!--
+Describe what you'd like Lance to do and why. There's no required format —
+mention the problem or use case, and a rough idea of the solution if you have
+one. Please search existing issues first to avoid duplicates.
+-->

.github/ISSUE_TEMPLATE/performance_issue.md

@@ -0,0 +1,12 @@
+---
+name: Performance issue
+about: Report slow operations, high memory use, or a performance regression.
+labels: performance
+---
+
+<!--
+Describe what's slow or memory-hungry. There's no required format, but concrete
+numbers (timings, memory, dataset shape) and a way to reproduce help a lot. If
+it's a regression, note the last version that performed well. Please search
+existing issues first to avoid duplicates.
+-->

.github/actions/setup-release-env/action.yml

@@ -4,7 +4,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
       with:
         python-version: "3.11"
 
@@ -14,7 +14,7 @@ runs:
         pip install bump-my-version packaging PyGithub PyYAML
 
     - name: Set up Rust
-      uses: actions-rs/toolchain@v1
+      uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1
       with:
         toolchain: stable
         override: true

.github/dependabot.yml

@@ -0,0 +1,91 @@
+version: 2
+updates:
+  # Rust — one grouped version-bump PR per directory each week, and security
+  # advisories that land together in a directory batched into a single PR
+  # instead of one PR per advisory.
+  - package-ecosystem: "cargo"
+    directories:
+      - "/"
+      - "/python"
+      - "/java/lance-jni"
+    versioning-strategy: lockfile-only
+    # Matches the 48h floor enforced by ci/check_dependency_age.py, so grouped
+    # bumps do not land a version that CI will then reject. Cooldown does not
+    # apply to security updates, which we still want as fast as possible.
+    cooldown:
+      default-days: 2
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      cargo:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      cargo-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Python (uv) — same weekly-grouped + grouped-security treatment.
+  - package-ecosystem: "uv"
+    directory: "/python"
+    versioning-strategy: lockfile-only
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      uv:
+        applies-to: version-updates
+        patterns:
+          - "*"
+      uv-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Standalone Rust lockfiles for dev/test fixtures. Dependabot security updates
+  # scan these regardless of config, so we group them; open-pull-requests-limit: 0
+  # disables the (unwanted) weekly version bumps for these crates.
+  - package-ecosystem: "cargo"
+    directories:
+      - "/memtest"
+      - "/test_data/fri_straddle_pre_6610/datagen"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      cargo-fixtures-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Java (Maven). Security updates fire regardless of config; group them and
+  # skip version bumps (limit 0) — we bump Java deps manually.
+  - package-ecosystem: "maven"
+    directory: "/java"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      maven-security:
+        applies-to: security-updates
+        patterns:
+          - "*"
+
+  # Python benchmark scripts (pip / requirements.txt). Group security updates
+  # and skip version bumps (limit 0).
+  - package-ecosystem: "pip"
+    directories:
+      - "/benchmarks/*"
+    open-pull-requests-limit: 0
+    schedule:
+      interval: "weekly"
+      day: "wednesday"
+    groups:
+      pip-security:
+        applies-to: security-updates
+        patterns:
+          - "*"