diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4db37dc..3ecda10 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: sudo apt-get -qq update sudo apt-get install -y clang-format cppcheck python3-pip python3-venv python3 -m venv "$HOME/.lint-venv" - "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS black flake8 pylint + "$HOME/.lint-venv/bin/pip" install pre-commit $PIP_DEPS ruff echo "$HOME/.lint-venv/bin" >> "$GITHUB_PATH" - name: pre-commit run: pre-commit run --all-files diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index abf7065..08a96db 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,26 +17,15 @@ repos: verbose: true entry: cppcheck args: [-q,--std=c++11] - - id: black - name: black + - id: ruff-check + name: ruff-check language: system - files: \.py$ + types: [python] verbose: true - entry: black - args: [-l,'100'] - - id: flake8 - name: flake8 + entry: ruff check + - id: ruff-format + name: ruff-format language: system - files: \.py$ + types: [python] verbose: true - entry: flake8 - args: [--max-line-length, "100", "--ignore=E501,W503,E722"] - - id: pylint - name: pylint - language: system - files: \.py$ - exclude: version\.py - verbose: true - entry: env PYTHONPATH=bindings/python pylint - args: [-d, "global-statement,missing-docstring,missing-module-docstring,line-too-long,too-many-arguments,too-many-positional-arguments,duplicate-code,redefined-outer-name,too-many-locals,bare-except,broad-except,import-outside-toplevel,protected-access,too-many-statements,too-many-branches,unspecified-encoding"] -# CI re-run trigger 2026-10-19 (run 39294337754 lost its macOS runner to GitHub capacity) + entry: ruff format diff --git a/CMakeLists.txt b/CMakeLists.txt index d095a63..067b952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,13 +3,6 @@ if(${CMAKE_VERSION} VERSION_LESS 3.12) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) endif() -# CMake 4 removed compatibility with projects declaring -# cmake_minimum_required() below 3.5; the pinned SQLiteCpp fetched -# below declares 3.1, which makes configure fail outright under -# CMake 4 (e.g. the macos-15 runner image). Raise the policy-version -# floor for fetched subprojects; harmless under older CMake. -set(CMAKE_POLICY_VERSION_MINIMUM 3.5) - include(FetchContent) if(${CMAKE_VERSION} VERSION_LESS 3.14) @@ -25,7 +18,7 @@ endif() FetchContent_Declare( sqlite_zstd_vfs GIT_REPOSITORY https://github.com/mlin/sqlite_zstd_vfs.git - GIT_TAG 2e9c7db + GIT_TAG 42257288a5f5d025d79967f8c2ec48f534cf8559 ) FetchContent_MakeAvailable(sqlite_zstd_vfs) FetchContent_MakeAvailable(sqlitecpp) diff --git a/Dockerfile b/Dockerfile index faec530..cd2ff47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM almalinux:8 as builder ARG CMAKE_VERSION=3.17.3 -ARG SQLITE_VERSION=3410200 +ARG SQLITE_VERSION=3540100 ARG ZSTD_VERSION=1.5.5 ARG CPU_ARCH=ivybridge ENV CFLAGS="-march=${CPU_ARCH} -O3" @@ -34,7 +34,7 @@ RUN mkdir -p /work/GenomicSQLite # SQLite WORKDIR /work -RUN wget -nv http://www.sqlite.org/2023/sqlite-amalgamation-${SQLITE_VERSION}.zip \ +RUN wget -nv http://www.sqlite.org/2026/sqlite-amalgamation-${SQLITE_VERSION}.zip \ && unzip -o sqlite-amalgamation-${SQLITE_VERSION}.zip WORKDIR /work/sqlite-amalgamation-${SQLITE_VERSION} RUN bash -c "gcc -shared -o libsqlite3.so.0 -fPIC -shared -Wl,-soname,libsqlite3.so.0 -g ${CFLAGS} ${SQLITE_CFLAGS} sqlite3.c & pid=$? \ diff --git a/README.md b/README.md index 352593f..f31a516 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ cmake --build build -j 4 --target genomicsqlite * htslib ≥ 1.9, samtools, and tabix * pigz -* Python ≥ 3.6 and packages: pytest pytest-xdist pre-commit black pylint flake8 +* Python ≥ 3.6 and packages: pytest pytest-xdist pre-commit ruff * JDK, mvn, rust * clang-format & cppcheck diff --git a/bindings/jdbc/genomicsqlite-jdbc/pom.xml b/bindings/jdbc/genomicsqlite-jdbc/pom.xml index 63b4894..b7b3646 100644 --- a/bindings/jdbc/genomicsqlite-jdbc/pom.xml +++ b/bindings/jdbc/genomicsqlite-jdbc/pom.xml @@ -61,7 +61,7 @@ junit junit - 4.13.1 + 4.13.2 test @@ -72,44 +72,44 @@ maven-clean-plugin - 3.1.0 + 3.5.0 maven-resources-plugin - 3.0.2 + 3.5.0 maven-compiler-plugin - 3.8.0 + 3.16.0 maven-surefire-plugin - 2.22.2 + 3.6.0 false maven-jar-plugin - 3.0.2 + 3.5.1 maven-install-plugin - 2.5.2 + 3.2.0 maven-deploy-plugin - 2.8.2 + 3.2.0 maven-site-plugin - 3.7.1 + 3.22.0 maven-project-info-reports-plugin - 3.0.0 + 3.9.0 com.diffplug.spotless diff --git a/bindings/python/genomicsqlite/__init__.py b/bindings/python/genomicsqlite/__init__.py index c1ec17e..22504bf 100755 --- a/bindings/python/genomicsqlite/__init__.py +++ b/bindings/python/genomicsqlite/__init__.py @@ -3,12 +3,12 @@ GenomicSQLite Python binding """ +import json import os -import sys import platform -import json import sqlite3 -from typing import Optional, NamedTuple, Dict, Any +import sys +from typing import Any, Dict, NamedTuple, Optional # noqa: UP035 _HERE = os.path.dirname(__file__) _YES = ("1", "true", "t", "yes", "y") @@ -182,9 +182,9 @@ def get_reference_sequences_by_rid( params = (assembly,) ans = {} for row in con.execute(sql, params): - assert ( - isinstance(row[0], int) and row[0] not in ans - ), "genomicsqlite: invalid or duplicate reference sequence rid" + assert isinstance(row[0], int) and row[0] not in ans, ( + "genomicsqlite: invalid or duplicate reference sequence rid" + ) ans[row[0]] = ReferenceSequence( rid=row[0], name=row[1], @@ -200,7 +200,7 @@ def get_reference_sequences_by_name( con: sqlite3.Connection, assembly: Optional[str] = None, schema: Optional[str] = None ) -> Dict[str, ReferenceSequence]: ans = {} - for _, item in get_reference_sequences_by_rid(con, assembly, schema).items(): + for item in get_reference_sequences_by_rid(con, assembly, schema).values(): assert item.name not in ans, "genomicsqlite: non-unique reference sequence names" ans[item.name] = item return ans @@ -333,12 +333,10 @@ def _cli(argv=None): print( " ".join( ( - ( - (arg if " " not in arg else f"'{arg}'") - + (" \\\n " if len(arg) > 50 else "") - ) - for arg in cmd + (arg if " " not in arg else f"'{arg}'") + + (" \\\n " if len(arg) > 50 else "") ) + for arg in cmd ) ) os.execvp("sqlite3", cmd) @@ -403,7 +401,7 @@ def _compact(dbfilename, argv): # sniff whether input db is currently compressed with zstd_vfs or not con = None - web = dbfilename.startswith("http:") or dbfilename.startswith("https:") + web = dbfilename.startswith(("http:", "https:")) if not web: con = sqlite3.connect(f"file:{urllib.parse.quote(dbfilename)}?mode=ro", uri=True) if next(con.execute("PRAGMA application_id"))[0] != 0x7A737464: diff --git a/bindings/python/genomicsqlite/__main__.py b/bindings/python/genomicsqlite/__main__.py index ba91ed0..bd19d9c 100644 --- a/bindings/python/genomicsqlite/__main__.py +++ b/bindings/python/genomicsqlite/__main__.py @@ -1,4 +1,5 @@ import sys + from . import _cli if __name__ == "__main__": diff --git a/bindings/python/setup.py b/bindings/python/setup.py index b9fee7b..cfd8a19 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from os import path -from setuptools import setup, find_packages # pylint: disable=import-error + +from setuptools import find_packages, setup # pylint: disable=import-error from version import get_version HERE = path.abspath(path.dirname(__file__)) diff --git a/bindings/python/version.py b/bindings/python/version.py index 5b7cfd8..964f594 100644 --- a/bindings/python/version.py +++ b/bindings/python/version.py @@ -76,7 +76,7 @@ def read_git_version(): return None match = re.search(_GIT_DESCRIPTION_RE, ver) if not match: - sys.stderr.write("version: git description (%s) is invalid, " "ignoring\n" % ver) + sys.stderr.write("version: git description (%s) is invalid, ignoring\n" % ver) return None commits = int(match.group("commits")) @@ -90,9 +90,7 @@ def read_release_version(): with open(RELEASE_VERSION_FILE) as infile: ver = infile.readline().strip() if not re.search(_PEP386_VERSION_RE, ver): - sys.stderr.write( - "version: release version (%s) is invalid, " "will use it anyway\n" % ver - ) + sys.stderr.write("version: release version (%s) is invalid, will use it anyway\n" % ver) return ver except: return None diff --git a/docs/guide_db.md b/docs/guide_db.md index 7b8abea..ac7ad07 100644 --- a/docs/guide_db.md +++ b/docs/guide_db.md @@ -94,9 +94,9 @@ First complete the [installation instructions](index.md). === "Python" ``` python3 dbconn = genomicsqlite.connect( - db_filename, - read_only=False, - **kwargs # genomicsqlite + sqlite3.connect() arguments + db_filename, + read_only=False, + **kwargs, # genomicsqlite + sqlite3.connect() arguments ) assert isinstance(dbconn, sqlite3.Connection) ``` diff --git a/docs/guide_gri.md b/docs/guide_gri.md index 8bf7f66..747e0db 100644 --- a/docs/guide_gri.md +++ b/docs/guide_gri.md @@ -21,11 +21,7 @@ The extension provides routines to populate a small `_gri_refseq` table describi === "Python" ``` python3 create_gri_sql = genomicsqlite.create_genomic_range_index_sql( - dbconn, - 'tableName', - 'chromosome', - 'beginPosition', - 'endPosition' + dbconn, "tableName", "chromosome", "beginPosition", "endPosition" ) dbconn.executescript(create_gri_sql) ``` @@ -229,9 +225,7 @@ The following routines support the aforementioned, recommended convention for st === "Python" ``` python3 - refseq_sql = genomicsqlite.put_reference_assembly_sql( - dbconn, 'GRCh38_no_alt_analysis_set' - ) + refseq_sql = genomicsqlite.put_reference_assembly_sql(dbconn, "GRCh38_no_alt_analysis_set") dbconn.executescript(refseq_sql) ``` @@ -307,8 +301,10 @@ Available assemblies: === "Python" ``` python3 refseq_sql = genomicsqlite.put_reference_sequence_sql( - dbconn, 'chr17', 83257441 - # optional: assembly, refget_id, meta (dict), rid + dbconn, + "chr17", + 83257441, + # optional: assembly, refget_id, meta (dict), rid ) dbconn.executescript(refseq_sql) ``` @@ -407,12 +403,13 @@ If the `rid` argument is omitted or -1 then it will be assigned automatically up === "Python" ``` python3 class ReferenceSequence(NamedTuple): - rid: int - name: str - length: int - assembly: Optional[str] - refget_id: Optional[str] - meta: Dict[str, Any] + rid: int + name: str + length: int + assembly: Optional[str] + refget_id: Optional[str] + meta: Dict[str, Any] + refseq_by_rid = genomicsqlite.get_reference_sequences_by_rid(dbconn) # refseq_by_rid: Dict[int, ReferenceSequence] @@ -492,12 +489,13 @@ The optional `assembly` argument restricts the retrieved sequences to those with === "Python" ``` python3 class ReferenceSequence(NamedTuple): - rid: int - name: str - length: int - assembly: Optional[str] - refget_id: Optional[str] - meta: Dict[str, Any] + rid: int + name: str + length: int + assembly: Optional[str] + refget_id: Optional[str] + meta: Dict[str, Any] + refseq_by_name = genomicsqlite.get_reference_sequences_by_name(dbconn) # refseq_by_name: Dict[str, ReferenceSequence] diff --git a/docs/guide_helpers.md b/docs/guide_helpers.md index d1cd9a7..95bc3e3 100644 --- a/docs/guide_helpers.md +++ b/docs/guide_helpers.md @@ -88,8 +88,8 @@ The [UINT collating sequence](https://www.sqlite.org/uintcseq.html) is also bund === "Python" ``` python3 - dbconn = sqlite3.connect('any.db', uri=True) - attach_sql = genomicsqlite.attach_sql(dbconn, 'compressed.db', 'db2') + dbconn = sqlite3.connect("any.db", uri=True) + attach_sql = genomicsqlite.attach_sql(dbconn, "compressed.db", "db2") # attach_sql() also takes configuration keyword arguments like # genomicsqlite.connect() dbconn.executescript(attach_sql) @@ -190,13 +190,13 @@ The [UINT collating sequence](https://www.sqlite.org/uintcseq.html) is also bund === "Python" ``` python3 - dbconn = sqlite3.connect('existing.db', uri=True) - vacuum_sql = genomicsqlite.vacuum_into_sql(dbconn, 'compressed.db') + dbconn = sqlite3.connect("existing.db", uri=True) + vacuum_sql = genomicsqlite.vacuum_into_sql(dbconn, "compressed.db") # vacuum_into_sql() also takes configuration keyword arguments like # genomicsqlite.connect() to control compression level & page sizes dbconn.executescript(vacuum_sql) - dbconn2 = genomicsqlite.connect('compressed.db') + dbconn2 = genomicsqlite.connect("compressed.db") ``` === "Java" diff --git a/docs/internals.md b/docs/internals.md index df34720..01dd569 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -70,13 +70,19 @@ The extension also exposes the routine to generate the SQL subquery, which you c === "Python" ``` python3 query = ( - 'SELECT * FROM tableName WHERE tableName._rowid_ IN ' + - genomicsqlite.genomic_range_rowids_sql(dbconn, 'tableName', - # defaults: - qrid='?1', qbeg='?2', qend='?3', - ceiling=-1, floor=-1) + "SELECT * FROM tableName WHERE tableName._rowid_ IN " + + genomicsqlite.genomic_range_rowids_sql( + dbconn, + "tableName", + # defaults: + qrid="?1", + qbeg="?2", + qend="?3", + ceiling=-1, + floor=-1, + ) ) - cursor = dbconn.execute(query, ('chr12',111803912,111804012)) + cursor = dbconn.execute(query, ("chr12", 111803912, 111804012)) ``` === "SQLiteCpp" diff --git a/loaders/common.hpp b/loaders/common.hpp index 99eca19..559fa80 100644 --- a/loaders/common.hpp +++ b/loaders/common.hpp @@ -5,6 +5,7 @@ #include "strlcpy.h" #include #include +#include #include #include #include diff --git a/loaders/sam_into_sqlite.cc b/loaders/sam_into_sqlite.cc index a210537..1788b35 100644 --- a/loaders/sam_into_sqlite.cc +++ b/loaders/sam_into_sqlite.cc @@ -304,7 +304,7 @@ int main(int argc, char *argv[]) { throw runtime_error("invalid header SQ"); } errno = 0; - sqlite3_int64 length = strtoll(ks->s, nullptr, 10); + int64_t length = strtoll(ks->s, nullptr, 10); if (errno || length <= 0) { throw runtime_error("invalid header SQ"); } @@ -391,7 +391,7 @@ int main(int argc, char *argv[]) { } insert_read.exec(); - sqlite3_int64 rowid = db->getLastInsertRowid(); + int64_t rowid = db->getLastInsertRowid(); insert_seqs.reset(); insert_seqs.clearBindings(); diff --git a/loaders/vcf_into_sqlite.cc b/loaders/vcf_into_sqlite.cc index 359f469..ba9480d 100644 --- a/loaders/vcf_into_sqlite.cc +++ b/loaders/vcf_into_sqlite.cc @@ -157,7 +157,7 @@ void insert_samples(bcf_hdr_t *hdr, const string &table_prefix, SQLite::Database SQLite::Statement stmt(db, "INSERT INTO " + table_prefix + "samples(sample_id,sample_name) VALUES(?,?)"); - for (sqlite3_int64 i = 0; i < bcf_hdr_nsamples(hdr); ++i) { + for (int64_t i = 0; i < bcf_hdr_nsamples(hdr); ++i) { stmt.bind(1, i); stmt.bind(2, hdr->samples[i]); stmt.exec(); @@ -486,12 +486,12 @@ unique_ptr format_helper::make(bcf_hdr_t *hdr, bcf1_t *rec, } void insert_genotypes(bcf_hdr_t *hdr, bcf1_t *rec, vector> &format_hrecs, - int ploidy, sqlite3_int64 variant_rowid, SQLite::Statement &stmt) { + int ploidy, int64_t variant_rowid, SQLite::Statement &stmt) { vector> format_helpers; for (auto &hrec : format_hrecs) { format_helpers.push_back(format_helper::make(hdr, rec, hrec, ploidy)); } - for (sqlite3_int64 sample = 0; sample < bcf_hdr_nsamples(hdr); ++sample) { + for (int64_t sample = 0; sample < bcf_hdr_nsamples(hdr); ++sample) { stmt.reset(); stmt.clearBindings(); stmt.bind(1, variant_rowid); @@ -697,7 +697,7 @@ int main(int argc, char *argv[]) { "unexpected: VCF header contig lines reference multiple assemblies"); } errno = 0; - sqlite3_int64 length = strtoull(ctg["length"].c_str(), nullptr, 10); + int64_t length = strtoull(ctg["length"].c_str(), nullptr, 10); if (length <= 0 || errno) { throw runtime_error("invalid contig length in VCF header"); } diff --git a/loaders/vcf_lines_into_sqlite.cc b/loaders/vcf_lines_into_sqlite.cc index f296132..7d13db3 100644 --- a/loaders/vcf_lines_into_sqlite.cc +++ b/loaders/vcf_lines_into_sqlite.cc @@ -24,7 +24,7 @@ void InsertLine(const string &line, SQLite::Statement &stmt) { // POS errno = 0; - sqlite3_int64 pos = strtoll(fields[1], nullptr, 10); + int64_t pos = strtoll(fields[1], nullptr, 10); if (errno || pos < 1) { throw runtime_error("invalid POS " + string(fields[1])); } @@ -33,11 +33,11 @@ void InsertLine(const string &line, SQLite::Statement &stmt) { // rlen if (strncmp(fields[7], "END=", 4)) { - stmt.bind(3, sqlite3_int64(strlen(fields[3]))); + stmt.bind(3, int64_t(strlen(fields[3]))); } else { vector info; split(fields[7], ';', back_inserter(info), 2); - sqlite3_int64 end = strtoll(info[0] + 4, nullptr, 10); + int64_t end = strtoll(info[0] + 4, nullptr, 10); if (errno || end < pos) { throw runtime_error("invalid END " + string(fields[4])); } diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..440afff --- /dev/null +++ b/ruff.toml @@ -0,0 +1,17 @@ +# Ruff: run on ruff's own defaults (project lint policy, 2026-10-20). +# Opt out of specific rules only where warranted; each opt-out is documented +# inline. No rule sets are selected or opted into. +line-length = 100 # exception to the ruff default (88): matches this codebase's black -l 100 formatting; avoids a whole-tree reformat + +[lint] +ignore = [ + "UP006", # non-pep585-annotation: binding annotations are evaluated at runtime; package declares python_requires>=3.6 + "UP045", # non-pep604-annotation-optional: same runtime-annotation / python>=3.6 constraint as UP006 +] + +[lint.per-file-ignores] +# bindings/python/version.py is vendored (third-party, public domain) and not +# hand-edited; the rules below fire only there. Scoping them to that file +# keeps them enforced on all hand-written code (which has no bare excepts, +# so restoring E722 enforcement there costs nothing). +"bindings/python/version.py" = ["E722", "UP031", "UP009", "PLE0605"] diff --git a/src/SQLiteVirtualTable.hpp b/src/SQLiteVirtualTable.hpp index 7c38264..e319098 100644 --- a/src/SQLiteVirtualTable.hpp +++ b/src/SQLiteVirtualTable.hpp @@ -8,6 +8,7 @@ #pragma once #include +#include #include #include #include @@ -35,7 +36,7 @@ class SQLiteVirtualTableCursor { // access current row virtual int Column(sqlite3_context *ctx, int colno) { return SQLITE_ERROR; } - virtual int Rowid(sqlite_int64 *pRowid) { return SQLITE_ERROR; } + virtual int Rowid(int64_t *pRowid) { return SQLITE_ERROR; } SQLiteVirtualTableCursor() { memset(&handle_, 0, sizeof(Handle)); @@ -246,8 +247,15 @@ template int RegisterSQLiteVirtualTable(sqlite3 *db, const cha ctx, colno)); }; p->xRowid = [](sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid) noexcept { - __STUB_CATCH( - reinterpret_cast(pCursor)->that->Rowid(pRowid)); + __STUB_CATCH(([&]() { + int64_t rowid; + int ans = + reinterpret_cast(pCursor)->that->Rowid(&rowid); + if (ans == SQLITE_OK) { + *pRowid = rowid; + } + return ans; + })()); }; #undef __STUB_CATCH return sqlite3_create_module_v2(db, zName, p, p, diff --git a/src/genomicsqlite.cc b/src/genomicsqlite.cc index 63ea5ee..f945808 100644 --- a/src/genomicsqlite.cc +++ b/src/genomicsqlite.cc @@ -538,7 +538,7 @@ extern "C" char *create_genomic_range_index_sql(const char *table, const char *r static void sqlfn_create_genomic_range_index_sql(sqlite3_context *ctx, int argc, sqlite3_value **argv) { string schema_table, rid, beg, end; - sqlite3_int64 floor = -1; + int64_t floor = -1; assert(argc == 4 || argc == 5); ARG_TEXT(schema_table, 0) ARG_TEXT(rid, 1) @@ -593,15 +593,15 @@ static pair DetectLevelRange(sqlite3 *dbconn, const string &schema_tab stmt = shared_ptr(pStmt, sqlite3_finalize); } - sqlite3_int64 min_lvl = 15, max_lvl = 0; + int64_t min_lvl = 15, max_lvl = 0; int rc; while ((rc = sqlite3_step(stmt.get())) == SQLITE_ROW) { // un-negating as we go if (sqlite3_column_type(stmt.get(), 0) == SQLITE_INTEGER) { - max_lvl = max(max_lvl, 0 - sqlite3_column_int64(stmt.get(), 0)); + max_lvl = max(max_lvl, 0 - sqlite3_column_int64(stmt.get(), 0)); } if (sqlite3_column_type(stmt.get(), 1) == SQLITE_INTEGER) { - min_lvl = min(min_lvl, 0 - sqlite3_column_int64(stmt.get(), 1)); + min_lvl = min(min_lvl, 0 - sqlite3_column_int64(stmt.get(), 1)); } } if (rc != SQLITE_DONE) { @@ -669,7 +669,7 @@ extern "C" char *genomic_range_rowids_sql(sqlite3 *dbconn, const char *table, co static void sqlfn_genomic_range_rowids_sql(sqlite3_context *ctx, int argc, sqlite3_value **argv) { string indexed_table, qrid = "?1", qbeg = "?2", qend = "?3"; - sqlite3_int64 ceiling = -1, floor = -1; + int64_t ceiling = -1, floor = -1; assert(argc >= 1 && argc <= 6); ARG_TEXT(indexed_table, 0) ARG_TEXT_OPTIONAL(qrid, 1) @@ -810,7 +810,7 @@ class GenomicRangeRowidsCursor : public SQLiteVirtualTableCursor { return SQLITE_OK; } - int Rowid(sqlite_int64 *pRowid) override { + int Rowid(int64_t *pRowid) override { assert(stmt_); *pRowid = sqlite3_column_int64(stmt_.get(), 0); return SQLITE_OK; @@ -968,7 +968,7 @@ class GenomicRangeIndexLevelsCursor : public SQLiteVirtualTableCursor { return SQLITE_OK; } - int Rowid(sqlite_int64 *pRowid) override { + int Rowid(int64_t *pRowid) override { assert(floor_ >= 0); *pRowid = 1; return SQLITE_OK; @@ -977,7 +977,7 @@ class GenomicRangeIndexLevelsCursor : public SQLiteVirtualTableCursor { private: sqlite3 *db_; levels_cache &cache_; - sqlite_int64 ceiling_ = -1, floor_ = -1; + int64_t ceiling_ = -1, floor_ = -1; }; class GenomicRangeIndexLevelsTVF : public SQLiteVirtualTable { @@ -1069,7 +1069,7 @@ extern "C" char *put_genomic_reference_sequence_sql(const char *name, sqlite3_in static void sqlfn_put_genomic_reference_sequence_sql(sqlite3_context *ctx, int argc, sqlite3_value **argv) { string name, assembly, refget_id, meta_json = "{}", schema; - sqlite3_int64 length, rid = -1; + int64_t length, rid = -1; assert(argc >= 2 && argc <= 7); ARG_TEXT(name, 0) ARG(length, 1, SQLITE_INTEGER, int64) diff --git a/test/MavenSmokeTest/pom.xml b/test/MavenSmokeTest/pom.xml index 6b93186..bd6c08e 100644 --- a/test/MavenSmokeTest/pom.xml +++ b/test/MavenSmokeTest/pom.xml @@ -14,8 +14,8 @@ UTF-8 - 1.7 - 1.7 + 1.8 + 1.8 @@ -39,41 +39,41 @@ maven-clean-plugin - 3.1.0 + 3.5.0 maven-resources-plugin - 3.0.2 + 3.5.0 maven-compiler-plugin - 3.8.0 + 3.16.0 maven-surefire-plugin - 2.22.1 + 3.6.0 maven-jar-plugin - 3.0.2 + 3.5.1 maven-install-plugin - 2.5.2 + 3.2.0 maven-deploy-plugin - 2.8.2 + 3.2.0 maven-site-plugin - 3.7.1 + 3.22.0 maven-project-info-reports-plugin - 3.0.0 + 3.9.0 diff --git a/test/check_elf_version_floor.py b/test/check_elf_version_floor.py index 04a9e76..0e65b97 100644 --- a/test/check_elf_version_floor.py +++ b/test/check_elf_version_floor.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """ Assert that ELF shared objects' required symbol versions stay at or below the project's compatibility ceilings -- the manylinux_2_28 floor asserted by CI for diff --git a/test/genomicsqlite_smoke_test.py b/test/genomicsqlite_smoke_test.py index 9baf123..1652d46 100755 --- a/test/genomicsqlite_smoke_test.py +++ b/test/genomicsqlite_smoke_test.py @@ -4,13 +4,13 @@ small test database and check GRI query plan on it. """ -import sys -import time import os -import subprocess -import traceback import platform +import subprocess +import sys import textwrap +import time +import traceback def main(): @@ -36,9 +36,8 @@ def main(): sysctl = subprocess.run( ["sysctl", "-n", "machdep.cpu.brand_string"], check=True, - universal_newlines=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + text=True, + capture_output=True, ) if sysctl.returncode == 0 and sysctl.stdout.strip(): print(": " + sysctl.stdout.strip()) @@ -46,7 +45,7 @@ def main(): print(": ???") else: print(": ???") - except Exception: + except Exception: # noqa: BLE001 print(": ???") env_keys = [ @@ -65,7 +64,7 @@ def main(): conn = sqlite3.connect(":memory:") print(next(conn.execute("SELECT sqlite_version()"))[0]) - sqlite_opts = list(row[0] for row in conn.execute("PRAGMA compile_options")) + sqlite_opts = [row[0] for row in conn.execute("PRAGMA compile_options")] print(f" {' '.join(sqlite_opts)}") print("genomicsqlite: ", end="") @@ -126,9 +125,9 @@ def main(): control = "SELECT exons1.id, exons2.id FROM exons1 LEFT JOIN exons2 NOT INDEXED ON NOT (exons2.end < exons1.pos OR exons2.pos > exons1.end) ORDER BY exons1.id, exons2.id" control_results = list(dbconn.execute(control)) assert len(control_results) == 1139, "unexpected control query results" - assert ( - results == control_results - ), "discrepancy between genomic_range_rowids_sql() and control results" + assert results == control_results, ( + "discrepancy between genomic_range_rowids_sql() and control results" + ) if gri_constraints != 3: print("GRI query opcodes:") @@ -154,9 +153,9 @@ def main(): genomic_range_rowids('exons2', exons1.chrom, exons1.pos, exons1.end, _gri_ceiling, _gri_floor) ORDER BY exons1.id, exons2.id """ - assert ( - list(dbconn.execute(query2)) == control_results - ), "discrepancy between genomic_range_rowids() and control results" + assert list(dbconn.execute(query2)) == control_results, ( + "discrepancy between genomic_range_rowids() and control results" + ) dbconn.close() print("\nGenomicSQLite smoke test OK =)\n") @@ -525,7 +524,7 @@ def main(): try: main() -except Exception: +except Exception: # noqa: BLE001 print() traceback.print_exc(file=sys.stdout) sys.exit(1) diff --git a/test/test_bam.py b/test/test_bam.py index 0748485..6d39c2b 100644 --- a/test/test_bam.py +++ b/test/test_bam.py @@ -1,6 +1,7 @@ import os -import subprocess import sqlite3 +import subprocess + import genomicsqlite HERE = os.path.dirname(__file__) @@ -55,10 +56,12 @@ def test_bam(tmp_path): count = next(con.execute("SELECT COUNT(DISTINCT qname) FROM NA12878_reads_seqs"))[0] assert count == 299205 - mq_hist = dict(con.execute(""" + mq_hist = dict( + con.execute(""" SELECT mq, COUNT(*) as count FROM (SELECT ifnull(json_extract(tags_json, '$.MQ'),0) AS mq FROM NA12878_reads NATURAL JOIN NA12878_reads_tags WHERE (flag & 3840) = 0) GROUP BY mq ORDER BY mq DESC - """)) + """) + ) assert (mq_hist[0], mq_hist[60]) == (2734, 520522) diff --git a/test/test_gri.py b/test/test_gri.py index 3ab2cda..9911d92 100644 --- a/test/test_gri.py +++ b/test/test_gri.py @@ -1,8 +1,9 @@ import os -import sqlite3 import random -import pytest +import sqlite3 + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build")) @@ -62,13 +63,13 @@ def test_indexing(): # for expl in opcodes: # if expl[1] in {"OpenRead", "OpenPseudo", "Column"}: # print(expl) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 # get from idx assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 # get from table @@ -79,8 +80,8 @@ def test_indexing(): for _ in range(50000): beg = random.randint(43044294 - 10000, 43044294 + 10000) end = beg + random.randint(1, random.choice([10, 100, 1000, 10000])) - ids = list(row[0] for row in con.execute(query, ("chr17", beg, end))) - control_ids = list(row[0] for row in con.execute(control_query, ("chr17", beg, end))) + ids = [row[0] for row in con.execute(query, ("chr17", beg, end))] + control_ids = [row[0] for row in con.execute(control_query, ("chr17", beg, end))] assert ids == control_ids total_results += len(control_ids) assert total_results == 189935 @@ -380,14 +381,12 @@ def test_query_in_sql(tmp_path): assert "USE TEMP B-TREE FOR ORDER BY" not in expl[3] assert next( - ( - expl[3] - for expl in con.execute( - "EXPLAIN QUERY PLAN SELECT _rowid_ FROM genomic_range_rowids('exons',?,?,?) ORDER BY _rowid_ DESC", - ("chr17", 43044294, 43048294), - ) - if "USE TEMP B-TREE FOR ORDER BY" in expl[3] + expl[3] + for expl in con.execute( + "EXPLAIN QUERY PLAN SELECT _rowid_ FROM genomic_range_rowids('exons',?,?,?) ORDER BY _rowid_ DESC", + ("chr17", 43044294, 43048294), ) + if "USE TEMP B-TREE FOR ORDER BY" in expl[3] ) with pytest.raises(sqlite3.OperationalError, match="domain error"): diff --git a/test/test_parse_genomic_range.py b/test/test_parse_genomic_range.py index 8b5e508..e561f13 100644 --- a/test/test_parse_genomic_range.py +++ b/test/test_parse_genomic_range.py @@ -1,6 +1,7 @@ import sqlite3 -import pytest + import genomicsqlite +import pytest def test_parse_genomic_range(): diff --git a/test/test_twobit.py b/test/test_twobit.py index f574d98..2c22afc 100644 --- a/test/test_twobit.py +++ b/test/test_twobit.py @@ -1,8 +1,9 @@ -import random import math +import random import sqlite3 -import pytest + import genomicsqlite +import pytest def test_twobit_random(): diff --git a/test/test_txdb.py b/test/test_txdb.py index b96bb52..f56b446 100644 --- a/test/test_txdb.py +++ b/test/test_txdb.py @@ -1,9 +1,10 @@ import os -import sys -import subprocess import random -import pytest +import subprocess +import sys + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) @@ -49,7 +50,7 @@ def test_txdbquery(genomicsqlite_txdb): assert next(conn.execute("PRAGMA page_size"))[0] == 16384 # one query - results = list( + results = [ t[0] for t in conn.execute( "SELECT tx_name FROM transcript WHERE _rowid_ IN " @@ -57,7 +58,7 @@ def test_txdbquery(genomicsqlite_txdb): + " ORDER BY tx_name", ("chr12", 111803912, 111804012), ) - ) + ] print(results) assert results == sorted( ["ENST00000416293.7", "ENST00000261733.7", "ENST00000548536.1", "ENST00000549106.1"] @@ -91,8 +92,8 @@ def test_txdbquery(genomicsqlite_txdb): chrom = random.choice(chroms) beg = random.randint(0, chrom[1] - 65536) end = beg + random.randint(1, random.choice([16, 256, 4096, 65536])) - ids = list(row[0] for row in conn.execute(query, (chrom[0], beg, end))) - control_ids = list(row[0] for row in conn.execute(control_query, (chrom[0], beg, end))) + ids = [row[0] for row in conn.execute(query, (chrom[0], beg, end))] + control_ids = [row[0] for row in conn.execute(control_query, (chrom[0], beg, end))] assert ids == control_ids total_results += len(control_ids) assert total_results in (7341, 2660) @@ -148,7 +149,7 @@ def test_web(): read_only=True, ) - results = list( + results = [ t[0] for t in conn.execute( "SELECT tx_name FROM transcript WHERE _rowid_ IN " @@ -156,7 +157,7 @@ def test_web(): + " ORDER BY tx_name", ("chr12", 111803912, 111804012), ) - ) + ] print(results) sys.stdout.flush() assert results == sorted( diff --git a/test/test_vcf.py b/test/test_vcf.py index bed3151..ea456c4 100644 --- a/test/test_vcf.py +++ b/test/test_vcf.py @@ -1,8 +1,9 @@ import os import sqlite3 import subprocess -import pytest + import genomicsqlite +import pytest HERE = os.path.dirname(__file__) BUILD = os.path.abspath(os.path.join(HERE, "..", "build")) @@ -49,22 +50,22 @@ def test_gnomad_sites_small(tmp_path): indexed += 1 assert indexed == 3 opcodes = list(con.execute("EXPLAIN " + query, rs671)) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 results = list(con.execute(query, rs671)) - results_rowids = set(vt[0] for vt in results) + results_rowids = {vt[0] for vt in results} assert next(vt for vt in results if vt[1] and "rs671" in vt[1]) control = "SELECT variant_rowid FROM gnomad_variants NATURAL JOIN _gri_refseq WHERE gri_refseq_name = ? AND NOT ((pos+rlen) < ? OR pos > ?)" - control_rowids = set(vt[0] for vt in con.execute(control, rs671)) + control_rowids = {vt[0] for vt in con.execute(control, rs671)} assert len(control_rowids) == 22 assert results_rowids == control_rowids diff --git a/test/test_vcf_lines.py b/test/test_vcf_lines.py index 7ebdb91..9c96f98 100644 --- a/test/test_vcf_lines.py +++ b/test/test_vcf_lines.py @@ -1,6 +1,7 @@ import os import sqlite3 import subprocess + import genomicsqlite import pytest @@ -43,22 +44,22 @@ def test_gnomad_sites_small(tmp_path): indexed += 1 assert indexed == 3 opcodes = list(con.execute("EXPLAIN " + query, rs671)) - accessed_cursors = list(opcode[2] for opcode in opcodes if opcode[1] == "Column") - table_rootpages = set( + accessed_cursors = [opcode[2] for opcode in opcodes if opcode[1] == "Column"] + table_rootpages = { row[0] for row in con.execute("SELECT rootpage FROM sqlite_master WHERE type='table'") - ) - table_cursors = set( + } + table_cursors = { opcode[2] for opcode in opcodes if opcode[1] == "OpenRead" and opcode[3] in table_rootpages - ) + } assert sum(1 for cursor in accessed_cursors if cursor not in table_cursors) > 1 assert sum(1 for cursor in accessed_cursors if cursor in table_cursors) == 1 results = list(con.execute(query, rs671)) - results_rowids = set(vt[0] for vt in results) + results_rowids = {vt[0] for vt in results} assert next(vt for vt in results if vt[1] and "rs671" in vt[1]) control = "SELECT rowid FROM gnomad_vcf_lines WHERE NOT ((POS+rlen) < ? OR POS > ?)" - control_rowids = set(vt[0] for vt in con.execute(control, (rs671[1], rs671[2]))) + control_rowids = {vt[0] for vt in con.execute(control, (rs671[1], rs671[2]))} assert len(control_rowids) == 22 assert results_rowids == control_rowids