Native: get_stats, get_metadata, PQ search and AsyncExecutor.transaction work; the NotOnNativeYet docs say what is true

docs/modules/ROOT/pages/installation.adoc

@@ -147,7 +147,7 @@ Each Python process starts its own JVM, and each JVM sees the whole container li
 
 == Native build notes (`{native-pkg-name}`)
 
-The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. A few API calls raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use.
+The native package is the same engine compiled to a GraalVM native-image shared library. It needs no JVM and no JPype, so the JVM options above do not apply to it. The calls that hand out a Java object (`get_java_database`, `get_java_document`) raise `NotOnNativeYet` (a `NotImplementedError`) instead of misbehaving. The list is in xref:which-package.adoc[Which package?]. Report problems in the {repo-url}/issues[issue tracker] and say which package you use.
 
 == Next steps
 

docs/modules/ROOT/pages/which-package.adoc

@@ -82,16 +82,16 @@ The API is the same. The runtime is not. These are the differences you can plan
 | Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64. The Windows wheel is the newest: it is built in CI, and its test run is the least proven of the four.
 | Linux x86_64 (glibc 2.34 or later), Linux aarch64, macOS arm64, Windows x64.
 
-| Java-object APIs (`get_java_database`, `get_java_*`)
-| Not available.
+| Java-object APIs (`get_java_database`, `get_java_document`, `get_java_*`)
+| Not available. They raise `NotOnNativeYet`.
 | Available.
 
 | JVM options (`jvm_kwargs`, `start_jvm`, `-X` and `-D` flags)
 | Do not exist. There is no JVM.
 | Available. See xref:installation.adoc#jvm-options[JVM options].
 
 | A few API calls
-| Raise `NotOnNativeYet` (a `NotImplementedError`): `AsyncExecutor.transaction`, the approximate search `find_nearest_approximate` over product quantization, `get_metadata` of a foreign index, and `get_stats`.
+| None, except the Java-object calls above, which raise `NotOnNativeYet` (a `NotImplementedError`).
 | Work.
 |===
 

native/src/main/java/org/humemai/nativelib/NativeExtras.java

@@ -25,6 +25,7 @@
 import com.arcadedb.database.RID;
 import com.arcadedb.database.async.DatabaseAsyncExecutor;
 import com.arcadedb.database.async.ErrorCallback;
+import com.arcadedb.database.async.OkCallback;
 import com.arcadedb.database.async.NewEdgeCallback;
 import com.arcadedb.engine.ErrorRecordCallback;
 import com.arcadedb.engine.WALFile;
@@ -39,6 +40,11 @@
 import com.arcadedb.function.CypherFunctionRegistry;
 import com.arcadedb.function.FunctionRegistry;
 import com.arcadedb.function.sql.DefaultSQLFunctionFactory;
+import com.arcadedb.index.IndexInternal;
+import com.arcadedb.index.TypeIndex;
+import com.arcadedb.index.vector.LSMVectorIndex;
+import com.arcadedb.schema.LSMVectorIndexMetadata;
+import com.arcadedb.utility.Pair;
 import com.arcadedb.function.sql.SQLFunctionAbstract;
 import com.arcadedb.graph.Edge;
 import com.arcadedb.graph.GraphBatch;
@@ -63,8 +69,10 @@
 
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -269,6 +277,81 @@ public static CCharPointer dbSet(final IsolateThread thread, final long h, final
     }
   }
 
+  // ---- vector indexes: what no statement reports (the engine's configuration of an index) and the PQ search no statement runs
+
+  private static String q(final String text) {
+    return text == null ? "null" : "\"" + text.replace("\\", "\\\\").replace("\"", "\\\"") + "\"";
+  }
+
+  /**
+   * ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed}. The index is the one the type has on the property; its bucket
+   * indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does.
+   */
+  @CEntryPoint(name = "arcadedb_n_vindex")
+  @SuppressWarnings("unchecked")
+  public static CCharPointer vindex(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) {
+    try {
+      final Database db = ArcadeNative.db(dbh);
+      final Map<String, Object> a = args(argsJson);
+      final String type = (String) a.get("type"), prop = (String) a.get("prop");
+      final TypeIndex typeIndex = db.getSchema().getType(type).getPolymorphicIndexByProperties(prop);
+      if (typeIndex == null)
+        throw new IllegalArgumentException("No vector index on " + type + "[" + prop + "]");
+      final List<LSMVectorIndex> parts = new ArrayList<>();
+      for (final IndexInternal index : typeIndex.getIndexesOnBuckets())
+        if (index instanceof LSMVectorIndex lsm)
+          parts.add(lsm);
+      if (parts.isEmpty())
+        throw new IllegalArgumentException("Underlying index is not an LSM vector index");
+      final StringBuilder sb = new StringBuilder("{\"ok\":true");
+      switch (ArcadeNative.str(op)) {
+      case "metadata" -> {
+        final LSMVectorIndex first = parts.get(0);
+        final LSMVectorIndexMetadata m = first.getMetadata();
+        sb.append(",\"bucket\":").append(q(first.getName())).append(",\"type_name\":").append(q(first.getTypeName()));
+        sb.append(",\"dimensions\":").append(m.dimensions).append(",\"similarity_function\":").append(q(String.valueOf(m.similarityFunction)));
+        sb.append(",\"id_property\":").append(q(m.idPropertyName)).append(",\"quantization\":").append(q(String.valueOf(m.quantizationType)));
+        sb.append(",\"max_connections\":").append(m.maxConnections).append(",\"beam_width\":").append(m.beamWidth);
+        sb.append(",\"ef_search\":").append(m.efSearch).append(",\"graph_build_cache_size\":").append(m.graphBuildCacheSize);
+        sb.append(",\"mutations_before_rebuild\":").append(m.mutationsBeforeRebuild).append(",\"store_vectors_in_graph\":").append(m.storeVectorsInGraph);
+        sb.append(",\"add_hierarchy\":").append(m.addHierarchy).append(",\"build_state\":").append(q(m.buildState));
+        sb.append(",\"pq_subspaces\":").append(m.pqSubspaces).append(",\"pq_clusters\":").append(m.pqClusters);
+        sb.append(",\"pq_center_globally\":").append(m.pqCenterGlobally).append(",\"pq_training_limit\":").append(m.pqTrainingLimit);
+      }
+      case "approximate" -> {
+        final List<Object> vector = (List<Object>) a.get("vector");
+        final float[] query = new float[vector.size()];
+        for (int i = 0; i < query.length; i++)
+          query[i] = ((Number) vector.get(i)).floatValue();
+        final int k = intOf(a.get("k"), 10);
+        Set<RID> allowed = null;
+        if (a.get("allowed") instanceof List<?> rids) {
+          allowed = new HashSet<>();
+          for (final Object rid : rids)
+            allowed.add(new RID(String.valueOf(rid)));
+        }
+        sb.append(",\"hits\":[");
+        boolean firstHit = true;
+        for (final LSMVectorIndex lsm : parts) {
+          final List<Pair<RID, Float>> hits = allowed != null ? lsm.findNeighborsFromVectorApproximate(query, k, allowed)
+              : lsm.findNeighborsFromVectorApproximate(query, k);
+          for (final Pair<RID, Float> hit : hits) {
+            if (!firstHit)
+              sb.append(',');
+            firstHit = false;
+            sb.append("[\"").append(hit.getFirst()).append("\",").append(hit.getSecond()).append(']');
+          }
+        }
+        sb.append(']');
+      }
+      default -> throw new IllegalArgumentException("unknown vector index op " + ArcadeNative.str(op));
+      }
+      return ArcadeNative.out(sb.append('}').toString());
+    } catch (final Throwable t) {
+      return ArcadeNative.err(t);
+    }
+  }
+
   // ---- GraphBatch
 
   private record BatchEntry(GraphBatch batch, Database db) {
@@ -543,6 +626,31 @@ public void onError(final Exception e) {
         else
           ex.query(lang, text, callback);
       }
+      case "transaction" -> {
+        // a block that runs on an async writer inside a transaction: Python runs it in the upcall; its failure is thrown here so the
+        // writer rolls back, retries a NeedRetryException and reports to the error callback (ok / err: per-call callbacks, slot: one writer)
+        final long okCb = a.get("ok") instanceof Number n ? n.longValue() : 0L;
+        final Database.TransactionScope scope = () -> {
+          final String reply = upcall(cb, event("tx", ""));
+          if (reply != null && reply.startsWith("{\"error\""))
+            throw new RuntimeException(String.valueOf(((Map<String, Object>) Json.parse(reply)).get("error")));
+        };
+        final int retries = intOf(a.get("retries"), 1);
+        final OkCallback ok = okCb == 0 ? null : () -> {
+          try {
+            fire("transaction ok callback", okCb, event("ok", ""));
+          } catch (final Throwable e) {
+            swallowed("transaction ok callback", e); // the writer goes on
+          }
+        };
+        final ErrorCallback error = errCb == 0 ? null : errorCallback(errCb);
+        if (a.get("retries") == null && ok == null && error == null && a.get("slot") == null)
+          ex.transaction(scope);
+        else if (a.get("slot") instanceof Number slot)
+          ex.transaction(scope, retries, ok, error, slot.intValue());
+        else
+          ex.transaction(scope, retries, ok, error);
+      }
       case "on_ok" -> ex.onOk(() -> {
         try {
           fire("on_ok callback", cb, event("ok", ""));

src/arcadedb/_native/__init__.py

@@ -1,21 +1,25 @@
-"""The native backend: the engine as a GraalVM native-image library, no JVM and no JPype.
+"""The native backend: the engine as a GraalVM native-image shared library (``libarcadedb``) that Python loads with ``ctypes``. No JVM, no JPype.
 
-It is the only backend of the `arcadedb` wheel. In a source checkout or in the `arcadedb-jvm` wheel, ``ARCADEDB_BACKEND=native``
-selects it (with ``ARCADEDB_NATIVE_LIB`` pointing at a library you built); the default there is JPype. The engine runs as a GraalVM native-image shared
-library (``libarcadedb``) that Python loads with ``ctypes``: no JVM, no JPype. The surface is the DSL-first core:
-databases, ``query``/``command`` for SQL and openCypher, transactions, results and the in-process server.
+It is the only backend of the ``arcadedb`` wheel. In a source checkout or in the ``arcadedb-jvm`` wheel, ``ARCADEDB_BACKEND=native`` selects it
+(with ``ARCADEDB_NATIVE_LIB`` pointing at a library you built); the default there is JPype. The public API is the JVM build's, and the same
+test suite (``tests/``) runs against both.
 
-What is not on native yet (each raises :class:`NotOnNativeYet`): ``GraphBatch``, the
-async executor, the Java-object APIs (``Document``/``Vertex``/``Edge`` wrappers, ``new_vertex``, ``new_document``,
-``schema``, ``get_java_database``), the columnar/Arrow readers other than a simple row conversion, importers and
-exporters that go through Java objects, and every JVM setting (``jvm_kwargs``, ``start_jvm``).
+On native: databases and the factory, ``query``/``command`` (SQL and openCypher), transactions and ``run_in_transaction``, results as rows,
+columns and Arrow, pandas, polars and numpy frames, the ``Document``/``Vertex``/``Edge`` wrappers, the schema (types, properties, indexes),
+vector indexes (``find_nearest``, the PQ ``find_nearest_approximate``, ``get_metadata``, ``get_stats``), ``insert_many``, ``GraphBatch``,
+the ``AsyncExecutor`` (``transaction`` included), Python functions in SQL and record callbacks, importers and exporters, and the in-process
+server.
+
+Not on native, by design (each raises :class:`NotOnNativeYet`): the Java objects of the JVM build. These are ``get_java_database``,
+``get_java_document``, and the Java object a ``Schema``, type or index method returns where the native wrapper has no such method.
+There is no JVM to start or tune: ``jvm_kwargs`` is ignored with a warning.
 """
 
 import os
 
 
 class NotOnNativeYet(NotImplementedError):
-    """The feature needs the Java object API, which the native backend does not expose (yet)."""
+    """The call hands out a Java object of the JVM build (``get_java_database``, ``get_java_document``, a Schema method's Java object), which the native build does not have."""
 
 
 # The backend used when ARCADEDB_BACKEND is unset. "jpype" here, which is the arcadedb-jvm wheel; the build of the native

src/arcadedb/_native/extras.py

@@ -693,8 +693,37 @@ def on_error(message):
             "scan_type", type=type_name, polymorphic=bool(polymorphic), cb=cb, err=err
         )
 
-    def transaction(self, *args, **kwargs):
-        not_yet("AsyncExecutor.transaction")
+    def transaction(
+        self,
+        tx_block: Callable[[], None],
+        retries: Optional[int] = None,
+        ok_callback: Optional[Callable[[], None]] = None,
+        error_callback: Optional[Callable[[Exception], None]] = None,
+        slot: Optional[int] = None,
+    ):
+        """Run ``tx_block`` on an async writer inside a transaction (the engine begins it and commits or rolls it back, retrying a conflict ``retries`` times)."""
+
+        def run(message):
+            tx_block()
+
+        args = {"cb": _register(run)}
+        if retries is not None:
+            args["retries"] = int(retries)
+        if ok_callback is not None:
+            args["ok"] = _register(lambda message: ok_callback())
+        if error_callback is not None:
+            args["err"] = self._error_callback(error_callback)
+        # a retry calls the block again, so the callbacks live until the executor has finished its work (wait_completion, close)
+        self._cb_ids.extend(v for k, v in args.items() if k in ("cb", "ok", "err"))
+        if slot is not None:
+            args["slot"] = int(slot)
+        self._op("transaction", **args)
+
+    def _release_callbacks(self):
+        """Forget the callbacks of the finished ``transaction`` calls."""
+        ids, self._cb_ids = self._cb_ids, []
+        for cid in ids:
+            _forget(cid)
 
     def wait_completion(self, timeout_ms: Optional[int] = None):
         if timeout_ms is None:
@@ -708,6 +737,7 @@ def wait_completion(self, timeout_ms: Optional[int] = None):
             raise TimeoutError(
                 f"Async operations did not complete within {timeout_ms}ms"
             )
+        self._release_callbacks()  # nothing is pending any more
 
     def on_ok(self, callback: Callable[[], None]):
         self._op("on_ok", cb=_register(lambda message: callback()))
@@ -724,6 +754,7 @@ def close(self):
             self._op("close")
         finally:
             self._closed = True
+            self._release_callbacks()
             owner, self._owner = self._owner, None
             if owner is not None:
                 owner._discard_async_executor(self)

src/arcadedb/_native/lib.py

@@ -176,6 +176,7 @@ class _Params(ctypes.Structure):
             "arcadedb_n_gb_create": [P, L, S],
             "arcadedb_n_gb": [P, L, S, S],
             "arcadedb_n_async": [P, L, S, S],
+            "arcadedb_n_vindex": [P, L, S, S],
             "arcadedb_n_import": [P, L, S],
             "arcadedb_n_export": [P, L, S, S, I, S],
             "arcadedb_n_cb": [P, S, S],