diff --git a/README.md b/README.md index d417bd13..975b2e3c 100644 --- a/README.md +++ b/README.md @@ -38,20 +38,20 @@ To install the library, add the following lines to your build config file. io.qdrant client - 1.19.0 + 1.20.0 ``` #### SBT ```sbt -libraryDependencies += "io.qdrant" % "client" % "1.19.0" +libraryDependencies += "io.qdrant" % "client" % "1.20.0" ``` #### Gradle ```gradle -implementation 'io.qdrant:client:1.19.0' +implementation 'io.qdrant:client:1.20.0' ``` > [!NOTE] diff --git a/build.gradle b/build.gradle index 394917be..cc5d540a 100644 --- a/build.gradle +++ b/build.gradle @@ -147,7 +147,7 @@ tasks.register('downloadProtos') { def fileName = Paths.get(entry.getName()).getFileName().toString() def dest = java.nio.file.Path.of(outputDirectory, fileName) logger.debug("writing {} to {}", fileName, dest) - Files.write(dest, lines, StandardOpenOption.CREATE) + Files.write(dest, lines, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING) } } } diff --git a/example/build.gradle b/example/build.gradle index c9780ab0..78435724 100644 --- a/example/build.gradle +++ b/example/build.gradle @@ -13,13 +13,13 @@ repositories { dependencies { // Qdrant Java client - implementation 'io.qdrant:client:1.19.0' + implementation 'io.qdrant:client:1.20.0' // gRPC dependencies - use the same version as Qdrant client - implementation 'io.grpc:grpc-netty-shaded:1.65.1' - implementation 'io.grpc:grpc-protobuf:1.65.1' - implementation 'io.grpc:grpc-stub:1.65.1' - implementation 'io.grpc:grpc-services:1.65.1' + implementation 'io.grpc:grpc-netty-shaded:1.84.2' + implementation 'io.grpc:grpc-protobuf:1.84.2' + implementation 'io.grpc:grpc-stub:1.84.2' + implementation 'io.grpc:grpc-services:1.84.2' // Protobuf runtime implementation 'com.google.protobuf:protobuf-java:3.25.4' diff --git a/gradle.properties b/gradle.properties index 2a1d835b..8f5850bb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,8 @@ # The version of qdrant to use to download protos -qdrantProtosVersion=v1.19.0 +qdrantProtosVersion=v1.20.0 # The version of qdrant docker image to run integration tests against -qdrantVersion=v1.19.0 +qdrantVersion=v1.20.0 # The version of the client to generate -packageVersion=1.19.0 +packageVersion=1.20.0 diff --git a/src/main/java/io/qdrant/client/ConditionFactory.java b/src/main/java/io/qdrant/client/ConditionFactory.java index 40c6a366..0e95b76c 100644 --- a/src/main/java/io/qdrant/client/ConditionFactory.java +++ b/src/main/java/io/qdrant/client/ConditionFactory.java @@ -162,6 +162,23 @@ public static Condition matchPrefix(String field, String prefix) { .build(); } + /** + * Match records where the given field contains the given substring. + * + * @param field The name of the field + * @param substring The substring to match + * @return a new instance of {@link Condition} + */ + public static Condition matchSubstring(String field, String substring) { + return Condition.newBuilder() + .setField( + FieldCondition.newBuilder() + .setKey(field) + .setMatch(Match.newBuilder().setSubstring(substring).build()) + .build()) + .build(); + } + /** * Match records where the given field matches the given boolean value. * diff --git a/src/main/java/io/qdrant/client/ExpressionFactory.java b/src/main/java/io/qdrant/client/ExpressionFactory.java index 604ab2cb..f57bf3dd 100644 --- a/src/main/java/io/qdrant/client/ExpressionFactory.java +++ b/src/main/java/io/qdrant/client/ExpressionFactory.java @@ -5,6 +5,8 @@ import io.qdrant.client.grpc.Points.DivExpression; import io.qdrant.client.grpc.Points.Expression; import io.qdrant.client.grpc.Points.GeoDistance; +import io.qdrant.client.grpc.Points.MaxExpression; +import io.qdrant.client.grpc.Points.MinExpression; import io.qdrant.client.grpc.Points.MultExpression; import io.qdrant.client.grpc.Points.PowExpression; import io.qdrant.client.grpc.Points.SumExpression; @@ -202,4 +204,34 @@ public static Expression gaussDecay(DecayParamsExpression decay) { public static Expression linDecay(DecayParamsExpression decay) { return Expression.newBuilder().setLinDecay(decay).build(); } + + /** + * Creates an {@link Expression} representing inverse hyperbolic cosine. + * + * @param expr The expression to apply acosh() to + * @return a new instance of {@link Expression} + */ + public static Expression acosh(Expression expr) { + return Expression.newBuilder().setAcosh(expr).build(); + } + + /** + * Creates an {@link Expression} for the maximum of values. + * + * @param max The maximum expression + * @return a new instance of {@link Expression} + */ + public static Expression max(MaxExpression max) { + return Expression.newBuilder().setMax(max).build(); + } + + /** + * Creates an {@link Expression} for the minimum of values. + * + * @param min The minimum expression + * @return a new instance of {@link Expression} + */ + public static Expression min(MinExpression min) { + return Expression.newBuilder().setMin(min).build(); + } } diff --git a/src/main/java/io/qdrant/client/QueryFactory.java b/src/main/java/io/qdrant/client/QueryFactory.java index dd6ba8e5..99b36743 100644 --- a/src/main/java/io/qdrant/client/QueryFactory.java +++ b/src/main/java/io/qdrant/client/QueryFactory.java @@ -19,6 +19,7 @@ import io.qdrant.client.grpc.Points.RelevanceFeedbackInput; import io.qdrant.client.grpc.Points.Rrf; import io.qdrant.client.grpc.Points.Sample; +import io.qdrant.client.grpc.Points.TextQuery; import io.qdrant.client.grpc.Points.VectorInput; import java.util.List; import java.util.UUID; @@ -108,6 +109,26 @@ public static Query formula(Formula formula) { return Query.newBuilder().setFormula(formula).build(); } + /** + * Creates a {@link Query} for BM25 text search. + * + * @param query The text to search for + * @return a new instance of {@link Query} + */ + public static Query text(String query) { + return text(TextQuery.newBuilder().setQuery(query).build()); + } + + /** + * Creates a {@link Query} for BM25 text search. + * + * @param text An instance of {@link TextQuery} + * @return a new instance of {@link Query} + */ + public static Query text(TextQuery text) { + return Query.newBuilder().setText(text).build(); + } + // region Nearest search queries /** diff --git a/src/test/java/io/qdrant/client/CollectionsTest.java b/src/test/java/io/qdrant/client/CollectionsTest.java index ea772716..0d113799 100644 --- a/src/test/java/io/qdrant/client/CollectionsTest.java +++ b/src/test/java/io/qdrant/client/CollectionsTest.java @@ -16,6 +16,8 @@ import io.qdrant.client.grpc.Collections.CollectionStatus; import io.qdrant.client.grpc.Collections.CreateCollection; import io.qdrant.client.grpc.Collections.Distance; +import io.qdrant.client.grpc.Collections.IdTrackerParams; +import io.qdrant.client.grpc.Collections.Memory; import io.qdrant.client.grpc.Collections.VectorParams; import io.qdrant.client.grpc.Collections.VectorsConfig; import io.qdrant.client.grpc.Points.CreateVectorNameRequest; @@ -140,6 +142,29 @@ public void updateCollection() throws ExecutionException, InterruptedException { .get(); } + @Test + public void createCollection_with_id_tracker() throws ExecutionException, InterruptedException { + client + .createCollectionAsync( + CreateCollection.newBuilder() + .setCollectionName(testName) + .setVectorsConfig( + VectorsConfig.newBuilder() + .setParams( + VectorParams.newBuilder() + .setDistance(Distance.Cosine) + .setSize(4) + .build()) + .build()) + .setIdTracker(IdTrackerParams.newBuilder().setMemory(Memory.Cached).build()) + .build()) + .get(); + + CollectionInfo collectionInfo = client.getCollectionInfoAsync(testName).get(); + assertEquals(Memory.Cached, collectionInfo.getConfig().getParams().getIdTracker().getMemory()); + assertTrue(collectionInfo.hasCreatedAt()); + } + @Test public void updateCollection_with_no_changes() throws ExecutionException, InterruptedException { CreateCollection createCollection = getCreateCollection(testName); diff --git a/src/test/java/io/qdrant/client/PointsTest.java b/src/test/java/io/qdrant/client/PointsTest.java index 3828854a..e8c58c4b 100644 --- a/src/test/java/io/qdrant/client/PointsTest.java +++ b/src/test/java/io/qdrant/client/PointsTest.java @@ -2,17 +2,28 @@ import static io.qdrant.client.ConditionFactory.hasId; import static io.qdrant.client.ConditionFactory.matchKeyword; +import static io.qdrant.client.ConditionFactory.matchSubstring; +import static io.qdrant.client.ConditionFactory.matchText; +import static io.qdrant.client.ExpressionFactory.acosh; +import static io.qdrant.client.ExpressionFactory.constant; +import static io.qdrant.client.ExpressionFactory.max; +import static io.qdrant.client.ExpressionFactory.min; +import static io.qdrant.client.ExpressionFactory.sum; +import static io.qdrant.client.ExpressionFactory.variable; import static io.qdrant.client.PointIdFactory.id; +import static io.qdrant.client.QueryFactory.formula; import static io.qdrant.client.QueryFactory.fusion; import static io.qdrant.client.QueryFactory.nearest; import static io.qdrant.client.QueryFactory.orderBy; import static io.qdrant.client.QueryFactory.sample; +import static io.qdrant.client.QueryFactory.text; import static io.qdrant.client.TargetVectorFactory.targetVector; import static io.qdrant.client.ValueFactory.value; import static io.qdrant.client.VectorFactory.vector; import static io.qdrant.client.VectorsFactory.vectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import io.grpc.Grpc; @@ -22,14 +33,25 @@ import io.qdrant.client.grpc.Collections.CollectionInfo; import io.qdrant.client.grpc.Collections.CreateCollection; import io.qdrant.client.grpc.Collections.Distance; +import io.qdrant.client.grpc.Collections.FloatIndexParams; +import io.qdrant.client.grpc.Collections.PayloadIndexParams; import io.qdrant.client.grpc.Collections.PayloadSchemaType; +import io.qdrant.client.grpc.Collections.TextIndexParams; +import io.qdrant.client.grpc.Collections.TextScoringParams; +import io.qdrant.client.grpc.Collections.TextScoringType; +import io.qdrant.client.grpc.Collections.TokenizerType; import io.qdrant.client.grpc.Collections.VectorParams; import io.qdrant.client.grpc.Collections.VectorsConfig; import io.qdrant.client.grpc.Common.Filter; import io.qdrant.client.grpc.Points; import io.qdrant.client.grpc.Points.BatchResult; +import io.qdrant.client.grpc.Points.Bm25Params; import io.qdrant.client.grpc.Points.DiscoverPoints; +import io.qdrant.client.grpc.Points.Expression; +import io.qdrant.client.grpc.Points.Formula; import io.qdrant.client.grpc.Points.Fusion; +import io.qdrant.client.grpc.Points.MaxExpression; +import io.qdrant.client.grpc.Points.MinExpression; import io.qdrant.client.grpc.Points.PointGroup; import io.qdrant.client.grpc.Points.PointStruct; import io.qdrant.client.grpc.Points.PointVectors; @@ -50,6 +72,8 @@ import io.qdrant.client.grpc.Points.ScrollResponse; import io.qdrant.client.grpc.Points.SearchPointGroups; import io.qdrant.client.grpc.Points.SearchPoints; +import io.qdrant.client.grpc.Points.SumExpression; +import io.qdrant.client.grpc.Points.TextQuery; import io.qdrant.client.grpc.Points.UpdateResult; import io.qdrant.client.grpc.Points.UpdateStatus; import io.qdrant.client.grpc.Points.VectorsOutput; @@ -779,6 +803,210 @@ public void queryWithSampling() throws ExecutionException, InterruptedException assertEquals(1, points.size()); } + @Test + public void queryFormula() throws ExecutionException, InterruptedException { + createAndSeedCollection(testName); + + Expression score = variable("$score"); + List points = + client + .queryAsync( + QueryPoints.newBuilder() + .setCollectionName(testName) + .addPrefetch(PrefetchQuery.newBuilder().setQuery(nearest(3.5f, 4.5f)).build()) + .setQuery( + formula( + Formula.newBuilder() + .setExpression( + sum( + SumExpression.newBuilder() + .addSum( + max( + MaxExpression.newBuilder() + .addMax(score) + .addMax(constant(2)) + .build())) + .addSum( + min( + MinExpression.newBuilder() + .addMin(score) + .addMin(constant(0)) + .build())) + .addSum(acosh(constant(1))) + .build())) + .build())) + .build()) + .get(); + + assertEquals(2, points.size()); + points.forEach(point -> assertEquals(2.0f, point.getScore(), 1e-6f)); + } + + @Test + public void queryText() throws ExecutionException, InterruptedException { + client + .createCollectionAsync( + testName, VectorParams.newBuilder().setDistance(Distance.Dot).setSize(4).build()) + .get(); + + UpdateResult result = + client + .createPayloadIndexAsync( + testName, + "title", + PayloadSchemaType.Text, + PayloadIndexParams.newBuilder() + .setTextIndexParams( + TextIndexParams.newBuilder() + .setTokenizer(TokenizerType.Word) + .setScoring( + TextScoringParams.newBuilder() + .setType(TextScoringType.Bm25) + .build()) + .build()) + .build(), + true, + null, + null) + .get(); + assertEquals(UpdateStatus.Completed, result.getStatus()); + + client + .upsertAsync( + testName, + ImmutableList.of( + textPoint(1, "title", "vector search engine"), + textPoint(2, "title", "search search search"), + textPoint(3, "title", "relational database"))) + .get(); + + List points = + client + .queryAsync( + QueryPoints.newBuilder() + .setCollectionName(testName) + .setQuery(text("vector search")) + .setUsing("title") + .build()) + .get(); + + assertEquals(2, points.size()); + assertEquals(id(1), points.get(0).getId()); + assertEquals(id(2), points.get(1).getId()); + + List tuned = + client + .queryAsync( + QueryPoints.newBuilder() + .setCollectionName(testName) + .setQuery( + text( + TextQuery.newBuilder() + .setQuery("vector search") + .setBm25(Bm25Params.newBuilder().setK(2.0f).setB(0.5f).build()) + .build())) + .setUsing("title") + .build()) + .get(); + + assertEquals(2, tuned.size()); + assertNotEquals(points.get(0).getScore(), tuned.get(0).getScore()); + } + + @Test + public void scrollWithSubstringMatch() throws ExecutionException, InterruptedException { + client + .createCollectionAsync( + testName, VectorParams.newBuilder().setDistance(Distance.Dot).setSize(4).build()) + .get(); + + client + .createPayloadIndexAsync(testName, "sku", PayloadSchemaType.Keyword, null, true, null, null) + .get(); + client + .upsertAsync( + testName, + ImmutableList.of(textPoint(1, "sku", "AB-1234-XY"), textPoint(2, "sku", "CD-5678-XY"))) + .get(); + + List points = + client + .scrollAsync( + ScrollPoints.newBuilder() + .setCollectionName(testName) + .setFilter(Filter.newBuilder().addMust(matchSubstring("sku", "1234")).build()) + .build()) + .get() + .getResultList(); + + assertEquals(1, points.size()); + assertEquals(id(1), points.get(0).getId()); + } + + @Test + public void scrollWithCodeTokenizer() throws ExecutionException, InterruptedException { + client + .createCollectionAsync( + testName, VectorParams.newBuilder().setDistance(Distance.Dot).setSize(4).build()) + .get(); + + client + .createPayloadIndexAsync( + testName, + "source", + PayloadSchemaType.Text, + PayloadIndexParams.newBuilder() + .setTextIndexParams( + TextIndexParams.newBuilder().setTokenizer(TokenizerType.Code).build()) + .build(), + true, + null, + null) + .get(); + client + .upsertAsync( + testName, + ImmutableList.of( + textPoint(10, "source", "fn getUserName() {}"), + textPoint(11, "source", "fn deleteFile() {}"))) + .get(); + + List points = + client + .scrollAsync( + ScrollPoints.newBuilder() + .setCollectionName(testName) + .setFilter( + Filter.newBuilder().addMust(matchText("source", "get_user_name")).build()) + .build()) + .get() + .getResultList(); + + assertEquals(1, points.size()); + assertEquals(id(10), points.get(0).getId()); + } + + @Test + public void createNestedFieldIndex() throws ExecutionException, InterruptedException { + createAndSeedCollection(testName); + + UpdateResult result = + client + .createPayloadIndexAsync( + testName, + "offers[].price", + PayloadSchemaType.Float, + PayloadIndexParams.newBuilder() + .setFloatIndexParams(FloatIndexParams.newBuilder().setNested("offers").build()) + .build(), + true, + null, + null) + .get(); + + assertEquals(UpdateStatus.Completed, result.getStatus()); + } + @Test public void queryGroups() throws ExecutionException, InterruptedException { createAndSeedCollection(testName); @@ -928,4 +1156,12 @@ private void createAndSeedCollection(String collectionName) .get(); assertEquals(UpdateStatus.Completed, result.getStatus()); } + + private static PointStruct textPoint(long id, String field, String text) { + return PointStruct.newBuilder() + .setId(id(id)) + .setVectors(vectors(0.1f, 0.2f, 0.3f, 0.4f)) + .putPayload(field, value(text)) + .build(); + } }