From b29d31029f0cebbaa3727e5c6c980f7b1ee70673 Mon Sep 17 00:00:00 2001 From: ZYinNJU <1754350460@qq.com> Date: Wed, 4 Sep 2024 20:49:36 +0800 Subject: [PATCH] Migrate Vearch from Gson to Jackson (#1706) ## Issue Closes #1680 ## Change Migrate Vearch from Gson to Jackson ## General checklist - [x] There are no breaking changes - [ ] I have added unit and integration tests for my change - [x] I have manually run all the unit and integration tests in the module I have added/changed, and they are all green - [x] I have manually run all the unit and integration tests in the [core](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core) and [main](https://github.com/langchain4j/langchain4j/tree/main/langchain4j) modules, and they are all green - [ ] I have added/updated the [documentation](https://github.com/langchain4j/langchain4j/tree/main/docs/docs) - [ ] I have added an example in the [examples repo](https://github.com/langchain4j/langchain4j-examples) (only for "big" features) - [ ] I have added/updated [Spring Boot starter(s)](https://github.com/langchain4j/langchain4j-spring) (if applicable) --- langchain4j-vearch/pom.xml | 2 +- .../store/embedding/vearch/BulkRequest.java | 17 ++++ .../store/embedding/vearch/BulkResponse.java | 25 +++++- .../vearch/CreateDatabaseRequest.java | 17 ++++ .../vearch/CreateDatabaseResponse.java | 20 ++++- .../embedding/vearch/CreateSpaceRequest.java | 25 +++++- .../embedding/vearch/CreateSpaceResponse.java | 20 ++++- .../vearch/ListDatabaseResponse.java | 20 ++++- .../embedding/vearch/ListSpaceResponse.java | 20 ++++- .../store/embedding/vearch/MetricType.java | 4 +- .../store/embedding/vearch/ModelParam.java | 19 +++++ .../embedding/vearch/ResponseWrapper.java | 19 +++++ .../embedding/vearch/RetrievalParam.java | 77 ++++++++++++++++++- .../store/embedding/vearch/RetrievalType.java | 17 +++- .../store/embedding/vearch/SearchRequest.java | 55 +++++++++++-- .../embedding/vearch/SearchResponse.java | 66 +++++++++++++--- .../store/embedding/vearch/SpaceEngine.java | 17 +++- .../embedding/vearch/SpacePropertyParam.java | 23 ++++++ .../embedding/vearch/SpacePropertyType.java | 16 ++-- .../embedding/vearch/SpaceStoreParam.java | 12 +++ .../embedding/vearch/SpaceStoreType.java | 9 +-- .../store/embedding/vearch/VearchApi.java | 2 +- .../store/embedding/vearch/VearchClient.java | 15 ++-- .../vearch/VearchEmbeddingStore.java | 76 +++++++++--------- .../vearch/DeleteSpaceLastOrderer.java | 32 -------- .../vearch/VearchEmbeddingStoreIT.java | 6 +- 26 files changed, 492 insertions(+), 139 deletions(-) delete mode 100644 langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/DeleteSpaceLastOrderer.java diff --git a/langchain4j-vearch/pom.xml b/langchain4j-vearch/pom.xml index 858a3ed94..890d9dacd 100644 --- a/langchain4j-vearch/pom.xml +++ b/langchain4j-vearch/pom.xml @@ -30,7 +30,7 @@ com.squareup.retrofit2 - converter-gson + converter-jackson diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkRequest.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkRequest.java index 4e994292b..d44ea85ab 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkRequest.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkRequest.java @@ -1,5 +1,9 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; @@ -7,10 +11,23 @@ import lombok.Setter; import java.util.List; import java.util.Map; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class BulkRequest { private List> documents; + + BulkRequest() { + + } + + BulkRequest(List> documents) { + this.documents = documents; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkResponse.java index bdb4290f7..b3ccf9d2b 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/BulkResponse.java @@ -1,17 +1,34 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class BulkResponse { private Integer status; private String error; - @SerializedName("_id") + @JsonProperty("_id") private String id; + + BulkResponse() { + + } + + BulkResponse(Integer status, String error, String id) { + this.status = status; + this.error = error; + this.id = id; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseRequest.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseRequest.java index 5edd16028..7beccff5a 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseRequest.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseRequest.java @@ -1,13 +1,30 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class CreateDatabaseRequest { private String name; + + CreateDatabaseRequest() { + + } + + CreateDatabaseRequest(String name) { + this.name = name; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseResponse.java index 34bb58f0a..08d721c21 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateDatabaseResponse.java @@ -1,14 +1,30 @@ package dev.langchain4j.store.embedding.vearch; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class CreateDatabaseResponse { private Long id; private String name; + + CreateDatabaseResponse() { + + } + + CreateDatabaseResponse(Long id, String name) { + this.id = id; + this.name = name; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceRequest.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceRequest.java index 187fc8017..8ac01bcb6 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceRequest.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceRequest.java @@ -1,8 +1,9 @@ package dev.langchain4j.store.embedding.vearch; -import dev.langchain4j.store.embedding.vearch.ModelParam; -import dev.langchain4j.store.embedding.vearch.SpaceEngine; -import dev.langchain4j.store.embedding.vearch.SpacePropertyParam; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; @@ -10,9 +11,14 @@ import lombok.Setter; import java.util.List; import java.util.Map; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class CreateSpaceRequest { private String name; @@ -21,4 +27,17 @@ class CreateSpaceRequest { private SpaceEngine engine; private Map properties; private List models; + + CreateSpaceRequest() { + + } + + CreateSpaceRequest(String name, Integer partitionNum, Integer replicaNum, SpaceEngine engine, Map properties, List models) { + this.name = name; + this.partitionNum = partitionNum; + this.replicaNum = replicaNum; + this.engine = engine; + this.properties = properties; + this.models = models; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceResponse.java index 74f2d958b..c3fc58ebf 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/CreateSpaceResponse.java @@ -1,14 +1,30 @@ package dev.langchain4j.store.embedding.vearch; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class CreateSpaceResponse { private Integer id; private String name; + + CreateSpaceResponse() { + + } + + CreateSpaceResponse(Integer id, String name) { + this.id = id; + this.name = name; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListDatabaseResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListDatabaseResponse.java index 68747fb14..e8c29397d 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListDatabaseResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListDatabaseResponse.java @@ -1,14 +1,30 @@ package dev.langchain4j.store.embedding.vearch; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class ListDatabaseResponse { private Integer id; private String name; + + ListDatabaseResponse() { + + } + + ListDatabaseResponse(Integer id, String name) { + this.id = id; + this.name = name; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListSpaceResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListSpaceResponse.java index ea9479a43..3dec218a2 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListSpaceResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ListSpaceResponse.java @@ -1,14 +1,30 @@ package dev.langchain4j.store.embedding.vearch; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public class ListSpaceResponse { private Integer id; private String name; + + ListSpaceResponse() { + + } + + ListSpaceResponse(Integer id, String name) { + this.id = id; + this.name = name; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/MetricType.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/MetricType.java index a18a2a8af..1870d3db1 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/MetricType.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/MetricType.java @@ -1,6 +1,6 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; +import com.fasterxml.jackson.annotation.JsonProperty; /** * if metric type is not set when searching, it will use the parameter specified when building the space @@ -12,6 +12,6 @@ public enum MetricType { /** * Inner Product */ - @SerializedName("InnerProduct") + @JsonProperty("InnerProduct") INNER_PRODUCT } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ModelParam.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ModelParam.java index 22054ade2..844813d82 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ModelParam.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ModelParam.java @@ -1,17 +1,36 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; import java.util.List; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public class ModelParam { private String modelId; private List fields; private String out; + + ModelParam() { + + } + + ModelParam(String modelId, List fields, String out) { + this.modelId = modelId; + this.fields = fields; + this.out = out; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ResponseWrapper.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ResponseWrapper.java index a74defe43..7188c8a91 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ResponseWrapper.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/ResponseWrapper.java @@ -1,15 +1,34 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class ResponseWrapper { private Integer code; private String msg; private T data; + + ResponseWrapper() { + + } + + ResponseWrapper(Integer code, String msg, T data) { + this.code = code; + this.msg = msg; + this.data = data; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalParam.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalParam.java index b35db1d0c..176d0ef75 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalParam.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalParam.java @@ -1,20 +1,35 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + /** * As a constraint of all engine type only * * @see RetrievalType */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public abstract class RetrievalParam { + protected RetrievalParam() { + + } + @Getter @Setter @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class IVFPQParam extends RetrievalParam { @Builder.Default @@ -31,11 +46,23 @@ public abstract class RetrievalParam { *

default 64, must be a multiple of 4

*/ private Integer nsubvector; + + public IVFPQParam() { + } + + public IVFPQParam(MetricType metricType, Integer ncentroids, Integer nsubvector) { + this.metricType = metricType; + this.ncentroids = ncentroids; + this.nsubvector = nsubvector; + } } @Getter @Setter @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class HNSWParam extends RetrievalParam { @Builder.Default @@ -52,13 +79,24 @@ public abstract class RetrievalParam { *

default 40

*

The higher the value, the better the construction effect, and the longer it takes

*/ - @SerializedName("efConstruction") private Integer efConstruction; + + public HNSWParam() { + } + + public HNSWParam(MetricType metricType, Integer nlinks, Integer efConstruction) { + this.metricType = metricType; + this.nlinks = nlinks; + this.efConstruction = efConstruction; + } } @Getter @Setter @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class GPUParam extends RetrievalParam { @Builder.Default @@ -75,11 +113,23 @@ public abstract class RetrievalParam { *

default 64

*/ private Integer nsubvector; + + public GPUParam() { + } + + public GPUParam(MetricType metricType, Integer ncentroids, Integer nsubvector) { + this.metricType = metricType; + this.ncentroids = ncentroids; + this.nsubvector = nsubvector; + } } @Getter @Setter @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class IVFFLATParam extends RetrievalParam { @Builder.Default @@ -90,6 +140,14 @@ public abstract class RetrievalParam { *

default 2048

*/ private Integer ncentroids; + + public IVFFLATParam() { + } + + public IVFFLATParam(MetricType metricType, Integer ncentroids) { + this.metricType = metricType; + this.ncentroids = ncentroids; + } } @Getter @@ -103,6 +161,14 @@ public abstract class RetrievalParam { *

default 256

*/ private Integer ncentroids; + + public BINARYIVFParam() { + + } + + public BINARYIVFParam(Integer ncentroids) { + this.ncentroids = ncentroids; + } } @Getter @@ -112,5 +178,12 @@ public abstract class RetrievalParam { @Builder.Default private MetricType metricType = MetricType.INNER_PRODUCT; + + public FLAT() { + } + + public FLAT(MetricType metricType) { + this.metricType = metricType; + } } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalType.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalType.java index 63538bb16..880b5e6bc 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalType.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/RetrievalType.java @@ -1,7 +1,15 @@ package dev.langchain4j.store.embedding.vearch; -import lombok.Getter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public enum RetrievalType { IVFPQ(RetrievalParam.IVFPQParam.class), @@ -11,10 +19,13 @@ public enum RetrievalType { BINARYIVF(RetrievalParam.BINARYIVFParam.class), FLAT(RetrievalParam.FLAT.class); - @Getter - private Class paramClass; + private final Class paramClass; RetrievalType(Class paramClass) { this.paramClass = paramClass; } + + public Class getParamClass() { + return paramClass; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchRequest.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchRequest.java index fa8abc936..948dda868 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchRequest.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchRequest.java @@ -1,35 +1,78 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; import java.util.List; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class SearchRequest { private QueryParam query; private Integer size; private List fields; - @Getter - @Setter - @Builder - public static class QueryParam { + SearchRequest() { - private List sum; + } + + SearchRequest(QueryParam query, Integer size, List fields) { + this.query = query; + this.size = size; + this.fields = fields; } @Getter @Setter @Builder - public static class VectorParam { + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) + static class QueryParam { + + private List sum; + + QueryParam() { + + } + + QueryParam(List sum) { + this.sum = sum; + } + } + + @Getter + @Setter + @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) + static class VectorParam { private String field; private List feature; private Double minScore; + + VectorParam() { + + } + + VectorParam(String field, List feature, Double minScore) { + this.field = field; + this.feature = feature; + this.minScore = minScore; + } } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchResponse.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchResponse.java index 00dbf7006..6d99c7b25 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchResponse.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SearchResponse.java @@ -1,48 +1,90 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; -import lombok.Builder; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; import java.util.List; import java.util.Map; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter -@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) class SearchResponse { private Integer took; - @SerializedName("timed_out") + @JsonProperty("timed_out") private Boolean timeout; /** * not support shards yet */ - @SerializedName("_shards") + @JsonProperty("_shards") private Object shards; private Hit hits; + SearchResponse() { + + } + + SearchResponse(Integer took, Boolean timeout, Hit hits, Object shards) { + this.took = took; + this.timeout = timeout; + this.hits = hits; + this.shards = shards; + } + @Getter @Setter - @Builder - public static class Hit { + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) + static class Hit { private Integer total; private Double maxScore; private List hits; + + Hit() { + + } + + Hit(Integer total, Double maxScore, List hits) { + this.total = total; + this.maxScore = maxScore; + this.hits = hits; + } } @Getter @Setter - @Builder - public static class SearchedDocument { + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) + static class SearchedDocument { - @SerializedName("_id") + @JsonProperty("_id") private String id; - @SerializedName("_score") + @JsonProperty("_score") private Double score; - @SerializedName("_source") + @JsonProperty("_source") private Map source; + + SearchedDocument() { + + } + + SearchedDocument(String id, Double score, Map source) { + this.id = id; + this.score = score; + this.source = source; + } } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceEngine.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceEngine.java index 1ede67568..50c6a6e77 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceEngine.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceEngine.java @@ -1,10 +1,19 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public class SpaceEngine { private String name; @@ -23,6 +32,10 @@ public class SpaceEngine { setRetrievalParam(retrievalParam); } + public static Builder builder() { + return new Builder(); + } + public void setRetrievalParam(RetrievalParam retrievalParam) { // do some constraint check Class clazz = retrievalType.getParamClass(); @@ -34,10 +47,6 @@ public class SpaceEngine { this.retrievalParam = retrievalParam; } - public static Builder builder() { - return new Builder(); - } - public static class Builder { private String name; diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyParam.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyParam.java index ad901981d..3782d73ce 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyParam.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyParam.java @@ -1,14 +1,25 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + /** * As a constraint type of all Space property only * * @see CreateSpaceRequest */ +@Getter +@Setter +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public abstract class SpacePropertyParam { protected SpacePropertyType type; @@ -19,6 +30,9 @@ public abstract class SpacePropertyParam { @Getter @Setter + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class StringParam extends SpacePropertyParam { /** @@ -44,6 +58,9 @@ public abstract class SpacePropertyParam { @Getter @Setter + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class IntegerParam extends SpacePropertyParam { /** @@ -66,6 +83,9 @@ public abstract class SpacePropertyParam { @Getter @Setter + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class FloatParam extends SpacePropertyParam { /** @@ -88,6 +108,9 @@ public abstract class SpacePropertyParam { @Getter @Setter + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class VectorParam extends SpacePropertyParam { private Boolean index; diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyType.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyType.java index de5f76470..bb41fbe90 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyType.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpacePropertyType.java @@ -1,26 +1,28 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; -import lombok.Getter; +import com.fasterxml.jackson.annotation.JsonProperty; public enum SpacePropertyType { /** * keyword is equivalent to string */ - @SerializedName("string") + @JsonProperty("string") STRING(SpacePropertyParam.StringParam.class), - @SerializedName("integer") + @JsonProperty("integer") INTEGER(SpacePropertyParam.IntegerParam.class), - @SerializedName("float") + @JsonProperty("float") FLOAT(SpacePropertyParam.FloatParam.class), - @SerializedName("vector") + @JsonProperty("vector") VECTOR(SpacePropertyParam.VectorParam.class); - @Getter private final Class paramClass; SpacePropertyType(Class paramClass) { this.paramClass = paramClass; } + + public Class getParamClass() { + return paramClass; + } } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreParam.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreParam.java index 48b4075bd..6eb8d3fd4 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreParam.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreParam.java @@ -1,12 +1,21 @@ package dev.langchain4j.store.embedding.vearch; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy; +import com.fasterxml.jackson.databind.annotation.JsonNaming; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; + @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(NON_NULL) +@JsonNaming(SnakeCaseStrategy.class) public class SpaceStoreParam { /** @@ -18,6 +27,9 @@ public class SpaceStoreParam { @Getter @Setter @Builder + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(NON_NULL) + @JsonNaming(SnakeCaseStrategy.class) public static class CompressRate { private Integer rate; diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreType.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreType.java index 0b09e6927..1906bb5f5 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreType.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/SpaceStoreType.java @@ -1,14 +1,13 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.annotations.SerializedName; -import lombok.Getter; +import com.fasterxml.jackson.annotation.JsonProperty; public enum SpaceStoreType { - @SerializedName("MemoryOnly") + @JsonProperty("MemoryOnly") MEMORY_ONLY, - @SerializedName("Mmap") + @JsonProperty("Mmap") M_MAP, - @SerializedName("RocksDB") + @JsonProperty("RocksDB") ROCKS_DB } diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchApi.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchApi.java index f28b0824e..a566efc1c 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchApi.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchApi.java @@ -6,7 +6,7 @@ import retrofit2.http.*; import java.util.List; -public interface VearchApi { +interface VearchApi { int OK = 200; diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchClient.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchClient.java index de7f4659e..a425b0957 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchClient.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchClient.java @@ -1,7 +1,6 @@ package dev.langchain4j.store.embedding.vearch; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; import dev.langchain4j.internal.Utils; import lombok.Builder; import okhttp3.MediaType; @@ -9,7 +8,7 @@ import okhttp3.OkHttpClient; import okhttp3.RequestBody; import retrofit2.Response; import retrofit2.Retrofit; -import retrofit2.converter.gson.GsonConverterFactory; +import retrofit2.converter.jackson.JacksonConverterFactory; import java.io.IOException; import java.time.Duration; @@ -17,14 +16,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; +import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT; import static dev.langchain4j.store.embedding.vearch.VearchApi.OK; class VearchClient { - private static final Gson GSON = new GsonBuilder() - .setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES) - .create(); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final VearchApi vearchApi; @@ -40,7 +37,7 @@ class VearchClient { Retrofit retrofit = new Retrofit.Builder() .baseUrl(Utils.ensureTrailingForwardSlash(baseUrl)) .client(okHttpClient) - .addConverterFactory(GsonConverterFactory.create(GSON)) + .addConverterFactory(JacksonConverterFactory.create(OBJECT_MAPPER)) .build(); vearchApi = retrofit.create(VearchApi.class); @@ -133,7 +130,7 @@ class VearchClient { fieldsExceptId.put(fieldName, value); } } - bodyString.append(GSON.toJson(fieldsExceptId)).append("\n"); + bodyString.append(OBJECT_MAPPER.writeValueAsString(fieldsExceptId)).append("\n"); } RequestBody body = RequestBody.create(bodyString.toString(), MediaType.parse("application/json; charset=utf-8")); Response> response = vearchApi.bulk(dbName, spaceName, body).execute(); diff --git a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStore.java b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStore.java index 4102efc1f..7c0dfeb76 100644 --- a/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStore.java +++ b/langchain4j-vearch/src/main/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStore.java @@ -56,44 +56,6 @@ public class VearchEmbeddingStore implements EmbeddingStore { return new Builder(); } - public static class Builder { - - private VearchConfig vearchConfig; - private String baseUrl; - private Duration timeout; - private Boolean normalizeEmbeddings; - - public Builder vearchConfig(VearchConfig vearchConfig) { - this.vearchConfig = vearchConfig; - return this; - } - - public Builder baseUrl(String baseUrl) { - this.baseUrl = baseUrl; - return this; - } - - public Builder timeout(Duration timeout) { - this.timeout = timeout; - return this; - } - - /** - * Set whether to normalize embedding when add to embedding store - * - * @param normalizeEmbeddings whether to normalize embedding when add to embedding store - * @return builder - */ - public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) { - this.normalizeEmbeddings = normalizeEmbeddings; - return this; - } - - public VearchEmbeddingStore build() { - return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings); - } - } - @Override public String add(Embedding embedding) { String id = randomUUID(); @@ -284,4 +246,42 @@ public class VearchEmbeddingStore implements EmbeddingStore { throw new RuntimeException("Unsupported SpacePropertyParam type " + param.type); } } + + public static class Builder { + + private VearchConfig vearchConfig; + private String baseUrl; + private Duration timeout; + private Boolean normalizeEmbeddings; + + public Builder vearchConfig(VearchConfig vearchConfig) { + this.vearchConfig = vearchConfig; + return this; + } + + public Builder baseUrl(String baseUrl) { + this.baseUrl = baseUrl; + return this; + } + + public Builder timeout(Duration timeout) { + this.timeout = timeout; + return this; + } + + /** + * Set whether to normalize embedding when add to embedding store + * + * @param normalizeEmbeddings whether to normalize embedding when add to embedding store + * @return builder + */ + public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) { + this.normalizeEmbeddings = normalizeEmbeddings; + return this; + } + + public VearchEmbeddingStore build() { + return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings); + } + } } diff --git a/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/DeleteSpaceLastOrderer.java b/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/DeleteSpaceLastOrderer.java deleted file mode 100644 index 109b1e2b4..000000000 --- a/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/DeleteSpaceLastOrderer.java +++ /dev/null @@ -1,32 +0,0 @@ -package dev.langchain4j.store.embedding.vearch; - -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.MethodOrdererContext; - -import java.util.ArrayList; -import java.util.List; - -public class DeleteSpaceLastOrderer implements MethodOrderer { - @Override - public void orderMethods(MethodOrdererContext methodOrdererContext) { - // should equal to VearchEmbeddingStoreIT#should_delete_space test name - String deleteSpaceTestName = "should_delete_space"; - List methodNames = new ArrayList<>(); - methodOrdererContext.getMethodDescriptors().forEach(methodDescriptor -> methodNames.add(methodDescriptor.getMethod().getName())); - methodNames.sort((methodName1, methodName2) -> { - // - if (methodName1.equals(deleteSpaceTestName)) { - return 1; - } else if (methodName2.equals(deleteSpaceTestName)) { - return -1; - } else { - return 0; - } - }); - methodOrdererContext.getMethodDescriptors().sort((md1, md2) -> { - int index1 = methodNames.indexOf(md1.getMethod().getName()); - int index2 = methodNames.indexOf(md2.getMethod().getName()); - return Integer.compare(index1, index2); - }); - } -} diff --git a/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStoreIT.java b/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStoreIT.java index b00b837d6..bacacc30f 100644 --- a/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStoreIT.java +++ b/langchain4j-vearch/src/test/java/dev/langchain4j/store/embedding/vearch/VearchEmbeddingStoreIT.java @@ -3,10 +3,12 @@ package dev.langchain4j.store.embedding.vearch; import dev.langchain4j.data.document.Metadata; import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.data.segment.TextSegment; -import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel; import dev.langchain4j.store.embedding.*; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Container;