Migrate Vearch from Gson to Jackson (#1706)

## Issue
Closes #1680 

## Change
Migrate Vearch from Gson to Jackson

## General checklist
<!-- Please double-check the following points and mark them like this:
[X] -->
- [x] There are no breaking changes
- [ ] I have added unit and integration tests for my change
- [x] I have manually run all the unit and integration tests in the
module I have added/changed, and they are all green
- [x] I have manually run all the unit and integration tests in the
[core](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core)
and
[main](https://github.com/langchain4j/langchain4j/tree/main/langchain4j)
modules, and they are all green
<!-- Before adding documentation and example(s) (below), please wait
until the PR is reviewed and approved. -->
- [ ] I have added/updated the
[documentation](https://github.com/langchain4j/langchain4j/tree/main/docs/docs)
- [ ] I have added an example in the [examples
repo](https://github.com/langchain4j/langchain4j-examples) (only for
"big" features)
- [ ] I have added/updated [Spring Boot
starter(s)](https://github.com/langchain4j/langchain4j-spring) (if
applicable)
This commit is contained in:
ZYinNJU 2024-09-04 20:49:36 +08:00 committed by GitHub
parent 673cf0c393
commit b29d31029f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 492 additions and 139 deletions

View File

@ -30,7 +30,7 @@
<dependency> <dependency>
<groupId>com.squareup.retrofit2</groupId> <groupId>com.squareup.retrofit2</groupId>
<artifactId>converter-gson</artifactId> <artifactId>converter-jackson</artifactId>
</dependency> </dependency>
<!-- DEPENDENCY CONFLICT RESOLUTION FOR OKHTTP (START) --> <!-- DEPENDENCY CONFLICT RESOLUTION FOR OKHTTP (START) -->

View File

@ -1,5 +1,9 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@ -7,10 +11,23 @@ import lombok.Setter;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class BulkRequest { class BulkRequest {
private List<Map<String, Object>> documents; private List<Map<String, Object>> documents;
BulkRequest() {
}
BulkRequest(List<Map<String, Object>> documents) {
this.documents = documents;
}
} }

View File

@ -1,17 +1,34 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class BulkResponse { class BulkResponse {
private Integer status; private Integer status;
private String error; private String error;
@SerializedName("_id") @JsonProperty("_id")
private String id; private String id;
BulkResponse() {
}
BulkResponse(Integer status, String error, String id) {
this.status = status;
this.error = error;
this.id = id;
}
} }

View File

@ -1,13 +1,30 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateDatabaseRequest { class CreateDatabaseRequest {
private String name; private String name;
CreateDatabaseRequest() {
}
CreateDatabaseRequest(String name) {
this.name = name;
}
} }

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateDatabaseResponse { class CreateDatabaseResponse {
private Long id; private Long id;
private String name; private String name;
CreateDatabaseResponse() {
}
CreateDatabaseResponse(Long id, String name) {
this.id = id;
this.name = name;
}
} }

View File

@ -1,8 +1,9 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import dev.langchain4j.store.embedding.vearch.ModelParam; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import dev.langchain4j.store.embedding.vearch.SpaceEngine; import com.fasterxml.jackson.annotation.JsonInclude;
import dev.langchain4j.store.embedding.vearch.SpacePropertyParam; import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
@ -10,9 +11,14 @@ import lombok.Setter;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateSpaceRequest { class CreateSpaceRequest {
private String name; private String name;
@ -21,4 +27,17 @@ class CreateSpaceRequest {
private SpaceEngine engine; private SpaceEngine engine;
private Map<String, SpacePropertyParam> properties; private Map<String, SpacePropertyParam> properties;
private List<ModelParam> models; private List<ModelParam> models;
CreateSpaceRequest() {
}
CreateSpaceRequest(String name, Integer partitionNum, Integer replicaNum, SpaceEngine engine, Map<String, SpacePropertyParam> properties, List<ModelParam> models) {
this.name = name;
this.partitionNum = partitionNum;
this.replicaNum = replicaNum;
this.engine = engine;
this.properties = properties;
this.models = models;
}
} }

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateSpaceResponse { class CreateSpaceResponse {
private Integer id; private Integer id;
private String name; private String name;
CreateSpaceResponse() {
}
CreateSpaceResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
} }

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class ListDatabaseResponse { class ListDatabaseResponse {
private Integer id; private Integer id;
private String name; private String name;
ListDatabaseResponse() {
}
ListDatabaseResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
} }

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class ListSpaceResponse { public class ListSpaceResponse {
private Integer id; private Integer id;
private String name; private String name;
ListSpaceResponse() {
}
ListSpaceResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
} }

View File

@ -1,6 +1,6 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonProperty;
/** /**
* if metric type is not set when searching, it will use the parameter specified when building the space * if metric type is not set when searching, it will use the parameter specified when building the space
@ -12,6 +12,6 @@ public enum MetricType {
/** /**
* Inner Product * Inner Product
*/ */
@SerializedName("InnerProduct") @JsonProperty("InnerProduct")
INNER_PRODUCT INNER_PRODUCT
} }

View File

@ -1,17 +1,36 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import java.util.List; import java.util.List;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class ModelParam { public class ModelParam {
private String modelId; private String modelId;
private List<String> fields; private List<String> fields;
private String out; private String out;
ModelParam() {
}
ModelParam(String modelId, List<String> fields, String out) {
this.modelId = modelId;
this.fields = fields;
this.out = out;
}
} }

View File

@ -1,15 +1,34 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class ResponseWrapper<T> { class ResponseWrapper<T> {
private Integer code; private Integer code;
private String msg; private String msg;
private T data; private T data;
ResponseWrapper() {
}
ResponseWrapper(Integer code, String msg, T data) {
this.code = code;
this.msg = msg;
this.data = data;
}
} }

View File

@ -1,20 +1,35 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
/** /**
* As a constraint of all engine type only * As a constraint of all engine type only
* *
* @see RetrievalType * @see RetrievalType
*/ */
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public abstract class RetrievalParam { public abstract class RetrievalParam {
protected RetrievalParam() {
}
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IVFPQParam extends RetrievalParam { public static class IVFPQParam extends RetrievalParam {
@Builder.Default @Builder.Default
@ -31,11 +46,23 @@ public abstract class RetrievalParam {
* <p>default 64, must be a multiple of 4</p> * <p>default 64, must be a multiple of 4</p>
*/ */
private Integer nsubvector; private Integer nsubvector;
public IVFPQParam() {
}
public IVFPQParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
this.metricType = metricType;
this.ncentroids = ncentroids;
this.nsubvector = nsubvector;
}
} }
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class HNSWParam extends RetrievalParam { public static class HNSWParam extends RetrievalParam {
@Builder.Default @Builder.Default
@ -52,13 +79,24 @@ public abstract class RetrievalParam {
* <p>default 40</p> * <p>default 40</p>
* <p>The higher the value, the better the construction effect, and the longer it takes</p> * <p>The higher the value, the better the construction effect, and the longer it takes</p>
*/ */
@SerializedName("efConstruction")
private Integer efConstruction; private Integer efConstruction;
public HNSWParam() {
}
public HNSWParam(MetricType metricType, Integer nlinks, Integer efConstruction) {
this.metricType = metricType;
this.nlinks = nlinks;
this.efConstruction = efConstruction;
}
} }
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class GPUParam extends RetrievalParam { public static class GPUParam extends RetrievalParam {
@Builder.Default @Builder.Default
@ -75,11 +113,23 @@ public abstract class RetrievalParam {
* <p>default 64</p> * <p>default 64</p>
*/ */
private Integer nsubvector; private Integer nsubvector;
public GPUParam() {
}
public GPUParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
this.metricType = metricType;
this.ncentroids = ncentroids;
this.nsubvector = nsubvector;
}
} }
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IVFFLATParam extends RetrievalParam { public static class IVFFLATParam extends RetrievalParam {
@Builder.Default @Builder.Default
@ -90,6 +140,14 @@ public abstract class RetrievalParam {
* <p>default 2048</p> * <p>default 2048</p>
*/ */
private Integer ncentroids; private Integer ncentroids;
public IVFFLATParam() {
}
public IVFFLATParam(MetricType metricType, Integer ncentroids) {
this.metricType = metricType;
this.ncentroids = ncentroids;
}
} }
@Getter @Getter
@ -103,6 +161,14 @@ public abstract class RetrievalParam {
* <p>default 256</p> * <p>default 256</p>
*/ */
private Integer ncentroids; private Integer ncentroids;
public BINARYIVFParam() {
}
public BINARYIVFParam(Integer ncentroids) {
this.ncentroids = ncentroids;
}
} }
@Getter @Getter
@ -112,5 +178,12 @@ public abstract class RetrievalParam {
@Builder.Default @Builder.Default
private MetricType metricType = MetricType.INNER_PRODUCT; private MetricType metricType = MetricType.INNER_PRODUCT;
public FLAT() {
}
public FLAT(MetricType metricType) {
this.metricType = metricType;
}
} }
} }

View File

@ -1,7 +1,15 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import lombok.Getter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public enum RetrievalType { public enum RetrievalType {
IVFPQ(RetrievalParam.IVFPQParam.class), IVFPQ(RetrievalParam.IVFPQParam.class),
@ -11,10 +19,13 @@ public enum RetrievalType {
BINARYIVF(RetrievalParam.BINARYIVFParam.class), BINARYIVF(RetrievalParam.BINARYIVFParam.class),
FLAT(RetrievalParam.FLAT.class); FLAT(RetrievalParam.FLAT.class);
@Getter private final Class<? extends RetrievalParam> paramClass;
private Class<? extends RetrievalParam> paramClass;
RetrievalType(Class<? extends RetrievalParam> paramClass) { RetrievalType(Class<? extends RetrievalParam> paramClass) {
this.paramClass = paramClass; this.paramClass = paramClass;
} }
public Class<? extends RetrievalParam> getParamClass() {
return paramClass;
}
} }

View File

@ -1,35 +1,78 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import java.util.List; import java.util.List;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class SearchRequest { class SearchRequest {
private QueryParam query; private QueryParam query;
private Integer size; private Integer size;
private List<String> fields; private List<String> fields;
@Getter SearchRequest() {
@Setter
@Builder
public static class QueryParam {
private List<VectorParam> sum; }
SearchRequest(QueryParam query, Integer size, List<String> fields) {
this.query = query;
this.size = size;
this.fields = fields;
} }
@Getter @Getter
@Setter @Setter
@Builder @Builder
public static class VectorParam { @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class QueryParam {
private List<VectorParam> sum;
QueryParam() {
}
QueryParam(List<VectorParam> sum) {
this.sum = sum;
}
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class VectorParam {
private String field; private String field;
private List<Float> feature; private List<Float> feature;
private Double minScore; private Double minScore;
VectorParam() {
}
VectorParam(String field, List<Float> feature, Double minScore) {
this.field = field;
this.feature = feature;
this.minScore = minScore;
}
} }
} }

View File

@ -1,48 +1,90 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Builder; import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class SearchResponse { class SearchResponse {
private Integer took; private Integer took;
@SerializedName("timed_out") @JsonProperty("timed_out")
private Boolean timeout; private Boolean timeout;
/** /**
* not support shards yet * not support shards yet
*/ */
@SerializedName("_shards") @JsonProperty("_shards")
private Object shards; private Object shards;
private Hit hits; private Hit hits;
SearchResponse() {
}
SearchResponse(Integer took, Boolean timeout, Hit hits, Object shards) {
this.took = took;
this.timeout = timeout;
this.hits = hits;
this.shards = shards;
}
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
public static class Hit { @JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class Hit {
private Integer total; private Integer total;
private Double maxScore; private Double maxScore;
private List<SearchedDocument> hits; private List<SearchedDocument> hits;
Hit() {
}
Hit(Integer total, Double maxScore, List<SearchedDocument> hits) {
this.total = total;
this.maxScore = maxScore;
this.hits = hits;
}
} }
@Getter @Getter
@Setter @Setter
@Builder @JsonIgnoreProperties(ignoreUnknown = true)
public static class SearchedDocument { @JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class SearchedDocument {
@SerializedName("_id") @JsonProperty("_id")
private String id; private String id;
@SerializedName("_score") @JsonProperty("_score")
private Double score; private Double score;
@SerializedName("_source") @JsonProperty("_source")
private Map<String, Object> source; private Map<String, Object> source;
SearchedDocument() {
}
SearchedDocument(String id, Double score, Map<String, Object> source) {
this.id = id;
this.score = score;
this.source = source;
}
} }
} }

View File

@ -1,10 +1,19 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class SpaceEngine { public class SpaceEngine {
private String name; private String name;
@ -23,6 +32,10 @@ public class SpaceEngine {
setRetrievalParam(retrievalParam); setRetrievalParam(retrievalParam);
} }
public static Builder builder() {
return new Builder();
}
public void setRetrievalParam(RetrievalParam retrievalParam) { public void setRetrievalParam(RetrievalParam retrievalParam) {
// do some constraint check // do some constraint check
Class<? extends RetrievalParam> clazz = retrievalType.getParamClass(); Class<? extends RetrievalParam> clazz = retrievalType.getParamClass();
@ -34,10 +47,6 @@ public class SpaceEngine {
this.retrievalParam = retrievalParam; this.retrievalParam = retrievalParam;
} }
public static Builder builder() {
return new Builder();
}
public static class Builder { public static class Builder {
private String name; private String name;

View File

@ -1,14 +1,25 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
/** /**
* As a constraint type of all Space property only * As a constraint type of all Space property only
* *
* @see CreateSpaceRequest * @see CreateSpaceRequest
*/ */
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public abstract class SpacePropertyParam { public abstract class SpacePropertyParam {
protected SpacePropertyType type; protected SpacePropertyType type;
@ -19,6 +30,9 @@ public abstract class SpacePropertyParam {
@Getter @Getter
@Setter @Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class StringParam extends SpacePropertyParam { public static class StringParam extends SpacePropertyParam {
/** /**
@ -44,6 +58,9 @@ public abstract class SpacePropertyParam {
@Getter @Getter
@Setter @Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IntegerParam extends SpacePropertyParam { public static class IntegerParam extends SpacePropertyParam {
/** /**
@ -66,6 +83,9 @@ public abstract class SpacePropertyParam {
@Getter @Getter
@Setter @Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class FloatParam extends SpacePropertyParam { public static class FloatParam extends SpacePropertyParam {
/** /**
@ -88,6 +108,9 @@ public abstract class SpacePropertyParam {
@Getter @Getter
@Setter @Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class VectorParam extends SpacePropertyParam { public static class VectorParam extends SpacePropertyParam {
private Boolean index; private Boolean index;

View File

@ -1,26 +1,28 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
public enum SpacePropertyType { public enum SpacePropertyType {
/** /**
* keyword is equivalent to string * keyword is equivalent to string
*/ */
@SerializedName("string") @JsonProperty("string")
STRING(SpacePropertyParam.StringParam.class), STRING(SpacePropertyParam.StringParam.class),
@SerializedName("integer") @JsonProperty("integer")
INTEGER(SpacePropertyParam.IntegerParam.class), INTEGER(SpacePropertyParam.IntegerParam.class),
@SerializedName("float") @JsonProperty("float")
FLOAT(SpacePropertyParam.FloatParam.class), FLOAT(SpacePropertyParam.FloatParam.class),
@SerializedName("vector") @JsonProperty("vector")
VECTOR(SpacePropertyParam.VectorParam.class); VECTOR(SpacePropertyParam.VectorParam.class);
@Getter
private final Class<? extends SpacePropertyParam> paramClass; private final Class<? extends SpacePropertyParam> paramClass;
SpacePropertyType(Class<? extends SpacePropertyParam> paramClass) { SpacePropertyType(Class<? extends SpacePropertyParam> paramClass) {
this.paramClass = paramClass; this.paramClass = paramClass;
} }
public Class<? extends SpacePropertyParam> getParamClass() {
return paramClass;
}
} }

View File

@ -1,12 +1,21 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder; import lombok.Builder;
import lombok.Getter; import lombok.Getter;
import lombok.Setter; import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class SpaceStoreParam { public class SpaceStoreParam {
/** /**
@ -18,6 +27,9 @@ public class SpaceStoreParam {
@Getter @Getter
@Setter @Setter
@Builder @Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class CompressRate { public static class CompressRate {
private Integer rate; private Integer rate;

View File

@ -1,14 +1,13 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName; import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Getter;
public enum SpaceStoreType { public enum SpaceStoreType {
@SerializedName("MemoryOnly") @JsonProperty("MemoryOnly")
MEMORY_ONLY, MEMORY_ONLY,
@SerializedName("Mmap") @JsonProperty("Mmap")
M_MAP, M_MAP,
@SerializedName("RocksDB") @JsonProperty("RocksDB")
ROCKS_DB ROCKS_DB
} }

View File

@ -6,7 +6,7 @@ import retrofit2.http.*;
import java.util.List; import java.util.List;
public interface VearchApi { interface VearchApi {
int OK = 200; int OK = 200;

View File

@ -1,7 +1,6 @@
package dev.langchain4j.store.embedding.vearch; package dev.langchain4j.store.embedding.vearch;
import com.google.gson.Gson; import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.GsonBuilder;
import dev.langchain4j.internal.Utils; import dev.langchain4j.internal.Utils;
import lombok.Builder; import lombok.Builder;
import okhttp3.MediaType; import okhttp3.MediaType;
@ -9,7 +8,7 @@ import okhttp3.OkHttpClient;
import okhttp3.RequestBody; import okhttp3.RequestBody;
import retrofit2.Response; import retrofit2.Response;
import retrofit2.Retrofit; import retrofit2.Retrofit;
import retrofit2.converter.gson.GsonConverterFactory; import retrofit2.converter.jackson.JacksonConverterFactory;
import java.io.IOException; import java.io.IOException;
import java.time.Duration; import java.time.Duration;
@ -17,14 +16,12 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES; import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
import static dev.langchain4j.store.embedding.vearch.VearchApi.OK; import static dev.langchain4j.store.embedding.vearch.VearchApi.OK;
class VearchClient { class VearchClient {
private static final Gson GSON = new GsonBuilder() private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
.setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES)
.create();
private final VearchApi vearchApi; private final VearchApi vearchApi;
@ -40,7 +37,7 @@ class VearchClient {
Retrofit retrofit = new Retrofit.Builder() Retrofit retrofit = new Retrofit.Builder()
.baseUrl(Utils.ensureTrailingForwardSlash(baseUrl)) .baseUrl(Utils.ensureTrailingForwardSlash(baseUrl))
.client(okHttpClient) .client(okHttpClient)
.addConverterFactory(GsonConverterFactory.create(GSON)) .addConverterFactory(JacksonConverterFactory.create(OBJECT_MAPPER))
.build(); .build();
vearchApi = retrofit.create(VearchApi.class); vearchApi = retrofit.create(VearchApi.class);
@ -133,7 +130,7 @@ class VearchClient {
fieldsExceptId.put(fieldName, value); fieldsExceptId.put(fieldName, value);
} }
} }
bodyString.append(GSON.toJson(fieldsExceptId)).append("\n"); bodyString.append(OBJECT_MAPPER.writeValueAsString(fieldsExceptId)).append("\n");
} }
RequestBody body = RequestBody.create(bodyString.toString(), MediaType.parse("application/json; charset=utf-8")); RequestBody body = RequestBody.create(bodyString.toString(), MediaType.parse("application/json; charset=utf-8"));
Response<List<BulkResponse>> response = vearchApi.bulk(dbName, spaceName, body).execute(); Response<List<BulkResponse>> response = vearchApi.bulk(dbName, spaceName, body).execute();

View File

@ -56,44 +56,6 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
return new Builder(); return new Builder();
} }
public static class Builder {
private VearchConfig vearchConfig;
private String baseUrl;
private Duration timeout;
private Boolean normalizeEmbeddings;
public Builder vearchConfig(VearchConfig vearchConfig) {
this.vearchConfig = vearchConfig;
return this;
}
public Builder baseUrl(String baseUrl) {
this.baseUrl = baseUrl;
return this;
}
public Builder timeout(Duration timeout) {
this.timeout = timeout;
return this;
}
/**
* Set whether to normalize embedding when add to embedding store
*
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
* @return builder
*/
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
this.normalizeEmbeddings = normalizeEmbeddings;
return this;
}
public VearchEmbeddingStore build() {
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
}
}
@Override @Override
public String add(Embedding embedding) { public String add(Embedding embedding) {
String id = randomUUID(); String id = randomUUID();
@ -284,4 +246,42 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
throw new RuntimeException("Unsupported SpacePropertyParam type " + param.type); throw new RuntimeException("Unsupported SpacePropertyParam type " + param.type);
} }
} }
public static class Builder {
private VearchConfig vearchConfig;
private String baseUrl;
private Duration timeout;
private Boolean normalizeEmbeddings;
public Builder vearchConfig(VearchConfig vearchConfig) {
this.vearchConfig = vearchConfig;
return this;
}
public Builder baseUrl(String baseUrl) {
this.baseUrl = baseUrl;
return this;
}
public Builder timeout(Duration timeout) {
this.timeout = timeout;
return this;
}
/**
* Set whether to normalize embedding when add to embedding store
*
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
* @return builder
*/
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
this.normalizeEmbeddings = normalizeEmbeddings;
return this;
}
public VearchEmbeddingStore build() {
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
}
}
} }

View File

@ -1,32 +0,0 @@
package dev.langchain4j.store.embedding.vearch;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.MethodOrdererContext;
import java.util.ArrayList;
import java.util.List;
public class DeleteSpaceLastOrderer implements MethodOrderer {
@Override
public void orderMethods(MethodOrdererContext methodOrdererContext) {
// should equal to VearchEmbeddingStoreIT#should_delete_space test name
String deleteSpaceTestName = "should_delete_space";
List<String> methodNames = new ArrayList<>();
methodOrdererContext.getMethodDescriptors().forEach(methodDescriptor -> methodNames.add(methodDescriptor.getMethod().getName()));
methodNames.sort((methodName1, methodName2) -> {
//
if (methodName1.equals(deleteSpaceTestName)) {
return 1;
} else if (methodName2.equals(deleteSpaceTestName)) {
return -1;
} else {
return 0;
}
});
methodOrdererContext.getMethodDescriptors().sort((md1, md2) -> {
int index1 = methodNames.indexOf(md1.getMethod().getName());
int index2 = methodNames.indexOf(md2.getMethod().getName());
return Integer.compare(index1, index2);
});
}
}

View File

@ -3,10 +3,12 @@ package dev.langchain4j.store.embedding.vearch;
import dev.langchain4j.data.document.Metadata; import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment; import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.store.embedding.*; import dev.langchain4j.store.embedding.*;
import org.junit.jupiter.api.*; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Container;