Migrate Vearch from Gson to Jackson (#1706)
## Issue Closes #1680 ## Change Migrate Vearch from Gson to Jackson ## General checklist <!-- Please double-check the following points and mark them like this: [X] --> - [x] There are no breaking changes - [ ] I have added unit and integration tests for my change - [x] I have manually run all the unit and integration tests in the module I have added/changed, and they are all green - [x] I have manually run all the unit and integration tests in the [core](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core) and [main](https://github.com/langchain4j/langchain4j/tree/main/langchain4j) modules, and they are all green <!-- Before adding documentation and example(s) (below), please wait until the PR is reviewed and approved. --> - [ ] I have added/updated the [documentation](https://github.com/langchain4j/langchain4j/tree/main/docs/docs) - [ ] I have added an example in the [examples repo](https://github.com/langchain4j/langchain4j-examples) (only for "big" features) - [ ] I have added/updated [Spring Boot starter(s)](https://github.com/langchain4j/langchain4j-spring) (if applicable)
This commit is contained in:
parent
673cf0c393
commit
b29d31029f
|
@ -30,7 +30,7 @@
|
|||
|
||||
<dependency>
|
||||
<groupId>com.squareup.retrofit2</groupId>
|
||||
<artifactId>converter-gson</artifactId>
|
||||
<artifactId>converter-jackson</artifactId>
|
||||
</dependency>
|
||||
|
||||
<!-- DEPENDENCY CONFLICT RESOLUTION FOR OKHTTP (START) -->
|
||||
|
|
|
@ -1,5 +1,9 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
@ -7,10 +11,23 @@ import lombok.Setter;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class BulkRequest {
|
||||
|
||||
private List<Map<String, Object>> documents;
|
||||
|
||||
BulkRequest() {
|
||||
|
||||
}
|
||||
|
||||
BulkRequest(List<Map<String, Object>> documents) {
|
||||
this.documents = documents;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,17 +1,34 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class BulkResponse {
|
||||
|
||||
private Integer status;
|
||||
private String error;
|
||||
@SerializedName("_id")
|
||||
@JsonProperty("_id")
|
||||
private String id;
|
||||
|
||||
BulkResponse() {
|
||||
|
||||
}
|
||||
|
||||
BulkResponse(Integer status, String error, String id) {
|
||||
this.status = status;
|
||||
this.error = error;
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,13 +1,30 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class CreateDatabaseRequest {
|
||||
|
||||
private String name;
|
||||
|
||||
CreateDatabaseRequest() {
|
||||
|
||||
}
|
||||
|
||||
CreateDatabaseRequest(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,14 +1,30 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class CreateDatabaseResponse {
|
||||
|
||||
private Long id;
|
||||
private String name;
|
||||
|
||||
CreateDatabaseResponse() {
|
||||
|
||||
}
|
||||
|
||||
CreateDatabaseResponse(Long id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import dev.langchain4j.store.embedding.vearch.ModelParam;
|
||||
import dev.langchain4j.store.embedding.vearch.SpaceEngine;
|
||||
import dev.langchain4j.store.embedding.vearch.SpacePropertyParam;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
@ -10,9 +11,14 @@ import lombok.Setter;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class CreateSpaceRequest {
|
||||
|
||||
private String name;
|
||||
|
@ -21,4 +27,17 @@ class CreateSpaceRequest {
|
|||
private SpaceEngine engine;
|
||||
private Map<String, SpacePropertyParam> properties;
|
||||
private List<ModelParam> models;
|
||||
|
||||
CreateSpaceRequest() {
|
||||
|
||||
}
|
||||
|
||||
CreateSpaceRequest(String name, Integer partitionNum, Integer replicaNum, SpaceEngine engine, Map<String, SpacePropertyParam> properties, List<ModelParam> models) {
|
||||
this.name = name;
|
||||
this.partitionNum = partitionNum;
|
||||
this.replicaNum = replicaNum;
|
||||
this.engine = engine;
|
||||
this.properties = properties;
|
||||
this.models = models;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,14 +1,30 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class CreateSpaceResponse {
|
||||
|
||||
private Integer id;
|
||||
private String name;
|
||||
|
||||
CreateSpaceResponse() {
|
||||
|
||||
}
|
||||
|
||||
CreateSpaceResponse(Integer id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,14 +1,30 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class ListDatabaseResponse {
|
||||
|
||||
private Integer id;
|
||||
private String name;
|
||||
|
||||
ListDatabaseResponse() {
|
||||
|
||||
}
|
||||
|
||||
ListDatabaseResponse(Integer id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,14 +1,30 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public class ListSpaceResponse {
|
||||
|
||||
private Integer id;
|
||||
private String name;
|
||||
|
||||
ListSpaceResponse() {
|
||||
|
||||
}
|
||||
|
||||
ListSpaceResponse(Integer id, String name) {
|
||||
this.id = id;
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
/**
|
||||
* if metric type is not set when searching, it will use the parameter specified when building the space
|
||||
|
@ -12,6 +12,6 @@ public enum MetricType {
|
|||
/**
|
||||
* Inner Product
|
||||
*/
|
||||
@SerializedName("InnerProduct")
|
||||
@JsonProperty("InnerProduct")
|
||||
INNER_PRODUCT
|
||||
}
|
||||
|
|
|
@ -1,17 +1,36 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public class ModelParam {
|
||||
|
||||
private String modelId;
|
||||
private List<String> fields;
|
||||
private String out;
|
||||
|
||||
ModelParam() {
|
||||
|
||||
}
|
||||
|
||||
ModelParam(String modelId, List<String> fields, String out) {
|
||||
this.modelId = modelId;
|
||||
this.fields = fields;
|
||||
this.out = out;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,15 +1,34 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class ResponseWrapper<T> {
|
||||
|
||||
private Integer code;
|
||||
private String msg;
|
||||
private T data;
|
||||
|
||||
ResponseWrapper() {
|
||||
|
||||
}
|
||||
|
||||
ResponseWrapper(Integer code, String msg, T data) {
|
||||
this.code = code;
|
||||
this.msg = msg;
|
||||
this.data = data;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,20 +1,35 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
/**
|
||||
* As a constraint of all engine type only
|
||||
*
|
||||
* @see RetrievalType
|
||||
*/
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public abstract class RetrievalParam {
|
||||
|
||||
protected RetrievalParam() {
|
||||
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class IVFPQParam extends RetrievalParam {
|
||||
|
||||
@Builder.Default
|
||||
|
@ -31,11 +46,23 @@ public abstract class RetrievalParam {
|
|||
* <p>default 64, must be a multiple of 4</p>
|
||||
*/
|
||||
private Integer nsubvector;
|
||||
|
||||
public IVFPQParam() {
|
||||
}
|
||||
|
||||
public IVFPQParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
|
||||
this.metricType = metricType;
|
||||
this.ncentroids = ncentroids;
|
||||
this.nsubvector = nsubvector;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class HNSWParam extends RetrievalParam {
|
||||
|
||||
@Builder.Default
|
||||
|
@ -52,13 +79,24 @@ public abstract class RetrievalParam {
|
|||
* <p>default 40</p>
|
||||
* <p>The higher the value, the better the construction effect, and the longer it takes</p>
|
||||
*/
|
||||
@SerializedName("efConstruction")
|
||||
private Integer efConstruction;
|
||||
|
||||
public HNSWParam() {
|
||||
}
|
||||
|
||||
public HNSWParam(MetricType metricType, Integer nlinks, Integer efConstruction) {
|
||||
this.metricType = metricType;
|
||||
this.nlinks = nlinks;
|
||||
this.efConstruction = efConstruction;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class GPUParam extends RetrievalParam {
|
||||
|
||||
@Builder.Default
|
||||
|
@ -75,11 +113,23 @@ public abstract class RetrievalParam {
|
|||
* <p>default 64</p>
|
||||
*/
|
||||
private Integer nsubvector;
|
||||
|
||||
public GPUParam() {
|
||||
}
|
||||
|
||||
public GPUParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
|
||||
this.metricType = metricType;
|
||||
this.ncentroids = ncentroids;
|
||||
this.nsubvector = nsubvector;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class IVFFLATParam extends RetrievalParam {
|
||||
|
||||
@Builder.Default
|
||||
|
@ -90,6 +140,14 @@ public abstract class RetrievalParam {
|
|||
* <p>default 2048</p>
|
||||
*/
|
||||
private Integer ncentroids;
|
||||
|
||||
public IVFFLATParam() {
|
||||
}
|
||||
|
||||
public IVFFLATParam(MetricType metricType, Integer ncentroids) {
|
||||
this.metricType = metricType;
|
||||
this.ncentroids = ncentroids;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
|
@ -103,6 +161,14 @@ public abstract class RetrievalParam {
|
|||
* <p>default 256</p>
|
||||
*/
|
||||
private Integer ncentroids;
|
||||
|
||||
public BINARYIVFParam() {
|
||||
|
||||
}
|
||||
|
||||
public BINARYIVFParam(Integer ncentroids) {
|
||||
this.ncentroids = ncentroids;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
|
@ -112,5 +178,12 @@ public abstract class RetrievalParam {
|
|||
|
||||
@Builder.Default
|
||||
private MetricType metricType = MetricType.INNER_PRODUCT;
|
||||
|
||||
public FLAT() {
|
||||
}
|
||||
|
||||
public FLAT(MetricType metricType) {
|
||||
this.metricType = metricType;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,15 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import lombok.Getter;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public enum RetrievalType {
|
||||
|
||||
IVFPQ(RetrievalParam.IVFPQParam.class),
|
||||
|
@ -11,10 +19,13 @@ public enum RetrievalType {
|
|||
BINARYIVF(RetrievalParam.BINARYIVFParam.class),
|
||||
FLAT(RetrievalParam.FLAT.class);
|
||||
|
||||
@Getter
|
||||
private Class<? extends RetrievalParam> paramClass;
|
||||
private final Class<? extends RetrievalParam> paramClass;
|
||||
|
||||
RetrievalType(Class<? extends RetrievalParam> paramClass) {
|
||||
this.paramClass = paramClass;
|
||||
}
|
||||
|
||||
public Class<? extends RetrievalParam> getParamClass() {
|
||||
return paramClass;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,35 +1,78 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class SearchRequest {
|
||||
|
||||
private QueryParam query;
|
||||
private Integer size;
|
||||
private List<String> fields;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
public static class QueryParam {
|
||||
SearchRequest() {
|
||||
|
||||
private List<VectorParam> sum;
|
||||
}
|
||||
|
||||
SearchRequest(QueryParam query, Integer size, List<String> fields) {
|
||||
this.query = query;
|
||||
this.size = size;
|
||||
this.fields = fields;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
public static class VectorParam {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
static class QueryParam {
|
||||
|
||||
private List<VectorParam> sum;
|
||||
|
||||
QueryParam() {
|
||||
|
||||
}
|
||||
|
||||
QueryParam(List<VectorParam> sum) {
|
||||
this.sum = sum;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
static class VectorParam {
|
||||
|
||||
private String field;
|
||||
private List<Float> feature;
|
||||
private Double minScore;
|
||||
|
||||
VectorParam() {
|
||||
|
||||
}
|
||||
|
||||
VectorParam(String field, List<Float> feature, Double minScore) {
|
||||
this.field = field;
|
||||
this.feature = feature;
|
||||
this.minScore = minScore;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,48 +1,90 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import lombok.Builder;
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
class SearchResponse {
|
||||
|
||||
private Integer took;
|
||||
@SerializedName("timed_out")
|
||||
@JsonProperty("timed_out")
|
||||
private Boolean timeout;
|
||||
/**
|
||||
* not support shards yet
|
||||
*/
|
||||
@SerializedName("_shards")
|
||||
@JsonProperty("_shards")
|
||||
private Object shards;
|
||||
private Hit hits;
|
||||
|
||||
SearchResponse() {
|
||||
|
||||
}
|
||||
|
||||
SearchResponse(Integer took, Boolean timeout, Hit hits, Object shards) {
|
||||
this.took = took;
|
||||
this.timeout = timeout;
|
||||
this.hits = hits;
|
||||
this.shards = shards;
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
public static class Hit {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
static class Hit {
|
||||
|
||||
private Integer total;
|
||||
private Double maxScore;
|
||||
private List<SearchedDocument> hits;
|
||||
|
||||
Hit() {
|
||||
|
||||
}
|
||||
|
||||
Hit(Integer total, Double maxScore, List<SearchedDocument> hits) {
|
||||
this.total = total;
|
||||
this.maxScore = maxScore;
|
||||
this.hits = hits;
|
||||
}
|
||||
}
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
public static class SearchedDocument {
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
static class SearchedDocument {
|
||||
|
||||
@SerializedName("_id")
|
||||
@JsonProperty("_id")
|
||||
private String id;
|
||||
@SerializedName("_score")
|
||||
@JsonProperty("_score")
|
||||
private Double score;
|
||||
@SerializedName("_source")
|
||||
@JsonProperty("_source")
|
||||
private Map<String, Object> source;
|
||||
|
||||
SearchedDocument() {
|
||||
|
||||
}
|
||||
|
||||
SearchedDocument(String id, Double score, Map<String, Object> source) {
|
||||
this.id = id;
|
||||
this.score = score;
|
||||
this.source = source;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,10 +1,19 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public class SpaceEngine {
|
||||
|
||||
private String name;
|
||||
|
@ -23,6 +32,10 @@ public class SpaceEngine {
|
|||
setRetrievalParam(retrievalParam);
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public void setRetrievalParam(RetrievalParam retrievalParam) {
|
||||
// do some constraint check
|
||||
Class<? extends RetrievalParam> clazz = retrievalType.getParamClass();
|
||||
|
@ -34,10 +47,6 @@ public class SpaceEngine {
|
|||
this.retrievalParam = retrievalParam;
|
||||
}
|
||||
|
||||
public static Builder builder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private String name;
|
||||
|
|
|
@ -1,14 +1,25 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
/**
|
||||
* As a constraint type of all Space property only
|
||||
*
|
||||
* @see CreateSpaceRequest
|
||||
*/
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public abstract class SpacePropertyParam {
|
||||
|
||||
protected SpacePropertyType type;
|
||||
|
@ -19,6 +30,9 @@ public abstract class SpacePropertyParam {
|
|||
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class StringParam extends SpacePropertyParam {
|
||||
|
||||
/**
|
||||
|
@ -44,6 +58,9 @@ public abstract class SpacePropertyParam {
|
|||
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class IntegerParam extends SpacePropertyParam {
|
||||
|
||||
/**
|
||||
|
@ -66,6 +83,9 @@ public abstract class SpacePropertyParam {
|
|||
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class FloatParam extends SpacePropertyParam {
|
||||
|
||||
/**
|
||||
|
@ -88,6 +108,9 @@ public abstract class SpacePropertyParam {
|
|||
|
||||
@Getter
|
||||
@Setter
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class VectorParam extends SpacePropertyParam {
|
||||
|
||||
private Boolean index;
|
||||
|
|
|
@ -1,26 +1,28 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import lombok.Getter;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public enum SpacePropertyType {
|
||||
|
||||
/**
|
||||
* keyword is equivalent to string
|
||||
*/
|
||||
@SerializedName("string")
|
||||
@JsonProperty("string")
|
||||
STRING(SpacePropertyParam.StringParam.class),
|
||||
@SerializedName("integer")
|
||||
@JsonProperty("integer")
|
||||
INTEGER(SpacePropertyParam.IntegerParam.class),
|
||||
@SerializedName("float")
|
||||
@JsonProperty("float")
|
||||
FLOAT(SpacePropertyParam.FloatParam.class),
|
||||
@SerializedName("vector")
|
||||
@JsonProperty("vector")
|
||||
VECTOR(SpacePropertyParam.VectorParam.class);
|
||||
|
||||
@Getter
|
||||
private final Class<? extends SpacePropertyParam> paramClass;
|
||||
|
||||
SpacePropertyType(Class<? extends SpacePropertyParam> paramClass) {
|
||||
this.paramClass = paramClass;
|
||||
}
|
||||
|
||||
public Class<? extends SpacePropertyParam> getParamClass() {
|
||||
return paramClass;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,12 +1,21 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonInclude;
|
||||
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
|
||||
import com.fasterxml.jackson.databind.annotation.JsonNaming;
|
||||
import lombok.Builder;
|
||||
import lombok.Getter;
|
||||
import lombok.Setter;
|
||||
|
||||
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
|
||||
|
||||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public class SpaceStoreParam {
|
||||
|
||||
/**
|
||||
|
@ -18,6 +27,9 @@ public class SpaceStoreParam {
|
|||
@Getter
|
||||
@Setter
|
||||
@Builder
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
@JsonInclude(NON_NULL)
|
||||
@JsonNaming(SnakeCaseStrategy.class)
|
||||
public static class CompressRate {
|
||||
|
||||
private Integer rate;
|
||||
|
|
|
@ -1,14 +1,13 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.annotations.SerializedName;
|
||||
import lombok.Getter;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
||||
public enum SpaceStoreType {
|
||||
|
||||
@SerializedName("MemoryOnly")
|
||||
@JsonProperty("MemoryOnly")
|
||||
MEMORY_ONLY,
|
||||
@SerializedName("Mmap")
|
||||
@JsonProperty("Mmap")
|
||||
M_MAP,
|
||||
@SerializedName("RocksDB")
|
||||
@JsonProperty("RocksDB")
|
||||
ROCKS_DB
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ import retrofit2.http.*;
|
|||
|
||||
import java.util.List;
|
||||
|
||||
public interface VearchApi {
|
||||
interface VearchApi {
|
||||
|
||||
int OK = 200;
|
||||
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import dev.langchain4j.internal.Utils;
|
||||
import lombok.Builder;
|
||||
import okhttp3.MediaType;
|
||||
|
@ -9,7 +8,7 @@ import okhttp3.OkHttpClient;
|
|||
import okhttp3.RequestBody;
|
||||
import retrofit2.Response;
|
||||
import retrofit2.Retrofit;
|
||||
import retrofit2.converter.gson.GsonConverterFactory;
|
||||
import retrofit2.converter.jackson.JacksonConverterFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
|
@ -17,14 +16,12 @@ import java.util.HashMap;
|
|||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
|
||||
import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
|
||||
import static dev.langchain4j.store.embedding.vearch.VearchApi.OK;
|
||||
|
||||
class VearchClient {
|
||||
|
||||
private static final Gson GSON = new GsonBuilder()
|
||||
.setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES)
|
||||
.create();
|
||||
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
|
||||
|
||||
private final VearchApi vearchApi;
|
||||
|
||||
|
@ -40,7 +37,7 @@ class VearchClient {
|
|||
Retrofit retrofit = new Retrofit.Builder()
|
||||
.baseUrl(Utils.ensureTrailingForwardSlash(baseUrl))
|
||||
.client(okHttpClient)
|
||||
.addConverterFactory(GsonConverterFactory.create(GSON))
|
||||
.addConverterFactory(JacksonConverterFactory.create(OBJECT_MAPPER))
|
||||
.build();
|
||||
|
||||
vearchApi = retrofit.create(VearchApi.class);
|
||||
|
@ -133,7 +130,7 @@ class VearchClient {
|
|||
fieldsExceptId.put(fieldName, value);
|
||||
}
|
||||
}
|
||||
bodyString.append(GSON.toJson(fieldsExceptId)).append("\n");
|
||||
bodyString.append(OBJECT_MAPPER.writeValueAsString(fieldsExceptId)).append("\n");
|
||||
}
|
||||
RequestBody body = RequestBody.create(bodyString.toString(), MediaType.parse("application/json; charset=utf-8"));
|
||||
Response<List<BulkResponse>> response = vearchApi.bulk(dbName, spaceName, body).execute();
|
||||
|
|
|
@ -56,44 +56,6 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
|
|||
return new Builder();
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private VearchConfig vearchConfig;
|
||||
private String baseUrl;
|
||||
private Duration timeout;
|
||||
private Boolean normalizeEmbeddings;
|
||||
|
||||
public Builder vearchConfig(VearchConfig vearchConfig) {
|
||||
this.vearchConfig = vearchConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder baseUrl(String baseUrl) {
|
||||
this.baseUrl = baseUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder timeout(Duration timeout) {
|
||||
this.timeout = timeout;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to normalize embedding when add to embedding store
|
||||
*
|
||||
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
|
||||
* @return builder
|
||||
*/
|
||||
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
|
||||
this.normalizeEmbeddings = normalizeEmbeddings;
|
||||
return this;
|
||||
}
|
||||
|
||||
public VearchEmbeddingStore build() {
|
||||
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String add(Embedding embedding) {
|
||||
String id = randomUUID();
|
||||
|
@ -284,4 +246,42 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
|
|||
throw new RuntimeException("Unsupported SpacePropertyParam type " + param.type);
|
||||
}
|
||||
}
|
||||
|
||||
public static class Builder {
|
||||
|
||||
private VearchConfig vearchConfig;
|
||||
private String baseUrl;
|
||||
private Duration timeout;
|
||||
private Boolean normalizeEmbeddings;
|
||||
|
||||
public Builder vearchConfig(VearchConfig vearchConfig) {
|
||||
this.vearchConfig = vearchConfig;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder baseUrl(String baseUrl) {
|
||||
this.baseUrl = baseUrl;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder timeout(Duration timeout) {
|
||||
this.timeout = timeout;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether to normalize embedding when add to embedding store
|
||||
*
|
||||
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
|
||||
* @return builder
|
||||
*/
|
||||
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
|
||||
this.normalizeEmbeddings = normalizeEmbeddings;
|
||||
return this;
|
||||
}
|
||||
|
||||
public VearchEmbeddingStore build() {
|
||||
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,32 +0,0 @@
|
|||
package dev.langchain4j.store.embedding.vearch;
|
||||
|
||||
import org.junit.jupiter.api.MethodOrderer;
|
||||
import org.junit.jupiter.api.MethodOrdererContext;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DeleteSpaceLastOrderer implements MethodOrderer {
|
||||
@Override
|
||||
public void orderMethods(MethodOrdererContext methodOrdererContext) {
|
||||
// should equal to VearchEmbeddingStoreIT#should_delete_space test name
|
||||
String deleteSpaceTestName = "should_delete_space";
|
||||
List<String> methodNames = new ArrayList<>();
|
||||
methodOrdererContext.getMethodDescriptors().forEach(methodDescriptor -> methodNames.add(methodDescriptor.getMethod().getName()));
|
||||
methodNames.sort((methodName1, methodName2) -> {
|
||||
//
|
||||
if (methodName1.equals(deleteSpaceTestName)) {
|
||||
return 1;
|
||||
} else if (methodName2.equals(deleteSpaceTestName)) {
|
||||
return -1;
|
||||
} else {
|
||||
return 0;
|
||||
}
|
||||
});
|
||||
methodOrdererContext.getMethodDescriptors().sort((md1, md2) -> {
|
||||
int index1 = methodNames.indexOf(md1.getMethod().getName());
|
||||
int index2 = methodNames.indexOf(md2.getMethod().getName());
|
||||
return Integer.compare(index1, index2);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -3,10 +3,12 @@ package dev.langchain4j.store.embedding.vearch;
|
|||
import dev.langchain4j.data.document.Metadata;
|
||||
import dev.langchain4j.data.embedding.Embedding;
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
|
||||
import dev.langchain4j.model.embedding.EmbeddingModel;
|
||||
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
|
||||
import dev.langchain4j.store.embedding.*;
|
||||
import org.junit.jupiter.api.*;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.junit.jupiter.Container;
|
||||
|
|
Loading…
Reference in New Issue