Migrate Vearch from Gson to Jackson (#1706)

## Issue
Closes #1680 

## Change
Migrate Vearch from Gson to Jackson

## General checklist
<!-- Please double-check the following points and mark them like this:
[X] -->
- [x] There are no breaking changes
- [ ] I have added unit and integration tests for my change
- [x] I have manually run all the unit and integration tests in the
module I have added/changed, and they are all green
- [x] I have manually run all the unit and integration tests in the
[core](https://github.com/langchain4j/langchain4j/tree/main/langchain4j-core)
and
[main](https://github.com/langchain4j/langchain4j/tree/main/langchain4j)
modules, and they are all green
<!-- Before adding documentation and example(s) (below), please wait
until the PR is reviewed and approved. -->
- [ ] I have added/updated the
[documentation](https://github.com/langchain4j/langchain4j/tree/main/docs/docs)
- [ ] I have added an example in the [examples
repo](https://github.com/langchain4j/langchain4j-examples) (only for
"big" features)
- [ ] I have added/updated [Spring Boot
starter(s)](https://github.com/langchain4j/langchain4j-spring) (if
applicable)
This commit is contained in:
ZYinNJU 2024-09-04 20:49:36 +08:00 committed by GitHub
parent 673cf0c393
commit b29d31029f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 492 additions and 139 deletions

View File

@ -30,7 +30,7 @@
<dependency>
<groupId>com.squareup.retrofit2</groupId>
<artifactId>converter-gson</artifactId>
<artifactId>converter-jackson</artifactId>
</dependency>
<!-- DEPENDENCY CONFLICT RESOLUTION FOR OKHTTP (START) -->

View File

@ -1,5 +1,9 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
@ -7,10 +11,23 @@ import lombok.Setter;
import java.util.List;
import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class BulkRequest {
private List<Map<String, Object>> documents;
BulkRequest() {
}
BulkRequest(List<Map<String, Object>> documents) {
this.documents = documents;
}
}

View File

@ -1,17 +1,34 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class BulkResponse {
private Integer status;
private String error;
@SerializedName("_id")
@JsonProperty("_id")
private String id;
BulkResponse() {
}
BulkResponse(Integer status, String error, String id) {
this.status = status;
this.error = error;
this.id = id;
}
}

View File

@ -1,13 +1,30 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateDatabaseRequest {
private String name;
CreateDatabaseRequest() {
}
CreateDatabaseRequest(String name) {
this.name = name;
}
}

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateDatabaseResponse {
private Long id;
private String name;
CreateDatabaseResponse() {
}
CreateDatabaseResponse(Long id, String name) {
this.id = id;
this.name = name;
}
}

View File

@ -1,8 +1,9 @@
package dev.langchain4j.store.embedding.vearch;
import dev.langchain4j.store.embedding.vearch.ModelParam;
import dev.langchain4j.store.embedding.vearch.SpaceEngine;
import dev.langchain4j.store.embedding.vearch.SpacePropertyParam;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
@ -10,9 +11,14 @@ import lombok.Setter;
import java.util.List;
import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateSpaceRequest {
private String name;
@ -21,4 +27,17 @@ class CreateSpaceRequest {
private SpaceEngine engine;
private Map<String, SpacePropertyParam> properties;
private List<ModelParam> models;
CreateSpaceRequest() {
}
CreateSpaceRequest(String name, Integer partitionNum, Integer replicaNum, SpaceEngine engine, Map<String, SpacePropertyParam> properties, List<ModelParam> models) {
this.name = name;
this.partitionNum = partitionNum;
this.replicaNum = replicaNum;
this.engine = engine;
this.properties = properties;
this.models = models;
}
}

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class CreateSpaceResponse {
private Integer id;
private String name;
CreateSpaceResponse() {
}
CreateSpaceResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
}

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class ListDatabaseResponse {
private Integer id;
private String name;
ListDatabaseResponse() {
}
ListDatabaseResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
}

View File

@ -1,14 +1,30 @@
package dev.langchain4j.store.embedding.vearch;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class ListSpaceResponse {
private Integer id;
private String name;
ListSpaceResponse() {
}
ListSpaceResponse(Integer id, String name) {
this.id = id;
this.name = name;
}
}

View File

@ -1,6 +1,6 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* if metric type is not set when searching, it will use the parameter specified when building the space
@ -12,6 +12,6 @@ public enum MetricType {
/**
* Inner Product
*/
@SerializedName("InnerProduct")
@JsonProperty("InnerProduct")
INNER_PRODUCT
}

View File

@ -1,17 +1,36 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class ModelParam {
private String modelId;
private List<String> fields;
private String out;
ModelParam() {
}
ModelParam(String modelId, List<String> fields, String out) {
this.modelId = modelId;
this.fields = fields;
this.out = out;
}
}

View File

@ -1,15 +1,34 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class ResponseWrapper<T> {
private Integer code;
private String msg;
private T data;
ResponseWrapper() {
}
ResponseWrapper(Integer code, String msg, T data) {
this.code = code;
this.msg = msg;
this.data = data;
}
}

View File

@ -1,20 +1,35 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
/**
* As a constraint of all engine type only
*
* @see RetrievalType
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public abstract class RetrievalParam {
protected RetrievalParam() {
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IVFPQParam extends RetrievalParam {
@Builder.Default
@ -31,11 +46,23 @@ public abstract class RetrievalParam {
* <p>default 64, must be a multiple of 4</p>
*/
private Integer nsubvector;
public IVFPQParam() {
}
public IVFPQParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
this.metricType = metricType;
this.ncentroids = ncentroids;
this.nsubvector = nsubvector;
}
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class HNSWParam extends RetrievalParam {
@Builder.Default
@ -52,13 +79,24 @@ public abstract class RetrievalParam {
* <p>default 40</p>
* <p>The higher the value, the better the construction effect, and the longer it takes</p>
*/
@SerializedName("efConstruction")
private Integer efConstruction;
public HNSWParam() {
}
public HNSWParam(MetricType metricType, Integer nlinks, Integer efConstruction) {
this.metricType = metricType;
this.nlinks = nlinks;
this.efConstruction = efConstruction;
}
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class GPUParam extends RetrievalParam {
@Builder.Default
@ -75,11 +113,23 @@ public abstract class RetrievalParam {
* <p>default 64</p>
*/
private Integer nsubvector;
public GPUParam() {
}
public GPUParam(MetricType metricType, Integer ncentroids, Integer nsubvector) {
this.metricType = metricType;
this.ncentroids = ncentroids;
this.nsubvector = nsubvector;
}
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IVFFLATParam extends RetrievalParam {
@Builder.Default
@ -90,6 +140,14 @@ public abstract class RetrievalParam {
* <p>default 2048</p>
*/
private Integer ncentroids;
public IVFFLATParam() {
}
public IVFFLATParam(MetricType metricType, Integer ncentroids) {
this.metricType = metricType;
this.ncentroids = ncentroids;
}
}
@Getter
@ -103,6 +161,14 @@ public abstract class RetrievalParam {
* <p>default 256</p>
*/
private Integer ncentroids;
public BINARYIVFParam() {
}
public BINARYIVFParam(Integer ncentroids) {
this.ncentroids = ncentroids;
}
}
@Getter
@ -112,5 +178,12 @@ public abstract class RetrievalParam {
@Builder.Default
private MetricType metricType = MetricType.INNER_PRODUCT;
public FLAT() {
}
public FLAT(MetricType metricType) {
this.metricType = metricType;
}
}
}

View File

@ -1,7 +1,15 @@
package dev.langchain4j.store.embedding.vearch;
import lombok.Getter;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public enum RetrievalType {
IVFPQ(RetrievalParam.IVFPQParam.class),
@ -11,10 +19,13 @@ public enum RetrievalType {
BINARYIVF(RetrievalParam.BINARYIVFParam.class),
FLAT(RetrievalParam.FLAT.class);
@Getter
private Class<? extends RetrievalParam> paramClass;
private final Class<? extends RetrievalParam> paramClass;
RetrievalType(Class<? extends RetrievalParam> paramClass) {
this.paramClass = paramClass;
}
public Class<? extends RetrievalParam> getParamClass() {
return paramClass;
}
}

View File

@ -1,35 +1,78 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class SearchRequest {
private QueryParam query;
private Integer size;
private List<String> fields;
@Getter
@Setter
@Builder
public static class QueryParam {
SearchRequest() {
private List<VectorParam> sum;
}
SearchRequest(QueryParam query, Integer size, List<String> fields) {
this.query = query;
this.size = size;
this.fields = fields;
}
@Getter
@Setter
@Builder
public static class VectorParam {
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class QueryParam {
private List<VectorParam> sum;
QueryParam() {
}
QueryParam(List<VectorParam> sum) {
this.sum = sum;
}
}
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class VectorParam {
private String field;
private List<Float> feature;
private Double minScore;
VectorParam() {
}
VectorParam(String field, List<Float> feature, Double minScore) {
this.field = field;
this.feature = feature;
this.minScore = minScore;
}
}
}

View File

@ -1,48 +1,90 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import lombok.Builder;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
class SearchResponse {
private Integer took;
@SerializedName("timed_out")
@JsonProperty("timed_out")
private Boolean timeout;
/**
* not support shards yet
*/
@SerializedName("_shards")
@JsonProperty("_shards")
private Object shards;
private Hit hits;
SearchResponse() {
}
SearchResponse(Integer took, Boolean timeout, Hit hits, Object shards) {
this.took = took;
this.timeout = timeout;
this.hits = hits;
this.shards = shards;
}
@Getter
@Setter
@Builder
public static class Hit {
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class Hit {
private Integer total;
private Double maxScore;
private List<SearchedDocument> hits;
Hit() {
}
Hit(Integer total, Double maxScore, List<SearchedDocument> hits) {
this.total = total;
this.maxScore = maxScore;
this.hits = hits;
}
}
@Getter
@Setter
@Builder
public static class SearchedDocument {
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
static class SearchedDocument {
@SerializedName("_id")
@JsonProperty("_id")
private String id;
@SerializedName("_score")
@JsonProperty("_score")
private Double score;
@SerializedName("_source")
@JsonProperty("_source")
private Map<String, Object> source;
SearchedDocument() {
}
SearchedDocument(String id, Double score, Map<String, Object> source) {
this.id = id;
this.score = score;
this.source = source;
}
}
}

View File

@ -1,10 +1,19 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class SpaceEngine {
private String name;
@ -23,6 +32,10 @@ public class SpaceEngine {
setRetrievalParam(retrievalParam);
}
public static Builder builder() {
return new Builder();
}
public void setRetrievalParam(RetrievalParam retrievalParam) {
// do some constraint check
Class<? extends RetrievalParam> clazz = retrievalType.getParamClass();
@ -34,10 +47,6 @@ public class SpaceEngine {
this.retrievalParam = retrievalParam;
}
public static Builder builder() {
return new Builder();
}
public static class Builder {
private String name;

View File

@ -1,14 +1,25 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
/**
* As a constraint type of all Space property only
*
* @see CreateSpaceRequest
*/
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public abstract class SpacePropertyParam {
protected SpacePropertyType type;
@ -19,6 +30,9 @@ public abstract class SpacePropertyParam {
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class StringParam extends SpacePropertyParam {
/**
@ -44,6 +58,9 @@ public abstract class SpacePropertyParam {
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class IntegerParam extends SpacePropertyParam {
/**
@ -66,6 +83,9 @@ public abstract class SpacePropertyParam {
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class FloatParam extends SpacePropertyParam {
/**
@ -88,6 +108,9 @@ public abstract class SpacePropertyParam {
@Getter
@Setter
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class VectorParam extends SpacePropertyParam {
private Boolean index;

View File

@ -1,26 +1,28 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import com.fasterxml.jackson.annotation.JsonProperty;
public enum SpacePropertyType {
/**
* keyword is equivalent to string
*/
@SerializedName("string")
@JsonProperty("string")
STRING(SpacePropertyParam.StringParam.class),
@SerializedName("integer")
@JsonProperty("integer")
INTEGER(SpacePropertyParam.IntegerParam.class),
@SerializedName("float")
@JsonProperty("float")
FLOAT(SpacePropertyParam.FloatParam.class),
@SerializedName("vector")
@JsonProperty("vector")
VECTOR(SpacePropertyParam.VectorParam.class);
@Getter
private final Class<? extends SpacePropertyParam> paramClass;
SpacePropertyType(Class<? extends SpacePropertyParam> paramClass) {
this.paramClass = paramClass;
}
public Class<? extends SpacePropertyParam> getParamClass() {
return paramClass;
}
}

View File

@ -1,12 +1,21 @@
package dev.langchain4j.store.embedding.vearch;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.PropertyNamingStrategies.SnakeCaseStrategy;
import com.fasterxml.jackson.databind.annotation.JsonNaming;
import lombok.Builder;
import lombok.Getter;
import lombok.Setter;
import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL;
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public class SpaceStoreParam {
/**
@ -18,6 +27,9 @@ public class SpaceStoreParam {
@Getter
@Setter
@Builder
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonInclude(NON_NULL)
@JsonNaming(SnakeCaseStrategy.class)
public static class CompressRate {
private Integer rate;

View File

@ -1,14 +1,13 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import com.fasterxml.jackson.annotation.JsonProperty;
public enum SpaceStoreType {
@SerializedName("MemoryOnly")
@JsonProperty("MemoryOnly")
MEMORY_ONLY,
@SerializedName("Mmap")
@JsonProperty("Mmap")
M_MAP,
@SerializedName("RocksDB")
@JsonProperty("RocksDB")
ROCKS_DB
}

View File

@ -6,7 +6,7 @@ import retrofit2.http.*;
import java.util.List;
public interface VearchApi {
interface VearchApi {
int OK = 200;

View File

@ -1,7 +1,6 @@
package dev.langchain4j.store.embedding.vearch;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
import dev.langchain4j.internal.Utils;
import lombok.Builder;
import okhttp3.MediaType;
@ -9,7 +8,7 @@ import okhttp3.OkHttpClient;
import okhttp3.RequestBody;
import retrofit2.Response;
import retrofit2.Retrofit;
import retrofit2.converter.gson.GsonConverterFactory;
import retrofit2.converter.jackson.JacksonConverterFactory;
import java.io.IOException;
import java.time.Duration;
@ -17,14 +16,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
import static com.fasterxml.jackson.databind.SerializationFeature.INDENT_OUTPUT;
import static dev.langchain4j.store.embedding.vearch.VearchApi.OK;
class VearchClient {
private static final Gson GSON = new GsonBuilder()
.setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES)
.create();
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final VearchApi vearchApi;
@ -40,7 +37,7 @@ class VearchClient {
Retrofit retrofit = new Retrofit.Builder()
.baseUrl(Utils.ensureTrailingForwardSlash(baseUrl))
.client(okHttpClient)
.addConverterFactory(GsonConverterFactory.create(GSON))
.addConverterFactory(JacksonConverterFactory.create(OBJECT_MAPPER))
.build();
vearchApi = retrofit.create(VearchApi.class);
@ -133,7 +130,7 @@ class VearchClient {
fieldsExceptId.put(fieldName, value);
}
}
bodyString.append(GSON.toJson(fieldsExceptId)).append("\n");
bodyString.append(OBJECT_MAPPER.writeValueAsString(fieldsExceptId)).append("\n");
}
RequestBody body = RequestBody.create(bodyString.toString(), MediaType.parse("application/json; charset=utf-8"));
Response<List<BulkResponse>> response = vearchApi.bulk(dbName, spaceName, body).execute();

View File

@ -56,44 +56,6 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
return new Builder();
}
public static class Builder {
private VearchConfig vearchConfig;
private String baseUrl;
private Duration timeout;
private Boolean normalizeEmbeddings;
public Builder vearchConfig(VearchConfig vearchConfig) {
this.vearchConfig = vearchConfig;
return this;
}
public Builder baseUrl(String baseUrl) {
this.baseUrl = baseUrl;
return this;
}
public Builder timeout(Duration timeout) {
this.timeout = timeout;
return this;
}
/**
* Set whether to normalize embedding when add to embedding store
*
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
* @return builder
*/
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
this.normalizeEmbeddings = normalizeEmbeddings;
return this;
}
public VearchEmbeddingStore build() {
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
}
}
@Override
public String add(Embedding embedding) {
String id = randomUUID();
@ -284,4 +246,42 @@ public class VearchEmbeddingStore implements EmbeddingStore<TextSegment> {
throw new RuntimeException("Unsupported SpacePropertyParam type " + param.type);
}
}
public static class Builder {
private VearchConfig vearchConfig;
private String baseUrl;
private Duration timeout;
private Boolean normalizeEmbeddings;
public Builder vearchConfig(VearchConfig vearchConfig) {
this.vearchConfig = vearchConfig;
return this;
}
public Builder baseUrl(String baseUrl) {
this.baseUrl = baseUrl;
return this;
}
public Builder timeout(Duration timeout) {
this.timeout = timeout;
return this;
}
/**
* Set whether to normalize embedding when add to embedding store
*
* @param normalizeEmbeddings whether to normalize embedding when add to embedding store
* @return builder
*/
public Builder normalizeEmbeddings(Boolean normalizeEmbeddings) {
this.normalizeEmbeddings = normalizeEmbeddings;
return this;
}
public VearchEmbeddingStore build() {
return new VearchEmbeddingStore(baseUrl, timeout, vearchConfig, normalizeEmbeddings);
}
}
}

View File

@ -1,32 +0,0 @@
package dev.langchain4j.store.embedding.vearch;
import org.junit.jupiter.api.MethodOrderer;
import org.junit.jupiter.api.MethodOrdererContext;
import java.util.ArrayList;
import java.util.List;
public class DeleteSpaceLastOrderer implements MethodOrderer {
@Override
public void orderMethods(MethodOrdererContext methodOrdererContext) {
// should equal to VearchEmbeddingStoreIT#should_delete_space test name
String deleteSpaceTestName = "should_delete_space";
List<String> methodNames = new ArrayList<>();
methodOrdererContext.getMethodDescriptors().forEach(methodDescriptor -> methodNames.add(methodDescriptor.getMethod().getName()));
methodNames.sort((methodName1, methodName2) -> {
//
if (methodName1.equals(deleteSpaceTestName)) {
return 1;
} else if (methodName2.equals(deleteSpaceTestName)) {
return -1;
} else {
return 0;
}
});
methodOrdererContext.getMethodDescriptors().sort((md1, md2) -> {
int index1 = methodNames.indexOf(md1.getMethod().getName());
int index2 = methodNames.indexOf(md2.getMethod().getName());
return Integer.compare(index1, index2);
});
}
}

View File

@ -3,10 +3,12 @@ package dev.langchain4j.store.embedding.vearch;
import dev.langchain4j.data.document.Metadata;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.model.embedding.onnx.allminilml6v2q.AllMiniLmL6V2QuantizedEmbeddingModel;
import dev.langchain4j.store.embedding.*;
import org.junit.jupiter.api.*;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.junit.jupiter.Container;