Merge #1970

1970: Use milli reexported tokenizer  r=curquiza a=ManyTheFish

Use milli reexported tokenizer instead of importing meilisearch-tokenizer dependency.

fix #1888

Co-authored-by: many <maxime@meilisearch.com>

Cargo.lock

@@ -1685,7 +1685,6 @@ dependencies = [
  "meilisearch-auth",
  "meilisearch-error",
  "meilisearch-lib",
- "meilisearch-tokenizer 0.2.5",
  "mime",
  "num_cpus",
  "obkv",
@@ -1751,7 +1750,6 @@ dependencies = [
  "lazy_static",
  "log",
  "meilisearch-error",
- "meilisearch-tokenizer 0.2.5",
  "milli",
  "mime",
  "mockall",
@@ -1784,22 +1782,6 @@ dependencies = [
  "whoami",
 ]
 
-[[package]]
-name = "meilisearch-tokenizer"
-version = "0.2.5"
-source = "git+https://github.com/meilisearch/tokenizer.git?tag=v0.2.5#c0b5cf741ed9485147f2cbe523f2214d4fa4c395"
-dependencies = [
- "character_converter",
- "cow-utils",
- "deunicode",
- "fst",
- "jieba-rs",
- "once_cell",
- "slice-group-by",
- "unicode-segmentation",
- "whatlang",
-]
-
 [[package]]
 name = "meilisearch-tokenizer"
 version = "0.2.6"
@@ -1867,7 +1849,7 @@ dependencies = [
  "linked-hash-map",
  "log",
  "logging_timer",
- "meilisearch-tokenizer 0.2.6",
+ "meilisearch-tokenizer",
  "memmap2",
  "obkv",
  "once_cell",

meilisearch-http/Cargo.toml

@@ -50,7 +50,6 @@ log = "0.4.14"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-lib = { path = "../meilisearch-lib" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
 obkv = "0.2.0"

meilisearch-lib/Cargo.toml

@@ -29,7 +29,6 @@ itertools = "0.10.1"
 lazy_static = "1.4.0"
 log = "0.4.14"
 meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
 milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.21.0" }
 mime = "0.3.16"
 num_cpus = "1.13.0"

meilisearch-lib/src/index/search.rs

@@ -4,7 +4,7 @@ use std::time::Instant;
 
 use either::Either;
 use indexmap::IndexMap;
-use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
+use milli::tokenizer::{Analyzer, AnalyzerConfig, Token};
 use milli::{AscDesc, FieldId, FieldsIdsMap, Filter, MatchingWords, SortError};
 use regex::Regex;
 use serde::{Deserialize, Serialize};