Upgrade Milli and Tokenizer

This commit is contained in:
Clémentine Urquizar 2021-05-03 15:39:53 +02:00 committed by Marin Postma
parent 77740829bd
commit eb03a3ccb1
No known key found for this signature in database
GPG Key ID: D5241F0C0C865F30
2 changed files with 85 additions and 102 deletions

185
Cargo.lock generated
View File

@ -1,7 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "actix-codec"
version = "0.4.0"
@ -54,7 +52,7 @@ dependencies = [
"flate2",
"futures-core",
"futures-util",
"h2 0.3.2",
"h2 0.3.3",
"http",
"httparse",
"itoa",
@ -83,7 +81,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbcb2b608f0accc2f5bcf3dd872194ce13d94ee45b571487035864cf966b04ef"
dependencies = [
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -215,7 +213,7 @@ checksum = "7f138ac357a674c3b480ddb7bbd894b13c1b6e8927d728bc9ea5e17eee2f8fc9"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -266,9 +264,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "0.7.15"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
dependencies = [
"memchr",
]
@ -321,7 +319,7 @@ checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -332,7 +330,7 @@ checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -354,11 +352,12 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.57"
version = "0.3.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ed203b9ba68b242c62b3fb7480f589dd49829be1edb3fe8fc8b4ffda2dcb8d"
checksum = "88fb5a785d6b44fd9d6700935608639af1b8356de1e55d5f7c2740f4faa15d82"
dependencies = [
"addr2line",
"cc",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
@ -460,9 +459,9 @@ dependencies = [
[[package]]
name = "bstr"
version = "0.2.15"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279"
dependencies = [
"lazy_static",
"memchr",
@ -691,7 +690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ed27e177f16d65f0f0c22a213e17c696ace5dd64b14258b52f9417ccb52db4"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.3",
"crossbeam-utils 0.8.4",
]
[[package]]
@ -702,17 +701,17 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils 0.8.3",
"crossbeam-utils 0.8.4",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.3",
"crossbeam-utils 0.8.4",
"lazy_static",
"memoffset",
"scopeguard",
@ -739,9 +738,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
@ -789,7 +788,7 @@ dependencies = [
"convert_case",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -893,7 +892,7 @@ checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
"synstructure",
]
@ -1049,7 +1048,7 @@ dependencies = [
"proc-macro-hack",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -1182,9 +1181,9 @@ dependencies = [
[[package]]
name = "h2"
version = "0.3.2"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc018e188373e2777d0ef2467ebff62a08e66c3f5857b23c8fbec3018210dc00"
checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
dependencies = [
"bytes 1.0.1",
"fnv",
@ -1391,15 +1390,15 @@ dependencies = [
[[package]]
name = "hyper"
version = "0.14.6"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f006b8784cfb01fe7aa9c46f5f5cd4cf5c85a8c612a0653ec97642979062665"
checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54"
dependencies = [
"bytes 1.0.1",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.2",
"h2 0.3.3",
"http",
"http-body 0.4.1",
"httparse",
@ -1436,7 +1435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [
"futures-util",
"hyper 0.14.6",
"hyper 0.14.7",
"log",
"rustls 0.19.1",
"tokio 1.5.0",
@ -1615,9 +1614,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.93"
version = "0.2.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41"
checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e"
[[package]]
name = "linked-hash-map"
@ -1656,9 +1655,9 @@ checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4"
[[package]]
name = "lock_api"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176"
checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb"
dependencies = [
"scopeguard",
]
@ -1761,7 +1760,7 @@ dependencies = [
"log",
"main_error",
"meilisearch-error",
"meilisearch-tokenizer 0.2.1",
"meilisearch-tokenizer",
"memmap",
"milli",
"mime",
@ -1798,22 +1797,6 @@ dependencies = [
"zip",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.1"
source = "git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.1#b7a89c682b9f5d23a1d8075a99cca76069fff6c6"
dependencies = [
"character_converter",
"cow-utils",
"deunicode",
"fst",
"jieba-rs",
"once_cell",
"slice-group-by",
"unicode-segmentation",
"whatlang",
]
[[package]]
name = "meilisearch-tokenizer"
version = "0.2.2"
@ -1832,9 +1815,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.3.4"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "memmap"
@ -1878,7 +1861,7 @@ dependencies = [
"linked-hash-map",
"log",
"logging_timer",
"meilisearch-tokenizer 0.2.2",
"meilisearch-tokenizer",
"memmap",
"obkv",
"once_cell",
@ -2001,7 +1984,7 @@ dependencies = [
"cfg-if 1.0.0",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -2103,9 +2086,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "ordered-float"
version = "2.1.1"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "766f840da25490628d8e63e529cd21c014f6600c6b8517add12a6fa6167a6218"
checksum = "b50b8919aecb97e5ee9aceef27e24f39c46b11831130f4a6b7b091ec5de0de12"
dependencies = [
"num-traits",
]
@ -2215,7 +2198,7 @@ dependencies = [
"pest_meta",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -2284,7 +2267,7 @@ checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -2319,9 +2302,9 @@ checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "predicates"
version = "1.0.7"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeb433456c1a57cc93554dea3ce40b4c19c4057e41c55d4a0f3d84ea71c325aa"
checksum = "f49cfaf7fdaa3bfacc6fa3e7054e65148878354a5cfddcf661df4c851f8021df"
dependencies = [
"difference",
"float-cmp",
@ -2355,7 +2338,7 @@ dependencies = [
"proc-macro-error-attr",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
"version_check",
]
@ -2397,7 +2380,7 @@ version = "1.0.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec"
dependencies = [
"unicode-xid 0.2.1",
"unicode-xid 0.2.2",
]
[[package]]
@ -2572,7 +2555,7 @@ checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils 0.8.3",
"crossbeam-utils 0.8.4",
"lazy_static",
"num_cpus",
]
@ -2588,18 +2571,18 @@ dependencies = [
[[package]]
name = "redox_syscall"
version = "0.2.6"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8270314b5ccceb518e7e578952f0b72b88222d02e8f77f5ecf7abbb673539041"
checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.4.5"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b"
dependencies = [
"aho-corasick",
"memchr",
@ -2617,9 +2600,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.23"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "remove_dir_all"
@ -2680,7 +2663,7 @@ dependencies = [
"futures-util",
"http",
"http-body 0.4.1",
"hyper 0.14.6",
"hyper 0.14.7",
"hyper-rustls 0.22.1",
"ipnet",
"js-sys",
@ -2725,9 +2708,9 @@ dependencies = [
[[package]]
name = "roaring"
version = "0.6.6"
version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4b2e7ab0bbb2d144558ae3f4761a0db06d21463b45756fc64c3393cdba3d447"
checksum = "536cfa885fc388b8ae69edf96d7970849b7d9c1395da1b8330f17715babf8a09"
dependencies = [
"bytemuck",
"byteorder",
@ -2736,9 +2719,9 @@ dependencies = [
[[package]]
name = "rustc-demangle"
version = "0.1.18"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232"
checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce"
[[package]]
name = "rustc_version"
@ -2904,7 +2887,7 @@ checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3040,9 +3023,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "snap"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a"
checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451"
[[package]]
name = "socket2"
@ -3104,7 +3087,7 @@ dependencies = [
"quote 1.0.9",
"serde",
"serde_derive",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3120,7 +3103,7 @@ dependencies = [
"serde_derive",
"serde_json",
"sha1",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3156,7 +3139,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3172,13 +3155,13 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.70"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883"
checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"unicode-xid 0.2.1",
"unicode-xid 0.2.2",
]
[[package]]
@ -3198,8 +3181,8 @@ checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"unicode-xid 0.2.1",
"syn 1.0.72",
"unicode-xid 0.2.2",
]
[[package]]
@ -3272,7 +3255,7 @@ checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3321,7 +3304,7 @@ dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"standback",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3395,7 +3378,7 @@ checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
]
[[package]]
@ -3466,9 +3449,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.25"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01ebdc2bb4498ab1ab5f5b73c5803825e60199229ccba0698170e3be0e7f959f"
checksum = "09adeb8c97449311ccd28a427f96fb563e7fd31aabf994189879d9da2394b89d"
dependencies = [
"cfg-if 1.0.0",
"log",
@ -3478,9 +3461,9 @@ dependencies = [
[[package]]
name = "tracing-core"
version = "0.1.17"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f50de3927f93d202783f4513cda820ab47ef17f624b03c096e86ef00c67e6b5f"
checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
dependencies = [
"lazy_static",
]
@ -3575,9 +3558,9 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
[[package]]
name = "unicode-xid"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "untrusted"
@ -3606,9 +3589,9 @@ checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593"
[[package]]
name = "utf8-width"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9071ac216321a4470a69fb2b28cfc68dcd1a39acd877c8be8e014df6772d8efa"
checksum = "7cf7d77f457ef8dfa11e4cd5933c5ddb5dc52a94664071951219a97710f0a32b"
[[package]]
name = "uuid"
@ -3699,7 +3682,7 @@ dependencies = [
"log",
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
"wasm-bindgen-shared",
]
@ -3733,7 +3716,7 @@ checksum = "d53739ff08c8a68b0fdbcd54c372b8ab800b1449ab3c9d706503bc7dd1621b2c"
dependencies = [
"proc-macro2 1.0.26",
"quote 1.0.9",
"syn 1.0.70",
"syn 1.0.72",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@ -3874,12 +3857,12 @@ dependencies = [
[[package]]
name = "zerocopy-derive"
version = "0.2.1"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc9c39e6d503229ffa00cc2954af4a751e6bbedf2a2c18e856eb3ece93d32495"
checksum = "d498dbd1fd7beb83c86709ae1c33ca50942889473473d287d56ce4770a18edfb"
dependencies = [
"proc-macro2 1.0.26",
"syn 1.0.70",
"syn 1.0.72",
"synstructure",
]

View File

@ -49,7 +49,7 @@ itertools = "0.10.0"
log = "0.4.8"
main_error = "0.1.0"
meilisearch-error = { path = "../meilisearch-error" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.1" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.2" }
memmap = "0.7.0"
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.2.0" }
mime = "0.3.16"