dnrops
/
meilisearch
mirror of https://github.com/meilisearch/meilisearch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix compile errors

This commit is contained in:
mpostma 2021-01-14 11:27:07 +01:00
parent 334933b874
commit 686f987180
No known key found for this signature in database
GPG Key ID: CBC8A7C1D7A28C3A
6 changed files with 59 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -1211,6 +1211,7 @@ dependencies = [
"lmdb-rkv-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
"zerocopy",
@ -1636,6 +1637,7 @@ dependencies = [
"memmap",
"milli",
"mime",
"obkv",
"once_cell",
"page_size",
"rand 0.7.3",

3
Cargo.toml
View File

@ -30,7 +30,7 @@ fst = "0.4.5"
futures = "0.3.7"
futures-util = "0.3.8"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] }
heed = { version = "0.10.6", default-features = false, features = ["serde", "lmdb", "sync-read-txn"] }
http = "0.2.1"
indexmap = { version = "1.3.2", features = ["serde-1"] }
log = "0.4.8"
@ -60,6 +60,7 @@ walkdir = "2.3.1"
whoami = "1.0.0"
dashmap = "4.0.2"
page_size = "0.4.2"
obkv = "0.1.1"
[dependencies.sentry]
default-features = false

5
src/data/mod.rs
View File

@ -3,11 +3,9 @@ mod updates;
pub use search::{SearchQuery, SearchResult};
use std::fs::create_dir_all;
use std::ops::Deref;
use std::sync::Arc;
use milli::Index;
use sha2::Digest;
use crate::{option::Opt, updates::Settings};
@ -29,8 +27,7 @@ impl Deref for Data {
#[derive(Clone)]
pub struct DataInner {
pub indexes: Arc<IndexController>,
pub update_queue: Arc<UpdateQueue>,
pub indexes: Arc<IndexController<UpdateQueue>>,
api_keys: ApiKeys,
options: Opt,
}

22
src/data/search.rs
View File

@ -19,18 +19,18 @@ const fn default_search_limit() -> usize { DEFAULT_SEARCH_LIMIT }
#[serde(rename_all = "camelCase", deny_unknown_fields)]
#[allow(dead_code)]
pub struct SearchQuery {
q: Option<String>,
offset: Option<usize>,
pub q: Option<String>,
pub offset: Option<usize>,
#[serde(default = "default_search_limit")]
limit: usize,
attributes_to_retrieve: Option<Vec<String>>,
attributes_to_crop: Option<Vec<String>>,
crop_length: Option<usize>,
attributes_to_highlight: Option<Vec<String>>,
filters: Option<String>,
matches: Option<bool>,
facet_filters: Option<Value>,
facets_distribution: Option<Vec<String>>,
pub limit: usize,
pub attributes_to_retrieve: Option<Vec<String>>,
pub attributes_to_crop: Option<Vec<String>>,
pub crop_length: Option<usize>,
pub attributes_to_highlight: Option<Vec<String>>,
pub filters: Option<String>,
pub matches: Option<bool>,
pub facet_filters: Option<Value>,
pub facets_distribution: Option<Vec<String>>,
}
#[derive(Serialize)]

86
src/index_controller/mod.rs
View File

@ -1,25 +1,26 @@
use std::fs::File;
use std::io::{Read, Write};
use std::path::Path;
use std::ops::Deref;
use std::path::{Path, PathBuf};
use anyhow::Result;
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use dashmap::mapref::one::Ref;
use heed::types::{Str, SerdeBincode};
use heed::{EnvOpenOptions, Env, Database};
use milli::{Index, FieldsIdsMap};
use milli::{Index, FieldsIdsMap, SearchResult, FieldId};
use serde::{Serialize, Deserialize};
use crate::data::{SearchQuery, SearchResult};
use crate::data::SearchQuery;
const CONTROLLER_META_FILENAME: &str = "index_controller_meta";
const INDEXES_CONTROLLER_FILENAME: &str = "indexes_db";
const INDEXES_DB_NAME: &str = "indexes_db";
trait UpdateStore {}
pub trait UpdateStore {}
pub struct IndexController<U> {
path: PathBuf,
update_store: U,
env: Env,
indexes_db: Database<Str, SerdeBincode<IndexMetadata>>,
@ -34,7 +35,8 @@ struct IndexControllerMeta {
impl IndexControllerMeta {
fn from_path(path: impl AsRef<Path>) -> Result<Option<IndexControllerMeta>> {
let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME);
let mut path = path.as_ref().to_path_buf();
path.push(CONTROLLER_META_FILENAME);
if path.exists() {
let mut file = File::open(path)?;
let mut buffer = Vec::new();
@ -47,7 +49,8 @@ impl IndexControllerMeta {
}
fn to_path(self, path: impl AsRef<Path>) -> Result<()> {
let path = path.as_ref().to_path_buf().push(CONTROLLER_META_FILENAME);
let mut path = path.as_ref().to_path_buf();
path.push(CONTROLLER_META_FILENAME);
if path.exists() {
Err(anyhow::anyhow!("Index controller metadata already exists"))
} else {
@ -67,39 +70,24 @@ struct IndexMetadata {
}
impl IndexMetadata {
fn open_index(&self, path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref().to_path_buf().push("indexes").push(&self.id);
Ok(Index::new(self.options, path)?)
fn open_index(&self, path: impl AsRef<Path>) -> Result<Index> {
// create a path in the form "db_path/indexes/index_id"
let mut path = path.as_ref().to_path_buf();
path.push("indexes");
path.push(&self.id);
Ok(Index::new(self.open_options, path)?)
}
}
struct IndexView<'a, U> {
txn: heed::RoTxn<'a>,
index: &'a Index,
index: Ref<'a, String, Index>,
update_store: &'a U,
}
struct IndexViewMut<'a, U> {
txn: heed::RwTxn<'a, 'a>,
index: &'a Index,
update_store: &'a U,
}
impl<'a, U> Deref for IndexViewMut<'a, U> {
type Target = IndexView<'a, U>;
fn deref(&self) -> &Self::Target {
IndexView {
txn: *self.txn,
index: self.index,
update_store: self.update_store,
}
}
}
impl<'a, U: UpdateStore> IndexView<'a, U> {
pub fn search(&self, search_query: SearchQuery) -> Result<SearchResult> {
let mut search = self.index.search(self.txn);
let mut search = self.index.search(&self.txn);
if let Some(query) = &search_query.q {
search.query(query);
}
@ -115,15 +103,15 @@ impl<'a, U: UpdateStore> IndexView<'a, U> {
}
pub fn fields_ids_map(&self) -> Result<FieldsIdsMap> {
self.index.fields_ids_map(self.txn)
Ok(self.index.fields_ids_map(&self.txn)?)
}
pub fn fields_displayed_fields_ids(&self) -> Result<FieldsIdsMap> {
self.index.fields_displayed_fields_ids(self.txn)
pub fn fields_displayed_fields_ids(&self) -> Result<Option<Vec<FieldId>>> {
Ok(self.index.displayed_fields_ids(&self.txn)?)
}
pub fn documents(&self, ids: &[u32]) -> Result<Vec<()>> {
self.index.documents(self.txn, ids)
pub fn documents(&self, ids: Vec<u32>) -> Result<Vec<(u32, obkv::KvReader<'_>)>> {
Ok(self.index.documents(&self.txn, ids)?)
}
}
@ -133,28 +121,29 @@ impl<U: UpdateStore> IndexController<U> {
pub fn new(path: impl AsRef<Path>, update_store: U) -> Result<Self> {
// If index controller metadata is present, we return the env, otherwise, we create a new
// metadata from scratch before returning a new env.
let env = match IndexControllerMeta::from_path(path)? {
let path = path.as_ref().to_path_buf();
let env = match IndexControllerMeta::from_path(&path)? {
Some(meta) => meta.open_options.open(INDEXES_CONTROLLER_FILENAME)?,
None => {
let open_options = EnvOpenOptions::new()
.map_size(page_size::get() * 1000);
let env = open_options.open(INDEXES_CONTROLLER_FILENAME)?;
let created_at = Utc::now();
let meta = IndexControllerMeta { open_options, created_at };
let meta = IndexControllerMeta { open_options: open_options.clone(), created_at };
meta.to_path(path)?;
env
}
};
let indexes = DashMap::new();
let indexes_db = match env.open_database(INDEXES_DB_NAME)? {
let indexes_db = match env.open_database(Some(INDEXES_DB_NAME))? {
Some(indexes_db) => indexes_db,
None => env.create_database(INDEXES_DB_NAME)?,
None => env.create_database(Some(INDEXES_DB_NAME))?,
};
Ok(Self { env, indexes, indexes_db, update_store })
Ok(Self { env, indexes, indexes_db, update_store, path })
}
pub fn get_or_create<S: AsRef<str>>(&mut self, name: S) -> Result<IndexViewMut<'_, U>> {
pub fn get_or_create<S: AsRef<str>>(&mut self, name: S) -> Result<IndexView<'_, U>> {
todo!()
}
@ -162,19 +151,26 @@ impl<U: UpdateStore> IndexController<U> {
/// check for its exixtence in the indexes map, and if it doesn't exist, the index db is check
/// for metadata to launch the index.
pub fn get<S: AsRef<str>>(&self, name: S) -> Result<Option<IndexView<'_, U>>> {
let update_store = &self.update_store;
match self.indexes.get(name.as_ref()) {
Some(index) => {
let txn = index.read_txn()?;
let update_store = &self.update_store;
Ok(Some(IndexView { index, update_store, txn }))
}
None => {
let txn = self.env.read_txn()?;
match self.indexes_db.get(&txn, name.as_ref())? {
Some(meta) => {
let index = meta.open_index()?;
let index = meta.open_index(self.path)?;
self.indexes.insert(name.as_ref().to_owned(), index);
Ok(self.indexes.get(name.as_ref()))
// TODO: create index view
match self.indexes.get(name.as_ref()) {
Some(index) => {
let txn = index.read_txn()?;
Ok(Some(IndexView { index, txn, update_store }))
}
None => Ok(None)
}
}
None => Ok(None)
}
@ -182,7 +178,7 @@ impl<U: UpdateStore> IndexController<U> {
}
}
pub fn get_mut<S: AsRef<str>>(&self, name: S) -> Result<Option<IndexViewMut<'_, U>>> {
pub fn get_mut<S: AsRef<str>>(&self, name: S) -> Result<Option<IndexView<'_, U>>> {
todo!()
}

2
src/updates/mod.rs
View File

@ -55,6 +55,8 @@ pub struct UpdateQueue {
inner: Arc<UpdateStore<UpdateMeta, UpdateResult, String>>,
}
impl crate::index_controller::UpdateStore for UpdateQueue {}
impl Deref for UpdateQueue {
type Target = Arc<UpdateStore<UpdateMeta, UpdateResult, String>>;