From 66f39aaa927ac1c254712d51a5337d148cfcceab Mon Sep 17 00:00:00 2001 From: mpostma Date: Wed, 29 Sep 2021 15:24:59 +0200 Subject: [PATCH] fix dump v3 --- .../dump_actor/loaders/mod.rs | 3 +- .../index_controller/dump_actor/loaders/v2.rs | 52 ------------------- .../src/index_controller/dump_actor/mod.rs | 46 ++++++++++++---- .../src/index_controller/update_file_store.rs | 5 ++ 4 files changed, 42 insertions(+), 64 deletions(-) delete mode 100644 meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs index ae6adc7cf..5b686dd05 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs @@ -1,2 +1,3 @@ pub mod v1; -pub mod v2; +pub mod v3; + diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs b/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs deleted file mode 100644 index 8280e9613..000000000 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::path::Path; - -use chrono::{DateTime, Utc}; -use log::info; -use serde::{Deserialize, Serialize}; - -use crate::index_controller::index_resolver::IndexResolver; -use crate::index_controller::update_file_store::UpdateFileStore; -use crate::index_controller::updates::store::UpdateStore; -use crate::options::IndexerOpts; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub struct MetadataV2 { - db_version: String, - index_db_size: usize, - update_db_size: usize, - dump_date: DateTime, -} - -impl MetadataV2 { - pub fn new(index_db_size: usize, update_db_size: usize) -> Self { - Self { - db_version: env!("CARGO_PKG_VERSION").to_string(), - index_db_size, - update_db_size, - dump_date: Utc::now(), - } - } - - pub fn load_dump( - self, - src: impl AsRef, - dst: impl AsRef, - index_db_size: usize, - update_db_size: usize, - indexing_options: &IndexerOpts, - ) -> anyhow::Result<()> { - info!( - "Loading dump from {}, dump database version: {}, dump version: V2", - self.dump_date, self.db_version - ); - - IndexResolver::load_dump(src.as_ref(), &dst, index_db_size, indexing_options)?; - UpdateFileStore::load_dump(src.as_ref(), &dst)?; - UpdateStore::load_dump(&src, &dst, update_db_size)?; - - info!("Loading indexes."); - - Ok(()) - } -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs index 4f32ade4d..12e819392 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/mod.rs @@ -8,7 +8,6 @@ use serde::{Deserialize, Serialize}; use tokio::fs::create_dir_all; use loaders::v1::MetadataV1; -use loaders::v2::MetadataV2; pub use actor::DumpActor; pub use handle_impl::*; @@ -18,6 +17,7 @@ use super::index_resolver::HardStateIndexResolver; use super::updates::UpdateSender; use crate::compression::{from_tar_gz, to_tar_gz}; use crate::index_controller::dump_actor::error::DumpActorError; +use crate::index_controller::dump_actor::loaders::v3; use crate::index_controller::updates::UpdateMsg; use crate::options::IndexerOpts; use error::Result; @@ -30,6 +30,27 @@ mod message; const META_FILE_NAME: &str = "metadata.json"; +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Metadata { + db_version: String, + index_db_size: usize, + update_db_size: usize, + dump_date: DateTime, +} + +impl Metadata { + pub fn new(index_db_size: usize, update_db_size: usize) -> Self { + Self { + db_version: env!("CARGO_PKG_VERSION").to_string(), + index_db_size, + update_db_size, + dump_date: Utc::now(), + } + } + +} + #[async_trait::async_trait] pub trait DumpActorHandle { /// Start the creation of a dump @@ -43,15 +64,16 @@ pub trait DumpActorHandle { #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "dumpVersion")] -pub enum Metadata { +pub enum MetadataVersion { V1(MetadataV1), - V2(MetadataV2), + V2(Metadata), + V3(Metadata), } -impl Metadata { - pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self { - let meta = MetadataV2::new(index_db_size, update_db_size); - Self::V2(meta) +impl MetadataVersion { + pub fn new_v3(index_db_size: usize, update_db_size: usize) -> Self { + let meta = Metadata::new(index_db_size, update_db_size); + Self::V3(meta) } } @@ -125,23 +147,25 @@ pub fn load_dump( let meta_path = tmp_src_path.join(META_FILE_NAME); let mut meta_file = File::open(&meta_path)?; - let meta: Metadata = serde_json::from_reader(&mut meta_file)?; + let meta: MetadataVersion = serde_json::from_reader(&mut meta_file)?; let tmp_dst = tempfile::tempdir()?; println!("temp path: {}", tmp_dst.path().display()); match meta { - Metadata::V1(meta) => { + MetadataVersion::V1(meta) => { meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)? } - Metadata::V2(meta) => meta.load_dump( + MetadataVersion::V3(meta) => v3::load_dump( + meta, &tmp_src_path, tmp_dst.path(), index_db_size, update_db_size, indexer_opts, )?, + MetadataVersion::V2(_) => todo!(), } // Persist and atomically rename the db let persisted_dump = tmp_dst.into_path(); @@ -173,7 +197,7 @@ impl DumpTask { let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??; let temp_dump_path = temp_dump_dir.path().to_owned(); - let meta = Metadata::new_v2(self.index_db_size, self.update_db_size); + let meta = MetadataVersion::new_v3(self.index_db_size, self.update_db_size); let meta_path = temp_dump_path.join(META_FILE_NAME); let mut meta_file = File::create(&meta_path)?; serde_json::to_writer(&mut meta_file, &meta)?; diff --git a/meilisearch-lib/src/index_controller/update_file_store.rs b/meilisearch-lib/src/index_controller/update_file_store.rs index 2a178d908..82ba0bffb 100644 --- a/meilisearch-lib/src/index_controller/update_file_store.rs +++ b/meilisearch-lib/src/index_controller/update_file_store.rs @@ -73,6 +73,11 @@ impl UpdateFileStore { let src_update_files_path = src.as_ref().join(UPDATE_FILES_PATH); let dst_update_files_path = dst.as_ref().join(UPDATE_FILES_PATH); + // No update files to load + if !src_update_files_path.exists() { + return Ok(()) + } + create_dir_all(&dst_update_files_path)?; println!("src_update file: {}", src_update_files_path.display());