1965: Reintroduce engine version file r=MarinPostma a=irevoire

Right now if you boot up MeiliSearch and point it to a DB directory created with a previous version of MeiliSearch the existing indexes will be deleted. This [used to be](51d7c84e73) prevented by a startup check which would compare the current engine version vs what was stored in the DB directory's version file, but this functionality seems to have been lost after a few refactorings of the code.

In order to go back to the old behavior we'll need to reintroduce the `VERSION` file that used to be present; I considered reusing the `metadata.json` file used in the dumps feature, but this seemed like the simpler and more approach. As the intent is just to restore functionality, the implementation is quite basic. I imagine that in the future we could build on this and do things like compatibility across major/minor versions and even migrating between formats.

This PR was made thanks to `@mbStavola` and is basically a port of his PR #1860 after a big refacto of the code #1796.

Closes #1840

Co-authored-by: Matt Stavola <m.freitas@offensive-security.com>
This commit is contained in:
bors[bot] 2021-12-06 13:39:37 +00:00 committed by GitHub
commit 948615537b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 91 additions and 0 deletions

View File

@ -39,6 +39,7 @@ use crate::update_file_store::UpdateFileStore;
mod dump_actor; mod dump_actor;
pub mod error; pub mod error;
pub mod versioning;
/// Concrete implementation of the IndexController, exposed by meilisearch-lib /// Concrete implementation of the IndexController, exposed by meilisearch-lib
pub type MeiliSearch = IndexController<HeedMetaStore, MapIndexStore>; pub type MeiliSearch = IndexController<HeedMetaStore, MapIndexStore>;
@ -162,6 +163,11 @@ impl IndexControllerBuilder {
.max_task_store_size .max_task_store_size
.ok_or_else(|| anyhow::anyhow!("Missing update database size"))?; .ok_or_else(|| anyhow::anyhow!("Missing update database size"))?;
let db_exists = db_path.as_ref().exists();
if db_exists {
versioning::check_version_file(db_path.as_ref())?;
}
if let Some(ref path) = self.import_snapshot { if let Some(ref path) = self.import_snapshot {
log::info!("Loading from snapshot {:?}", path); log::info!("Loading from snapshot {:?}", path);
load_snapshot( load_snapshot(
@ -189,6 +195,8 @@ impl IndexControllerBuilder {
let meta_env = options.open(&db_path)?; let meta_env = options.open(&db_path)?;
let update_file_store = UpdateFileStore::new(&db_path)?; let update_file_store = UpdateFileStore::new(&db_path)?;
// Create or overwrite the version file for this DB
versioning::create_version_file(db_path.as_ref())?;
let index_resolver = Arc::new(create_index_resolver( let index_resolver = Arc::new(create_index_resolver(
&db_path, &db_path,

View File

@ -0,0 +1,16 @@
#[derive(thiserror::Error, Debug)]
pub enum VersionFileError {
#[error("Version file is missing or the previous MeiliSearch engine version was below 0.24.0. Use a dump to update Meilisearch.")]
MissingVersionFile,
#[error("Version file is corrupted and thus MeiliSearch is unable to determine the version of the database.")]
MalformedVersionFile,
#[error(
"Expected MeiliSearch engine version: {major}.{minor}.{patch}, current engine version: {}. To update Meilisearch use a dump.",
env!("CARGO_PKG_VERSION").to_string()
)]
VersionMismatch {
major: String,
minor: String,
patch: String,
},
}

View File

@ -0,0 +1,56 @@
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
use self::error::VersionFileError;
mod error;
pub const VERSION_FILE_NAME: &str = "VERSION";
static VERSION_MAJOR: &str = env!("CARGO_PKG_VERSION_MAJOR");
static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
// Persists the version of the current MeiliSearch binary to a VERSION file
pub fn create_version_file(db_path: &Path) -> anyhow::Result<()> {
let version_path = db_path.join(VERSION_FILE_NAME);
fs::write(
version_path,
format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH),
)?;
Ok(())
}
// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
let version_path = db_path.join(VERSION_FILE_NAME);
match fs::read_to_string(&version_path) {
Ok(version) => {
let version_components = version.split('.').collect::<Vec<_>>();
let (major, minor, patch) = match &version_components[..] {
[major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
_ => return Err(VersionFileError::MalformedVersionFile.into()),
};
if major != VERSION_MAJOR || minor != VERSION_MINOR {
return Err(VersionFileError::VersionMismatch {
major,
minor,
patch,
}
.into());
}
}
Err(error) => {
return match error.kind() {
ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()),
_ => Err(error.into()),
}
}
}
Ok(())
}

View File

@ -9,6 +9,7 @@ use tokio::time::sleep;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::compression::from_tar_gz; use crate::compression::from_tar_gz;
use crate::index_controller::versioning::VERSION_FILE_NAME;
use crate::tasks::task::Job; use crate::tasks::task::Job;
use crate::tasks::TaskStore; use crate::tasks::TaskStore;
@ -102,6 +103,7 @@ impl SnapshotJob {
let temp_snapshot_dir = tempfile::tempdir()?; let temp_snapshot_dir = tempfile::tempdir()?;
let temp_snapshot_path = temp_snapshot_dir.path(); let temp_snapshot_path = temp_snapshot_dir.path();
self.snapshot_version_file(temp_snapshot_path)?;
self.snapshot_meta_env(temp_snapshot_path)?; self.snapshot_meta_env(temp_snapshot_path)?;
self.snapshot_file_store(temp_snapshot_path)?; self.snapshot_file_store(temp_snapshot_path)?;
self.snapshot_indexes(temp_snapshot_path)?; self.snapshot_indexes(temp_snapshot_path)?;
@ -133,6 +135,15 @@ impl SnapshotJob {
Ok(()) Ok(())
} }
fn snapshot_version_file(&self, path: &Path) -> anyhow::Result<()> {
let dst = path.join(VERSION_FILE_NAME);
let src = self.src_path.join(VERSION_FILE_NAME);
fs::copy(src, dst)?;
Ok(())
}
fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> { fn snapshot_meta_env(&self, path: &Path) -> anyhow::Result<()> {
let mut options = heed::EnvOpenOptions::new(); let mut options = heed::EnvOpenOptions::new();
options.map_size(self.meta_env_size); options.map_size(self.meta_env_size);