From 26e7bdf702fa8184b482495580abc4f5cf3c0176 Mon Sep 17 00:00:00 2001 From: ad hoc Date: Mon, 30 May 2022 17:19:29 +0200 Subject: [PATCH] add boilerplate for dump v5 --- meilisearch-lib/src/dump/compat/mod.rs | 1 + meilisearch-lib/src/dump/compat/v4.rs | 1 + meilisearch-lib/src/dump/loaders/mod.rs | 1 + meilisearch-lib/src/dump/loaders/v3.rs | 1 - meilisearch-lib/src/dump/loaders/v4.rs | 37 ++++--------------- meilisearch-lib/src/dump/loaders/v5.rs | 47 +++++++++++++++++++++++++ meilisearch-lib/src/dump/mod.rs | 14 +++++--- 7 files changed, 67 insertions(+), 35 deletions(-) create mode 100644 meilisearch-lib/src/dump/compat/v4.rs create mode 100644 meilisearch-lib/src/dump/loaders/v5.rs diff --git a/meilisearch-lib/src/dump/compat/mod.rs b/meilisearch-lib/src/dump/compat/mod.rs index 93f3f9dd7..9abac24c7 100644 --- a/meilisearch-lib/src/dump/compat/mod.rs +++ b/meilisearch-lib/src/dump/compat/mod.rs @@ -1,5 +1,6 @@ pub mod v2; pub mod v3; +pub mod v4; /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. pub fn asc_ranking_rule(text: &str) -> Option<&str> { diff --git a/meilisearch-lib/src/dump/compat/v4.rs b/meilisearch-lib/src/dump/compat/v4.rs new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/meilisearch-lib/src/dump/compat/v4.rs @@ -0,0 +1 @@ + diff --git a/meilisearch-lib/src/dump/loaders/mod.rs b/meilisearch-lib/src/dump/loaders/mod.rs index ecc305652..199b20c02 100644 --- a/meilisearch-lib/src/dump/loaders/mod.rs +++ b/meilisearch-lib/src/dump/loaders/mod.rs @@ -1,3 +1,4 @@ pub mod v2; pub mod v3; pub mod v4; +pub mod v5; diff --git a/meilisearch-lib/src/dump/loaders/v3.rs b/meilisearch-lib/src/dump/loaders/v3.rs index 8e76b67e0..0a2ea438b 100644 --- a/meilisearch-lib/src/dump/loaders/v3.rs +++ b/meilisearch-lib/src/dump/loaders/v3.rs @@ -66,7 +66,6 @@ pub fn load_dump( index_db_size, meta_env_size, indexing_options, - "V5", ) } diff --git a/meilisearch-lib/src/dump/loaders/v4.rs b/meilisearch-lib/src/dump/loaders/v4.rs index 7f0ade714..df7b33cfa 100644 --- a/meilisearch-lib/src/dump/loaders/v4.rs +++ b/meilisearch-lib/src/dump/loaders/v4.rs @@ -1,16 +1,9 @@ use std::path::Path; -use std::sync::Arc; use log::info; -use meilisearch_auth::AuthController; -use milli::heed::EnvOpenOptions; -use crate::analytics; use crate::dump::Metadata; -use crate::index_resolver::IndexResolver; use crate::options::IndexerOpts; -use crate::tasks::TaskStore; -use crate::update_file_store::UpdateFileStore; pub fn load_dump( meta: Metadata, @@ -19,31 +12,15 @@ pub fn load_dump( index_db_size: usize, meta_env_size: usize, indexing_options: &IndexerOpts, - version: &str, ) -> anyhow::Result<()> { - info!( - "Loading dump from {}, dump database version: {}, dump version: {}", - meta.dump_date, meta.db_version, version - ); + info!("Patching dump V4 to dump V5..."); - let mut options = EnvOpenOptions::new(); - options.map_size(meta_env_size); - options.max_dbs(100); - let env = Arc::new(options.open(&dst)?); - - IndexResolver::load_dump( - src.as_ref(), - &dst, + super::v5::load_dump( + meta, + src, + dst, index_db_size, - env.clone(), + meta_env_size, indexing_options, - )?; - UpdateFileStore::load_dump(src.as_ref(), &dst)?; - TaskStore::load_dump(&src, env)?; - AuthController::load_dump(&src, &dst)?; - analytics::copy_user_id(src.as_ref(), dst.as_ref()); - - info!("Loading indexes."); - - Ok(()) + ) } diff --git a/meilisearch-lib/src/dump/loaders/v5.rs b/meilisearch-lib/src/dump/loaders/v5.rs new file mode 100644 index 000000000..fcb4224bb --- /dev/null +++ b/meilisearch-lib/src/dump/loaders/v5.rs @@ -0,0 +1,47 @@ +use std::{path::Path, sync::Arc}; + +use log::info; +use meilisearch_auth::AuthController; +use milli::heed::EnvOpenOptions; + +use crate::analytics; +use crate::dump::Metadata; +use crate::index_resolver::IndexResolver; +use crate::options::IndexerOpts; +use crate::tasks::TaskStore; +use crate::update_file_store::UpdateFileStore; + +pub fn load_dump( + meta: Metadata, + src: impl AsRef, + dst: impl AsRef, + index_db_size: usize, + meta_env_size: usize, + indexing_options: &IndexerOpts, +) -> anyhow::Result<()> { + info!( + "Loading dump from {}, dump database version: {}, dump version: V5", + meta.dump_date, meta.db_version + ); + + let mut options = EnvOpenOptions::new(); + options.map_size(meta_env_size); + options.max_dbs(100); + let env = Arc::new(options.open(&dst)?); + + IndexResolver::load_dump( + src.as_ref(), + &dst, + index_db_size, + env.clone(), + indexing_options, + )?; + UpdateFileStore::load_dump(src.as_ref(), &dst)?; + TaskStore::load_dump(&src, env)?; + AuthController::load_dump(&src, &dst)?; + analytics::copy_user_id(src.as_ref(), dst.as_ref()); + + info!("Loading indexes."); + + Ok(()) +} diff --git a/meilisearch-lib/src/dump/mod.rs b/meilisearch-lib/src/dump/mod.rs index ab1c63d6d..ea7b9f3dc 100644 --- a/meilisearch-lib/src/dump/mod.rs +++ b/meilisearch-lib/src/dump/mod.rs @@ -11,7 +11,7 @@ use tempfile::TempDir; use crate::compression::from_tar_gz; use crate::options::IndexerOpts; -use self::loaders::{v2, v3, v4}; +use self::loaders::{v2, v3, v4, v5}; pub use handler::{generate_uid, DumpHandler}; @@ -69,7 +69,6 @@ impl MetadataVersion { meta_env_size: usize, indexing_options: &IndexerOpts, ) -> anyhow::Result<()> { - let version = self.version(); match self { MetadataVersion::V1(_meta) => { anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.") @@ -90,14 +89,21 @@ impl MetadataVersion { meta_env_size, indexing_options, )?, - MetadataVersion::V4(meta) | MetadataVersion::V5(meta) => v4::load_dump( + MetadataVersion::V4(meta) => v4::load_dump( + meta, + src, + dst, + index_db_size, + meta_env_size, + indexing_options, + )?, + MetadataVersion::V5(meta) => v5::load_dump( meta, src, dst, index_db_size, meta_env_size, indexing_options, - version, )?, }