add boilerplate for dump v5

This commit is contained in:
ad hoc 2022-05-30 17:19:29 +02:00
parent 3441cc6c36
commit 26e7bdf702
No known key found for this signature in database
GPG Key ID: 4F00A782990CC643
7 changed files with 67 additions and 35 deletions

View File

@ -1,5 +1,6 @@
pub mod v2; pub mod v2;
pub mod v3; pub mod v3;
pub mod v4;
/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
pub fn asc_ranking_rule(text: &str) -> Option<&str> { pub fn asc_ranking_rule(text: &str) -> Option<&str> {

View File

@ -0,0 +1 @@

View File

@ -1,3 +1,4 @@
pub mod v2; pub mod v2;
pub mod v3; pub mod v3;
pub mod v4; pub mod v4;
pub mod v5;

View File

@ -66,7 +66,6 @@ pub fn load_dump(
index_db_size, index_db_size,
meta_env_size, meta_env_size,
indexing_options, indexing_options,
"V5",
) )
} }

View File

@ -1,16 +1,9 @@
use std::path::Path; use std::path::Path;
use std::sync::Arc;
use log::info; use log::info;
use meilisearch_auth::AuthController;
use milli::heed::EnvOpenOptions;
use crate::analytics;
use crate::dump::Metadata; use crate::dump::Metadata;
use crate::index_resolver::IndexResolver;
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
use crate::tasks::TaskStore;
use crate::update_file_store::UpdateFileStore;
pub fn load_dump( pub fn load_dump(
meta: Metadata, meta: Metadata,
@ -19,31 +12,15 @@ pub fn load_dump(
index_db_size: usize, index_db_size: usize,
meta_env_size: usize, meta_env_size: usize,
indexing_options: &IndexerOpts, indexing_options: &IndexerOpts,
version: &str,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
info!( info!("Patching dump V4 to dump V5...");
"Loading dump from {}, dump database version: {}, dump version: {}",
meta.dump_date, meta.db_version, version
);
let mut options = EnvOpenOptions::new(); super::v5::load_dump(
options.map_size(meta_env_size); meta,
options.max_dbs(100); src,
let env = Arc::new(options.open(&dst)?); dst,
IndexResolver::load_dump(
src.as_ref(),
&dst,
index_db_size, index_db_size,
env.clone(), meta_env_size,
indexing_options, indexing_options,
)?; )
UpdateFileStore::load_dump(src.as_ref(), &dst)?;
TaskStore::load_dump(&src, env)?;
AuthController::load_dump(&src, &dst)?;
analytics::copy_user_id(src.as_ref(), dst.as_ref());
info!("Loading indexes.");
Ok(())
} }

View File

@ -0,0 +1,47 @@
use std::{path::Path, sync::Arc};
use log::info;
use meilisearch_auth::AuthController;
use milli::heed::EnvOpenOptions;
use crate::analytics;
use crate::dump::Metadata;
use crate::index_resolver::IndexResolver;
use crate::options::IndexerOpts;
use crate::tasks::TaskStore;
use crate::update_file_store::UpdateFileStore;
pub fn load_dump(
meta: Metadata,
src: impl AsRef<Path>,
dst: impl AsRef<Path>,
index_db_size: usize,
meta_env_size: usize,
indexing_options: &IndexerOpts,
) -> anyhow::Result<()> {
info!(
"Loading dump from {}, dump database version: {}, dump version: V5",
meta.dump_date, meta.db_version
);
let mut options = EnvOpenOptions::new();
options.map_size(meta_env_size);
options.max_dbs(100);
let env = Arc::new(options.open(&dst)?);
IndexResolver::load_dump(
src.as_ref(),
&dst,
index_db_size,
env.clone(),
indexing_options,
)?;
UpdateFileStore::load_dump(src.as_ref(), &dst)?;
TaskStore::load_dump(&src, env)?;
AuthController::load_dump(&src, &dst)?;
analytics::copy_user_id(src.as_ref(), dst.as_ref());
info!("Loading indexes.");
Ok(())
}

View File

@ -11,7 +11,7 @@ use tempfile::TempDir;
use crate::compression::from_tar_gz; use crate::compression::from_tar_gz;
use crate::options::IndexerOpts; use crate::options::IndexerOpts;
use self::loaders::{v2, v3, v4}; use self::loaders::{v2, v3, v4, v5};
pub use handler::{generate_uid, DumpHandler}; pub use handler::{generate_uid, DumpHandler};
@ -69,7 +69,6 @@ impl MetadataVersion {
meta_env_size: usize, meta_env_size: usize,
indexing_options: &IndexerOpts, indexing_options: &IndexerOpts,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let version = self.version();
match self { match self {
MetadataVersion::V1(_meta) => { MetadataVersion::V1(_meta) => {
anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.") anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
@ -90,14 +89,21 @@ impl MetadataVersion {
meta_env_size, meta_env_size,
indexing_options, indexing_options,
)?, )?,
MetadataVersion::V4(meta) | MetadataVersion::V5(meta) => v4::load_dump( MetadataVersion::V4(meta) => v4::load_dump(
meta,
src,
dst,
index_db_size,
meta_env_size,
indexing_options,
)?,
MetadataVersion::V5(meta) => v5::load_dump(
meta, meta,
src, src,
dst, dst,
index_db_size, index_db_size,
meta_env_size, meta_env_size,
indexing_options, indexing_options,
version,
)?, )?,
} }