2444: add boilerplate for dump v5 r=MarinPostma a=MarinPostma

add the boilerplate files for dump v5


Co-authored-by: ad hoc <postma.marin@protonmail.com>
Co-authored-by: ManyTheFish <many@meilisearch.com>
This commit is contained in:
bors[bot] 2022-05-31 08:56:52 +00:00 committed by GitHub
commit 5450fecaef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 95 additions and 33 deletions

View File

@ -1,5 +1,6 @@
pub mod v2;
pub mod v3;
pub mod v4;
/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
pub fn asc_ranking_rule(text: &str) -> Option<&str> {

View File

@ -0,0 +1 @@

View File

@ -1,3 +1,4 @@
pub mod v2;
pub mod v3;
pub mod v4;
pub mod v5;

View File

@ -66,7 +66,6 @@ pub fn load_dump(
index_db_size,
meta_env_size,
indexing_options,
"V5",
)
}

View File

@ -1,16 +1,12 @@
use std::fs;
use std::path::Path;
use std::sync::Arc;
use fs_extra::dir::{self, CopyOptions};
use log::info;
use meilisearch_auth::AuthController;
use milli::heed::EnvOpenOptions;
use tempfile::tempdir;
use crate::analytics;
use crate::dump::Metadata;
use crate::index_resolver::IndexResolver;
use crate::options::IndexerOpts;
use crate::tasks::TaskStore;
use crate::update_file_store::UpdateFileStore;
pub fn load_dump(
meta: Metadata,
@ -19,31 +15,42 @@ pub fn load_dump(
index_db_size: usize,
meta_env_size: usize,
indexing_options: &IndexerOpts,
version: &str,
) -> anyhow::Result<()> {
info!(
"Loading dump from {}, dump database version: {}, dump version: {}",
meta.dump_date, meta.db_version, version
);
info!("Patching dump V4 to dump V5...");
let mut options = EnvOpenOptions::new();
options.map_size(meta_env_size);
options.max_dbs(100);
let env = Arc::new(options.open(&dst)?);
let patched_dir = tempdir()?;
let options = CopyOptions::default();
IndexResolver::load_dump(
src.as_ref(),
&dst,
index_db_size,
env.clone(),
indexing_options,
// Indexes
dir::copy(src.as_ref().join("indexes"), patched_dir.path(), &options)?;
// Index uuids
dir::copy(
src.as_ref().join("index_uuids"),
patched_dir.path(),
&options,
)?;
UpdateFileStore::load_dump(src.as_ref(), &dst)?;
TaskStore::load_dump(&src, env)?;
AuthController::load_dump(&src, &dst)?;
analytics::copy_user_id(src.as_ref(), dst.as_ref());
info!("Loading indexes.");
// Metadata
fs::copy(
src.as_ref().join("metadata.json"),
patched_dir.path().join("metadata.json"),
)?;
Ok(())
// Updates
dir::copy(src.as_ref().join("updates"), patched_dir.path(), &options)?;
// Keys
if src.as_ref().join("keys").exists() {
fs::copy(src.as_ref().join("keys"), patched_dir.path().join("keys"))?;
}
super::v5::load_dump(
meta,
patched_dir.path(),
dst,
index_db_size,
meta_env_size,
indexing_options,
)
}

View File

@ -0,0 +1,47 @@
use std::{path::Path, sync::Arc};
use log::info;
use meilisearch_auth::AuthController;
use milli::heed::EnvOpenOptions;
use crate::analytics;
use crate::dump::Metadata;
use crate::index_resolver::IndexResolver;
use crate::options::IndexerOpts;
use crate::tasks::TaskStore;
use crate::update_file_store::UpdateFileStore;
pub fn load_dump(
meta: Metadata,
src: impl AsRef<Path>,
dst: impl AsRef<Path>,
index_db_size: usize,
meta_env_size: usize,
indexing_options: &IndexerOpts,
) -> anyhow::Result<()> {
info!(
"Loading dump from {}, dump database version: {}, dump version: V5",
meta.dump_date, meta.db_version
);
let mut options = EnvOpenOptions::new();
options.map_size(meta_env_size);
options.max_dbs(100);
let env = Arc::new(options.open(&dst)?);
IndexResolver::load_dump(
src.as_ref(),
&dst,
index_db_size,
env.clone(),
indexing_options,
)?;
UpdateFileStore::load_dump(src.as_ref(), &dst)?;
TaskStore::load_dump(&src, env)?;
AuthController::load_dump(&src, &dst)?;
analytics::copy_user_id(src.as_ref(), dst.as_ref());
info!("Loading indexes.");
Ok(())
}

View File

@ -11,7 +11,7 @@ use tempfile::TempDir;
use crate::compression::from_tar_gz;
use crate::options::IndexerOpts;
use self::loaders::{v2, v3, v4};
use self::loaders::{v2, v3, v4, v5};
pub use handler::{generate_uid, DumpHandler};
@ -69,7 +69,6 @@ impl MetadataVersion {
meta_env_size: usize,
indexing_options: &IndexerOpts,
) -> anyhow::Result<()> {
let version = self.version();
match self {
MetadataVersion::V1(_meta) => {
anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
@ -90,14 +89,21 @@ impl MetadataVersion {
meta_env_size,
indexing_options,
)?,
MetadataVersion::V4(meta) | MetadataVersion::V5(meta) => v4::load_dump(
MetadataVersion::V4(meta) => v4::load_dump(
meta,
src,
dst,
index_db_size,
meta_env_size,
indexing_options,
)?,
MetadataVersion::V5(meta) => v5::load_dump(
meta,
src,
dst,
index_db_size,
meta_env_size,
indexing_options,
version,
)?,
}