make scheduler accept multiple batch handlers

meilisearch-lib/src/dump/loaders/mod.rs

@@ -0,0 +1,3 @@
+pub mod v2;
+pub mod v3;
+pub mod v4;

meilisearch-lib/src/dump/loaders/v1.rs

@@ -0,0 +1,24 @@
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+
+use crate::index_controller::IndexMetadata;
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MetadataV1 {
+    pub db_version: String,
+    indexes: Vec<IndexMetadata>,
+}
+
+impl MetadataV1 {
+    #[allow(dead_code, unreachable_code, unused_variables)]
+    pub fn load_dump(
+        self,
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        size: usize,
+        indexer_options: &IndexerOpts,
+    ) -> anyhow::Result<()> {
+        anyhow::bail!("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")
+}

meilisearch-lib/src/dump/loaders/v2.rs

@@ -0,0 +1,216 @@
+use std::fs::{File, OpenOptions};
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use serde_json::{Deserializer, Value};
+use tempfile::NamedTempFile;
+
+use crate::dump::compat::{self, v2, v3};
+use crate::dump::Metadata;
+use crate::options::IndexerOpts;
+
+/// The dump v2 reads the dump folder and patches all the needed file to make it compatible with a
+/// dump v3, then calls the dump v3 to actually handle the dump.
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    log::info!("Patching dump V2 to dump V3...");
+    let indexes_path = src.as_ref().join("indexes");
+
+    let dir_entries = std::fs::read_dir(indexes_path)?;
+    for entry in dir_entries {
+        let entry = entry?;
+
+        // rename the index folder
+        let path = entry.path();
+        let new_path = patch_index_uuid_path(&path).expect("invalid index folder.");
+
+        std::fs::rename(path, &new_path)?;
+
+        let settings_path = new_path.join("meta.json");
+
+        patch_settings(settings_path)?;
+    }
+
+    let update_dir = src.as_ref().join("updates");
+    let update_path = update_dir.join("data.jsonl");
+    patch_updates(update_dir, update_path)?;
+
+    super::v3::load_dump(
+        meta,
+        src,
+        dst,
+        index_db_size,
+        update_db_size,
+        indexing_options,
+    )
+}
+
+fn patch_index_uuid_path(path: &Path) -> Option<PathBuf> {
+    let uuid = path.file_name()?.to_str()?.trim_start_matches("index-");
+    let new_path = path.parent()?.join(uuid);
+    Some(new_path)
+}
+
+fn patch_settings(path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut meta_file = File::open(&path)?;
+    let mut meta: Value = serde_json::from_reader(&mut meta_file)?;
+
+    // We first deserialize the dump meta into a serde_json::Value and change
+    // the custom ranking rules settings from the old format to the new format.
+    if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") {
+        patch_custom_ranking_rules(ranking_rules);
+    }
+
+    let mut meta_file = OpenOptions::new().truncate(true).write(true).open(path)?;
+
+    serde_json::to_writer(&mut meta_file, &meta)?;
+
+    Ok(())
+}
+
+fn patch_updates(dir: impl AsRef<Path>, path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut output_update_file = NamedTempFile::new_in(&dir)?;
+    let update_file = File::open(&path)?;
+
+    let stream = Deserializer::from_reader(update_file).into_iter::<v2::UpdateEntry>();
+
+    for update in stream {
+        let update_entry = update?;
+
+        let update_entry = v3::UpdateEntry::from(update_entry);
+
+        serde_json::to_writer(&mut output_update_file, &update_entry)?;
+        output_update_file.write_all(b"\n")?;
+    }
+
+    output_update_file.flush()?;
+    output_update_file.persist(path)?;
+
+    Ok(())
+}
+
+/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`.
+///
+/// This is done for compatibility reasons, and to avoid a new dump version,
+/// since the new syntax was introduced soon after the new dump version.
+fn patch_custom_ranking_rules(ranking_rules: &mut Value) {
+    *ranking_rules = match ranking_rules.take() {
+        Value::Array(values) => values
+            .into_iter()
+            .filter_map(|value| match value {
+                Value::String(s) if s.starts_with("asc") => compat::asc_ranking_rule(&s)
+                    .map(|f| format!("{}:asc", f))
+                    .map(Value::String),
+                Value::String(s) if s.starts_with("desc") => compat::desc_ranking_rule(&s)
+                    .map(|f| format!("{}:desc", f))
+                    .map(Value::String),
+                otherwise => Some(otherwise),
+            })
+            .collect(),
+        otherwise => otherwise,
+    }
+}
+
+impl From<v2::UpdateEntry> for v3::UpdateEntry {
+    fn from(v2::UpdateEntry { uuid, update }: v2::UpdateEntry) -> Self {
+        let update = match update {
+            v2::UpdateStatus::Processing(meta) => v3::UpdateStatus::Processing(meta.into()),
+            v2::UpdateStatus::Enqueued(meta) => v3::UpdateStatus::Enqueued(meta.into()),
+            v2::UpdateStatus::Processed(meta) => v3::UpdateStatus::Processed(meta.into()),
+            v2::UpdateStatus::Aborted(_) => unreachable!("Updates could never be aborted."),
+            v2::UpdateStatus::Failed(meta) => v3::UpdateStatus::Failed(meta.into()),
+        };
+
+        Self { uuid, update }
+    }
+}
+
+impl From<v2::Failed> for v3::Failed {
+    fn from(other: v2::Failed) -> Self {
+        let v2::Failed {
+            from,
+            error,
+            failed_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            msg: error.message,
+            code: v2::error_code_from_str(&error.error_code)
+                .expect("Invalid update: Invalid error code"),
+            failed_at,
+        }
+    }
+}
+
+impl From<v2::Processing> for v3::Processing {
+    fn from(other: v2::Processing) -> Self {
+        let v2::Processing {
+            from,
+            started_processing_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            started_processing_at,
+        }
+    }
+}
+
+impl From<v2::Enqueued> for v3::Enqueued {
+    fn from(other: v2::Enqueued) -> Self {
+        let v2::Enqueued {
+            update_id,
+            meta,
+            enqueued_at,
+            content,
+        } = other;
+
+        let meta = match meta {
+            v2::UpdateMeta::DocumentsAddition {
+                method,
+                primary_key,
+                ..
+            } => {
+                v3::Update::DocumentAddition {
+                    primary_key,
+                    method,
+                    // Just ignore if the uuid is no present. If it is needed later, an error will
+                    // be thrown.
+                    content_uuid: content.unwrap_or_default(),
+                }
+            }
+            v2::UpdateMeta::ClearDocuments => v3::Update::ClearDocuments,
+            v2::UpdateMeta::DeleteDocuments { ids } => v3::Update::DeleteDocuments(ids),
+            v2::UpdateMeta::Settings(settings) => v3::Update::Settings(settings),
+        };
+
+        Self {
+            update_id,
+            meta,
+            enqueued_at,
+        }
+    }
+}
+
+impl From<v2::Processed> for v3::Processed {
+    fn from(other: v2::Processed) -> Self {
+        let v2::Processed {
+            from,
+            success,
+            processed_at,
+        } = other;
+
+        Self {
+            success,
+            processed_at,
+            from: from.into(),
+        }
+    }
+}

meilisearch-lib/src/dump/loaders/v3.rs

@@ -0,0 +1,136 @@
+use std::collections::HashMap;
+use std::fs::{self, File};
+use std::io::{BufReader, BufWriter, Write};
+use std::path::Path;
+
+use anyhow::Context;
+use fs_extra::dir::{self, CopyOptions};
+use log::info;
+use tempfile::tempdir;
+use uuid::Uuid;
+
+use crate::dump::compat::v3;
+use crate::dump::Metadata;
+use crate::index_resolver::meta_store::{DumpEntry, IndexMeta};
+use crate::options::IndexerOpts;
+use crate::tasks::task::{Task, TaskId};
+
+/// dump structure for V3:
+/// .
+/// ├── indexes
+/// │   └── 25f10bb8-6ea8-42f0-bd48-ad5857f77648
+/// │       ├── documents.jsonl
+/// │       └── meta.json
+/// ├── index_uuids
+/// │   └── data.jsonl
+/// ├── metadata.json
+/// └── updates
+///     └── data.jsonl
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    meta_env_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!("Patching dump V3 to dump V4...");
+
+    let patched_dir = tempdir()?;
+
+    let options = CopyOptions::default();
+    dir::copy(src.as_ref().join("indexes"), patched_dir.path(), &options)?;
+    dir::copy(
+        src.as_ref().join("index_uuids"),
+        patched_dir.path(),
+        &options,
+    )?;
+
+    let uuid_map = patch_index_meta(
+        src.as_ref().join("index_uuids/data.jsonl"),
+        patched_dir.path(),
+    )?;
+
+    fs::copy(
+        src.as_ref().join("metadata.json"),
+        patched_dir.path().join("metadata.json"),
+    )?;
+
+    patch_updates(&src, patched_dir.path(), uuid_map)?;
+
+    super::v4::load_dump(
+        meta,
+        patched_dir.path(),
+        dst,
+        index_db_size,
+        meta_env_size,
+        indexing_options,
+    )
+}
+
+fn patch_index_meta(
+    path: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+) -> anyhow::Result<HashMap<Uuid, String>> {
+    let file = BufReader::new(File::open(path)?);
+    let dst = dst.as_ref().join("index_uuids");
+    fs::create_dir_all(&dst)?;
+    let mut dst_file = File::create(dst.join("data.jsonl"))?;
+
+    let map = serde_json::Deserializer::from_reader(file)
+        .into_iter::<v3::DumpEntry>()
+        .try_fold(HashMap::new(), |mut map, entry| -> anyhow::Result<_> {
+            let entry = entry?;
+            map.insert(entry.uuid, entry.uid.clone());
+            let meta = IndexMeta {
+                uuid: entry.uuid,
+                // This is lost information, we patch it to 0;
+                creation_task_id: 0,
+            };
+            let entry = DumpEntry {
+                uid: entry.uid,
+                index_meta: meta,
+            };
+            serde_json::to_writer(&mut dst_file, &entry)?;
+            dst_file.write_all(b"\n")?;
+            Ok(map)
+        })?;
+
+    dst_file.flush()?;
+
+    Ok(map)
+}
+
+fn patch_updates(
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    uuid_map: HashMap<Uuid, String>,
+) -> anyhow::Result<()> {
+    let dst = dst.as_ref().join("updates");
+    fs::create_dir_all(&dst)?;
+
+    let mut dst_file = BufWriter::new(File::create(dst.join("data.jsonl"))?);
+    let src_file = BufReader::new(File::open(src.as_ref().join("updates/data.jsonl"))?);
+
+    serde_json::Deserializer::from_reader(src_file)
+        .into_iter::<v3::UpdateEntry>()
+        .enumerate()
+        .try_for_each(|(task_id, entry)| -> anyhow::Result<()> {
+            let entry = entry?;
+            let name = uuid_map
+                .get(&entry.uuid)
+                .with_context(|| format!("Unknown index uuid: {}", entry.uuid))?
+                .clone();
+            serde_json::to_writer(
+                &mut dst_file,
+                &Task::from((entry.update, name, task_id as TaskId)),
+            )?;
+            dst_file.write_all(b"\n")?;
+            Ok(())
+        })?;
+
+    dst_file.flush()?;
+
+    Ok(())
+}

meilisearch-lib/src/dump/loaders/v4.rs

@@ -0,0 +1,48 @@
+use std::path::Path;
+use std::sync::Arc;
+
+use log::info;
+use meilisearch_auth::AuthController;
+use milli::heed::EnvOpenOptions;
+
+use crate::analytics;
+use crate::dump::Metadata;
+use crate::index_resolver::IndexResolver;
+use crate::options::IndexerOpts;
+use crate::tasks::TaskStore;
+use crate::update_file_store::UpdateFileStore;
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    meta_env_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!(
+        "Loading dump from {}, dump database version: {}, dump version: V4",
+        meta.dump_date, meta.db_version
+    );
+
+    let mut options = EnvOpenOptions::new();
+    options.map_size(meta_env_size);
+    options.max_dbs(100);
+    let env = Arc::new(options.open(&dst)?);
+
+    IndexResolver::load_dump(
+        src.as_ref(),
+        &dst,
+        index_db_size,
+        env.clone(),
+        indexing_options,
+    )?;
+    UpdateFileStore::load_dump(src.as_ref(), &dst)?;
+    TaskStore::load_dump(&src, env)?;
+    AuthController::load_dump(&src, &dst)?;
+    analytics::copy_user_id(src.as_ref(), dst.as_ref());
+
+    info!("Loading indexes.");
+
+    Ok(())
+}