From c752c14c462724d516c476f4cf0e0ba428cf1b9b Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 26 Oct 2021 13:02:40 +0200 Subject: [PATCH] refactorize the dump and snapshot --- meilisearch-lib/src/analytics.rs | 13 ++---------- .../src/index_controller/dump_actor/actor.rs | 4 ++-- .../index_controller/dump_actor/loaders/v3.rs | 2 +- .../src/index_controller/dump_actor/mod.rs | 20 +++++++++---------- meilisearch-lib/src/index_controller/mod.rs | 1 + .../src/index_controller/snapshot.rs | 12 +++++++++++ 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/meilisearch-lib/src/analytics.rs b/meilisearch-lib/src/analytics.rs index 76e673e5a..9dd8d3219 100644 --- a/meilisearch-lib/src/analytics.rs +++ b/meilisearch-lib/src/analytics.rs @@ -1,17 +1,8 @@ use std::{fs, path::Path}; -/// To load a dump we get the user id from the source directory; -/// If there was a user-id, write it to the new destination if not ignore the error -pub fn load_dump(src: &Path, dst: &Path) { +/// Copy the `user-id` contained in one db to another. Ignore all errors. +pub fn copy_user_id(src: &Path, dst: &Path) { if let Ok(user_id) = fs::read_to_string(src.join("user-id")) { let _ = fs::write(dst.join("user-id"), &user_id); } } - -/// To load a dump we get the user id either from the source directory; -/// If there was a user-id, write it to the new destination if not ignore the error -pub fn write_dump(src: &Path, dst: &Path) { - if let Ok(user_id) = fs::read_to_string(src) { - let _ = fs::write(dst, &user_id); - } -} diff --git a/meilisearch-lib/src/index_controller/dump_actor/actor.rs b/meilisearch-lib/src/index_controller/dump_actor/actor.rs index 896f86e3b..03c139c1d 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/actor.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/actor.rs @@ -121,8 +121,8 @@ where ret.send(Ok(info)).expect("Dump actor is dead"); let task = DumpTask { - path: self.dump_path.clone(), - analytics_path: self.analytics_path.clone(), + dump_path: self.dump_path.clone(), + db_path: self.analytics_path.clone(), index_resolver: self.index_resolver.clone(), update_sender: self.update.clone(), uid: uid.clone(), diff --git a/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs b/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs index 8a67edc67..1eea55451 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs @@ -25,7 +25,7 @@ pub fn load_dump( IndexResolver::load_dump(src.as_ref(), &dst, index_db_size, indexing_options)?; UpdateFileStore::load_dump(src.as_ref(), &dst)?; UpdateStore::load_dump(&src, &dst, update_db_size)?; - analytics::load_dump(src.as_ref(), dst.as_ref()); + analytics::copy_user_id(src.as_ref(), dst.as_ref()); info!("Loading indexes."); diff --git a/meilisearch-lib/src/index_controller/dump_actor/mod.rs b/meilisearch-lib/src/index_controller/dump_actor/mod.rs index c73d5bab7..844dbf768 100644 --- a/meilisearch-lib/src/index_controller/dump_actor/mod.rs +++ b/meilisearch-lib/src/index_controller/dump_actor/mod.rs @@ -223,8 +223,8 @@ pub fn load_dump( } struct DumpTask { - path: PathBuf, - analytics_path: PathBuf, + dump_path: PathBuf, + db_path: PathBuf, index_resolver: Arc>, update_sender: UpdateSender, uid: String, @@ -240,7 +240,7 @@ where async fn run(self) -> Result<()> { trace!("Performing dump."); - create_dir_all(&self.path).await?; + create_dir_all(&self.dump_path).await?; let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??; let temp_dump_path = temp_dump_dir.path().to_owned(); @@ -249,7 +249,7 @@ where let meta_path = temp_dump_path.join(META_FILE_NAME); let mut meta_file = File::create(&meta_path)?; serde_json::to_writer(&mut meta_file, &meta)?; - analytics::write_dump(&self.analytics_path, &temp_dump_path.join("user-id")); + analytics::copy_user_id(&self.db_path, &temp_dump_path); create_dir_all(&temp_dump_path.join("indexes")).await?; let uuids = self.index_resolver.dump(temp_dump_path.clone()).await?; @@ -257,11 +257,11 @@ where UpdateMsg::dump(&self.update_sender, uuids, temp_dump_path.clone()).await?; let dump_path = tokio::task::spawn_blocking(move || -> Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?; + let temp_dump_file = tempfile::NamedTempFile::new_in(&self.dump_path)?; to_tar_gz(temp_dump_path, temp_dump_file.path()) .map_err(|e| DumpActorError::Internal(e.into()))?; - let dump_path = self.path.join(self.uid).with_extension("dump"); + let dump_path = self.dump_path.join(self.uid).with_extension("dump"); temp_dump_file.persist(&dump_path)?; Ok(dump_path) @@ -341,9 +341,9 @@ mod test { create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap(); let task = DumpTask { - path: tmp.path().to_owned(), + dump_path: tmp.path().into(), // this should do nothing - analytics_path: tmp.path().join("user-id"), + db_path: tmp.path().into(), index_resolver, update_sender, uid: String::from("test"), @@ -371,9 +371,9 @@ mod test { create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap(); let task = DumpTask { - path: tmp.path().to_owned(), + dump_path: tmp.path().into(), // this should do nothing - analytics_path: tmp.path().join("user-id"), + db_path: tmp.path().into(), index_resolver, update_sender, uid: String::from("test"), diff --git a/meilisearch-lib/src/index_controller/mod.rs b/meilisearch-lib/src/index_controller/mod.rs index 4074de1ba..631bab9e9 100644 --- a/meilisearch-lib/src/index_controller/mod.rs +++ b/meilisearch-lib/src/index_controller/mod.rs @@ -189,6 +189,7 @@ impl IndexControllerBuilder { .ok_or_else(|| anyhow::anyhow!("Snapshot interval not provided."))?, self.snapshot_dir .ok_or_else(|| anyhow::anyhow!("Snapshot path not provided."))?, + db_path.as_ref().into(), db_path .as_ref() .file_name() diff --git a/meilisearch-lib/src/index_controller/snapshot.rs b/meilisearch-lib/src/index_controller/snapshot.rs index 07bf75199..e3fb7e66e 100644 --- a/meilisearch-lib/src/index_controller/snapshot.rs +++ b/meilisearch-lib/src/index_controller/snapshot.rs @@ -8,6 +8,7 @@ use tokio::fs; use tokio::task::spawn_blocking; use tokio::time::sleep; +use crate::analytics; use crate::compression::from_tar_gz; use crate::index_controller::updates::UpdateMsg; @@ -21,6 +22,7 @@ pub struct SnapshotService { update_sender: UpdateSender, snapshot_period: Duration, snapshot_path: PathBuf, + db_path: PathBuf, db_name: String, } @@ -34,6 +36,7 @@ where update_sender: UpdateSender, snapshot_period: Duration, snapshot_path: PathBuf, + db_path: PathBuf, db_name: String, ) -> Self { Self { @@ -41,6 +44,7 @@ where update_sender, snapshot_period, snapshot_path, + db_path, db_name, } } @@ -71,6 +75,8 @@ where .snapshot(temp_snapshot_path.clone()) .await?; + analytics::copy_user_id(&self.db_path, &temp_snapshot_path.clone()); + if indexes.is_empty() { return Ok(()); } @@ -211,6 +217,8 @@ mod test { update_sender, Duration::from_millis(100), snapshot_path.path().to_owned(), + // this should do nothing + snapshot_path.path().to_owned(), "data.ms".to_string(), ); @@ -243,6 +251,8 @@ mod test { update_sender, Duration::from_millis(100), snapshot_path.path().to_owned(), + // this should do nothing + snapshot_path.path().to_owned(), "data.ms".to_string(), ); @@ -292,6 +302,8 @@ mod test { update_sender, Duration::from_millis(100), snapshot_path.path().to_owned(), + // this should do nothing + snapshot_path.path().to_owned(), "data.ms".to_string(), );