diff --git a/meilisearch-http/src/data/mod.rs b/meilisearch-http/src/data/mod.rs index c7979210e..39cfed626 100644 --- a/meilisearch-http/src/data/mod.rs +++ b/meilisearch-http/src/data/mod.rs @@ -108,6 +108,10 @@ impl Data { Ok(self.index_controller.get_all_stats().await?) } + pub async fn dump(&self) -> anyhow::Result { + Ok(self.index_controller.dump(self.options.dumps_dir.clone()).await?) + } + #[inline] pub fn http_payload_size_limit(&self) -> usize { self.options.http_payload_size_limit.get_bytes() as usize diff --git a/meilisearch-http/src/index_controller/dump/mod.rs b/meilisearch-http/src/index_controller/dump/mod.rs index 2bd5f167e..a44d4235b 100644 --- a/meilisearch-http/src/index_controller/dump/mod.rs +++ b/meilisearch-http/src/index_controller/dump/mod.rs @@ -1,16 +1,20 @@ mod v1; mod v2; -use std::{fs::File, path::{Path}, sync::Arc}; +use std::{collections::HashSet, fs::{File}, path::{Path, PathBuf}, sync::Arc}; use anyhow::bail; +use chrono::Utc; use heed::EnvOpenOptions; use log::{error, info}; use milli::update::{IndexDocumentsMethod, UpdateBuilder, UpdateFormat}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; +use tokio::task::spawn_blocking; +use tokio::fs; +use uuid::Uuid; -use super::IndexMetadata; +use super::{IndexController, IndexMetadata, update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle}; use crate::index::Index; use crate::index_controller::uuid_resolver; use crate::helpers::compression; @@ -22,7 +26,7 @@ enum DumpVersion { } impl DumpVersion { - // const CURRENT: Self = Self::V2; + const CURRENT: Self = Self::V2; /// Select the good importation function from the `DumpVersion` of metadata pub fn import_index(self, size: usize, dump_path: &Path, index_path: &Path) -> anyhow::Result<()> { @@ -42,7 +46,6 @@ pub struct Metadata { } impl Metadata { - /* /// Create a Metadata with the current dump version of meilisearch. pub fn new(indexes: Vec, db_version: String) -> Self { Metadata { @@ -51,7 +54,6 @@ impl Metadata { dump_version: DumpVersion::CURRENT, } } - */ /// Extract Metadata from `metadata.json` file present at provided `dir_path` fn from_path(dir_path: &Path) -> anyhow::Result { @@ -63,105 +65,73 @@ impl Metadata { Ok(metadata) } - /* /// Write Metadata in `metadata.json` file at provided `dir_path` - fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { + pub async fn to_path(&self, dir_path: &Path) -> anyhow::Result<()> { let path = dir_path.join("metadata.json"); - let file = File::create(path)?; - - serde_json::to_writer(file, &self)?; - - Ok(()) - } - */ -} - -/* -pub struct DumpService { - uuid_resolver_handle: R, - update_handle: U, - dump_path: PathBuf, - db_name: String, -} - -impl DumpService -where - U: UpdateActorHandle, - R: UuidResolverHandle, -{ - pub fn new( - uuid_resolver_handle: R, - update_handle: U, - dump_path: PathBuf, - db_name: String, - ) -> Self { - Self { - uuid_resolver_handle, - update_handle, - dump_path, - db_name, - } - } - - pub async fn run(self) { - if let Err(e) = self.perform_dump().await { - error!("{}", e); - } - } - - async fn perform_dump(&self) -> anyhow::Result<()> { - /* - info!("Performing dump."); - - let dump_dir = self.dump_path.clone(); - fs::create_dir_all(&dump_dir).await?; - let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; - let temp_dump_path = temp_dump_dir.path().to_owned(); - - let uuids = self - .uuid_resolver_handle - .dump(temp_dump_path.clone()) - .await?; - - if uuids.is_empty() { - return Ok(()); - } - - let tasks = uuids - .iter() - .map(|&uuid| self.update_handle.dump(uuid, temp_dump_path.clone())) - .collect::>(); - - futures::future::try_join_all(tasks).await?; - - let dump_dir = self.dump_path.clone(); - let dump_path = self.dump_path.join(format!("{}.dump", self.db_name)); - let dump_path = spawn_blocking(move || -> anyhow::Result { - let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; - let temp_dump_file_path = temp_dump_file.path().to_owned(); - compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; - temp_dump_file.persist(&dump_path)?; - Ok(dump_path) - }) - .await??; - - info!("Created dump in {:?}.", dump_path); - */ + tokio::fs::write(path, serde_json::to_string(self)?).await?; Ok(()) } } -*/ + +/// Generate uid from creation date +fn generate_uid() -> String { + Utc::now().format("%Y%m%d-%H%M%S%3f").to_string() +} + +pub async fn perform_dump(index_controller: &IndexController, dump_path: PathBuf) -> anyhow::Result { + info!("Performing dump."); + + let dump_dir = dump_path.clone(); + let uid = generate_uid(); + fs::create_dir_all(&dump_dir).await?; + let temp_dump_dir = spawn_blocking(move || tempfile::tempdir_in(dump_dir)).await??; + let temp_dump_path = temp_dump_dir.path().to_owned(); + + let uuids = index_controller.uuid_resolver.list().await?; + // maybe we could just keep the vec as-is + let uuids: HashSet<(String, Uuid)> = uuids.into_iter().collect(); + + if uuids.is_empty() { + return Ok(uid); + } + + let indexes = index_controller.list_indexes().await?; + + // we create one directory by index + for meta in indexes.iter() { + tokio::fs::create_dir(temp_dump_path.join(&meta.uid)).await?; + } + + let metadata = Metadata::new(indexes, env!("CARGO_PKG_VERSION").to_string()); + metadata.to_path(&temp_dump_path).await?; + + index_controller.update_handle.dump(uuids, temp_dump_path.clone()).await?; + let dump_dir = dump_path.clone(); + let dump_path = dump_path.join(format!("{}.dump", uid)); + let dump_path = spawn_blocking(move || -> anyhow::Result { + let temp_dump_file = tempfile::NamedTempFile::new_in(dump_dir)?; + let temp_dump_file_path = temp_dump_file.path().to_owned(); + compression::to_tar_gz(temp_dump_path, temp_dump_file_path)?; + temp_dump_file.persist(&dump_path)?; + Ok(dump_path) + }) + .await??; + + info!("Created dump in {:?}.", dump_path); + + Ok(uid) +} /* /// Write Settings in `settings.json` file at provided `dir_path` fn settings_to_path(settings: &Settings, dir_path: &Path) -> anyhow::Result<()> { - let path = dir_path.join("settings.json"); - let file = File::create(path)?; +let path = dir_path.join("settings.json"); +let file = File::create(path)?; - serde_json::to_writer(file, settings)?; +serde_json::to_writer(file, settings)?; - Ok(()) +Ok(()) } */ diff --git a/meilisearch-http/src/index_controller/index_actor/actor.rs b/meilisearch-http/src/index_controller/index_actor/actor.rs index ecf71b62b..623b42ddc 100644 --- a/meilisearch-http/src/index_controller/index_actor/actor.rs +++ b/meilisearch-http/src/index_controller/index_actor/actor.rs @@ -122,8 +122,8 @@ impl IndexActor { Snapshot { uuid, path, ret } => { let _ = ret.send(self.handle_snapshot(uuid, path).await); } - Dump { uuid, path, ret } => { - let _ = ret.send(self.handle_dump(uuid, path).await); + Dump { uid, uuid, path, ret } => { + let _ = ret.send(self.handle_dump(&uid, uuid, path).await); } GetStats { uuid, ret } => { let _ = ret.send(self.handle_get_stats(uuid).await); @@ -312,24 +312,52 @@ impl IndexActor { Ok(()) } - async fn handle_dump(&self, uuid: Uuid, mut path: PathBuf) -> IndexResult<()> { + /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the + /// documents and all the settings. + async fn handle_dump(&self, uid: &str, uuid: Uuid, path: PathBuf) -> IndexResult<()> { use tokio::fs::create_dir_all; + use std::io::prelude::*; - path.push("indexes"); create_dir_all(&path) .await .map_err(|e| IndexError::Error(e.into()))?; if let Some(index) = self.store.get(uuid).await? { - let mut index_path = path.join(format!("index-{}", uuid)); - create_dir_all(&index_path) - .await - .map_err(|e| IndexError::Error(e.into()))?; - index_path.push("data.mdb"); + let documents_path = path.join(uid).join("documents.jsonl"); + let settings_path = path.join(uid).join("settings.json"); + spawn_blocking(move || -> anyhow::Result<()> { + // first we dump all the documents + let file = File::create(documents_path)?; + let mut file = std::io::BufWriter::new(file); + // Get write txn to wait for ongoing write transaction before dump. - let _txn = index.write_txn()?; - index.env.copy_to_path(index_path, CompactionOption::Enabled)?; + let txn = index.write_txn()?; + let documents_ids = index.documents_ids(&txn)?; + // TODO: TAMO: calling this function here can consume **a lot** of RAM, we should + // use some kind of iterators -> waiting for a milli release + let documents = index.documents(&txn, documents_ids)?; + + let fields_ids_map = index.fields_ids_map(&txn)?; + // we want to save **all** the fields in the dump. + let fields_to_dump: Vec = fields_ids_map.iter().map(|(id, _)| id).collect(); + + for (_doc_id, document) in documents { + let json = milli::obkv_to_json(&fields_to_dump, &fields_ids_map, document)?; + file.write_all(serde_json::to_string(&json)?.as_bytes())?; + file.write_all(b"\n")?; + } + + + // then we dump all the settings + let file = File::create(settings_path)?; + let mut file = std::io::BufWriter::new(file); + let settings = index.settings()?; + + file.write_all(serde_json::to_string(&settings)?.as_bytes())?; + file.write_all(b"\n")?; + + Ok(()) }) .await diff --git a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs index 26aa189d0..64b63e5f0 100644 --- a/meilisearch-http/src/index_controller/index_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/index_actor/handle_impl.rs @@ -136,9 +136,9 @@ impl IndexActorHandle for IndexActorHandleImpl { Ok(receiver.await.expect("IndexActor has been killed")?) } - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()> { + async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()> { let (ret, receiver) = oneshot::channel(); - let msg = IndexMsg::Dump { uuid, path, ret }; + let msg = IndexMsg::Dump { uid, uuid, path, ret }; let _ = self.sender.send(msg).await; Ok(receiver.await.expect("IndexActor has been killed")?) } diff --git a/meilisearch-http/src/index_controller/index_actor/message.rs b/meilisearch-http/src/index_controller/index_actor/message.rs index 714a30ecc..37faa1e31 100644 --- a/meilisearch-http/src/index_controller/index_actor/message.rs +++ b/meilisearch-http/src/index_controller/index_actor/message.rs @@ -61,6 +61,7 @@ pub enum IndexMsg { ret: oneshot::Sender>, }, Dump { + uid: String, uuid: Uuid, path: PathBuf, ret: oneshot::Sender>, diff --git a/meilisearch-http/src/index_controller/index_actor/mod.rs b/meilisearch-http/src/index_controller/index_actor/mod.rs index e06658ff8..0145a33d9 100644 --- a/meilisearch-http/src/index_controller/index_actor/mod.rs +++ b/meilisearch-http/src/index_controller/index_actor/mod.rs @@ -97,7 +97,7 @@ pub trait IndexActorHandle { index_settings: IndexSettings, ) -> IndexResult; async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; - async fn dump(&self, uuid: Uuid, path: PathBuf) -> IndexResult<()>; + async fn dump(&self, uid: String, uuid: Uuid, path: PathBuf) -> IndexResult<()>; async fn get_index_stats(&self, uuid: Uuid) -> IndexResult; } diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index ebcc9ed76..6ea42c73d 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::PathBuf}; use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -378,6 +378,13 @@ impl IndexController { Ok(stats) } + pub async fn dump(&self, path: PathBuf) -> anyhow::Result { + eprintln!("index_controller::mod called"); + let res = dump::perform_dump(self, path).await?; + eprintln!("index_controller::mod finished"); + Ok(res) + } + pub async fn get_all_stats(&self) -> anyhow::Result { let update_infos = self.update_handle.get_info().await?; let mut database_size = self.get_uuids_size().await? + update_infos.size; diff --git a/meilisearch-http/src/index_controller/update_actor/actor.rs b/meilisearch-http/src/index_controller/update_actor/actor.rs index fee907001..64794bc6f 100644 --- a/meilisearch-http/src/index_controller/update_actor/actor.rs +++ b/meilisearch-http/src/index_controller/update_actor/actor.rs @@ -235,11 +235,11 @@ where Ok(()) } - async fn handle_dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + async fn handle_dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { let index_handle = self.index_handle.clone(); let update_store = self.store.clone(); tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - update_store.dump(&uuids, &path)?; + update_store.dump(&uuids, path.to_path_buf())?; // Perform the snapshot of each index concurently. Only a third of the capabilities of // the index actor at a time not to put too much pressure on the index actor @@ -247,7 +247,7 @@ where let handle = &index_handle; let mut stream = futures::stream::iter(uuids.iter()) - .map(|&uuid| handle.dump(uuid, path.clone())) + .map(|(uid, uuid)| handle.dump(uid.clone(), *uuid, path.clone())) .buffer_unordered(CONCURRENT_INDEX_MSG / 3); Handle::current().block_on(async { diff --git a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs index 09a242377..a497a3c5c 100644 --- a/meilisearch-http/src/index_controller/update_actor/handle_impl.rs +++ b/meilisearch-http/src/index_controller/update_actor/handle_impl.rs @@ -71,16 +71,16 @@ where receiver.await.expect("update actor killed.") } - async fn get_info(&self) -> Result { + async fn dump(&self, uuids: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()> { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::GetInfo { ret }; + let msg = UpdateMsg::Dump { uuids, path, ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } - async fn dump(&self, uuids: HashSet, path: PathBuf) -> Result<()> { + async fn get_info(&self) -> Result { let (ret, receiver) = oneshot::channel(); - let msg = UpdateMsg::Dump { uuids, path, ret }; + let msg = UpdateMsg::GetInfo { ret }; let _ = self.sender.send(msg).await; receiver.await.expect("update actor killed.") } diff --git a/meilisearch-http/src/index_controller/update_actor/message.rs b/meilisearch-http/src/index_controller/update_actor/message.rs index 37df2af32..4103ca121 100644 --- a/meilisearch-http/src/index_controller/update_actor/message.rs +++ b/meilisearch-http/src/index_controller/update_actor/message.rs @@ -32,7 +32,7 @@ pub enum UpdateMsg { ret: oneshot::Sender>, }, Dump { - uuids: HashSet, + uuids: HashSet<(String, Uuid)>, path: PathBuf, ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/update_actor/mod.rs b/meilisearch-http/src/index_controller/update_actor/mod.rs index eeca6629f..05b793e45 100644 --- a/meilisearch-http/src/index_controller/update_actor/mod.rs +++ b/meilisearch-http/src/index_controller/update_actor/mod.rs @@ -40,7 +40,7 @@ pub trait UpdateActorHandle { async fn update_status(&self, uuid: Uuid, id: u64) -> Result; async fn delete(&self, uuid: Uuid) -> Result<()>; async fn snapshot(&self, uuid: HashSet, path: PathBuf) -> Result<()>; - async fn dump(&self, uuid: HashSet, path: PathBuf) -> Result<()>; + async fn dump(&self, uuid: HashSet<(String, Uuid)>, path: PathBuf) -> Result<()>; async fn get_info(&self) -> Result; async fn update( &self, diff --git a/meilisearch-http/src/index_controller/update_actor/update_store.rs b/meilisearch-http/src/index_controller/update_actor/update_store.rs index 27018764f..d767dfa93 100644 --- a/meilisearch-http/src/index_controller/update_actor/update_store.rs +++ b/meilisearch-http/src/index_controller/update_actor/update_store.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +use std::{borrow::Cow, path::PathBuf}; use std::collections::{BTreeMap, HashSet}; use std::convert::TryInto; use std::fs::{copy, create_dir_all, remove_file, File}; @@ -294,6 +294,7 @@ impl UpdateStore { result: UpdateStatus, index_uuid: Uuid, ) -> heed::Result<()> { + // TODO: TAMO: load already processed updates let mut wtxn = self.env.write_txn()?; let (_global_id, update_id) = self.next_update_id(&mut wtxn, index_uuid)?; self.updates.remap_key_type::().put(&mut wtxn, &(index_uuid, update_id), &result)?; @@ -516,31 +517,34 @@ impl UpdateStore { Ok(()) } - pub fn dump(&self, uuids: &HashSet, path: impl AsRef) -> anyhow::Result<()> { + pub fn dump(&self, uuids: &HashSet<(String, Uuid)>, path: PathBuf) -> anyhow::Result<()> { + use std::io::prelude::*; let state_lock = self.state.write(); - state_lock.swap(State::Snapshoting); // TODO: rename the state + state_lock.swap(State::Snapshoting); // TODO: TAMO rename the state somehow let txn = self.env.write_txn()?; - let update_path = path.as_ref().join("updates"); - create_dir_all(&update_path)?; + for (uid, uuid) in uuids.iter() { + let file = File::create(path.join(uid).join("updates.jsonl"))?; + let mut file = std::io::BufWriter::new(file); - // acquire write lock to prevent further writes during dump - create_dir_all(&update_path)?; - let db_path = update_path.join("data.mdb"); + for update in &self.list(*uuid)? { + serde_json::to_writer(&mut file, update)?; + file.write_all(b"\n")?; + } + } - // TODO: everything - // create db dump - self.env.copy_to_path(&db_path, CompactionOption::Enabled)?; + // TODO: TAMO: the updates + // already processed updates seems to works, but I've not tried with currently running updates - let update_files_path = update_path.join("update_files"); + let update_files_path = path.join("update_files"); create_dir_all(&update_files_path)?; let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data(); for entry in pendings { let ((_, uuid, _), pending) = entry?; - if uuids.contains(&uuid) { + if uuids.iter().any(|(_, id)| id == &uuid) { if let Some(path) = pending.decode()?.content_path() { let name = path.file_name().unwrap(); let to = update_files_path.join(name); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs index df83ceba9..253326276 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/actor.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/actor.rs @@ -41,9 +41,6 @@ impl UuidResolverActor { Some(SnapshotRequest { path, ret }) => { let _ = ret.send(self.handle_snapshot(path).await); } - Some(DumpRequest { path, ret }) => { - let _ = ret.send(self.handle_dump(path).await); - } Some(GetSize { ret }) => { let _ = ret.send(self.handle_get_size().await); } @@ -85,10 +82,6 @@ impl UuidResolverActor { self.store.snapshot(path).await } - async fn handle_dump(&self, path: PathBuf) -> Result> { - self.store.dump(path).await - } - async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> { if !is_index_uid_valid(&uid) { return Err(UuidError::BadlyFormatted(uid)); diff --git a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs index d9e9a20fc..db4c482bd 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs @@ -68,7 +68,6 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - /// TODO: we should merge this function with the dump function async fn snapshot(&self, path: PathBuf) -> Result> { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::SnapshotRequest { path, ret }; @@ -78,15 +77,6 @@ impl UuidResolverHandle for UuidResolverHandleImpl { .expect("Uuid resolver actor has been killed")?) } - async fn dump(&self, path: PathBuf) -> Result> { - let (ret, receiver) = oneshot::channel(); - let msg = UuidResolveMsg::DumpRequest { path, ret }; - let _ = self.sender.send(msg).await; - Ok(receiver - .await - .expect("Uuid resolver actor has been killed")?) - } - async fn get_size(&self) -> Result { let (ret, receiver) = oneshot::channel(); let msg = UuidResolveMsg::GetSize { ret }; diff --git a/meilisearch-http/src/index_controller/uuid_resolver/message.rs b/meilisearch-http/src/index_controller/uuid_resolver/message.rs index 78f62eea2..a72bf0587 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/message.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/message.rs @@ -31,10 +31,6 @@ pub enum UuidResolveMsg { path: PathBuf, ret: oneshot::Sender>>, }, - DumpRequest { - path: PathBuf, - ret: oneshot::Sender>>, - }, GetSize { ret: oneshot::Sender>, }, diff --git a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs index aca730db9..0cbb2895b 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/mod.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/mod.rs @@ -32,7 +32,6 @@ pub trait UuidResolverHandle { async fn delete(&self, name: String) -> anyhow::Result; async fn list(&self) -> anyhow::Result>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } diff --git a/meilisearch-http/src/index_controller/uuid_resolver/store.rs b/meilisearch-http/src/index_controller/uuid_resolver/store.rs index 917e0b4a5..a781edcba 100644 --- a/meilisearch-http/src/index_controller/uuid_resolver/store.rs +++ b/meilisearch-http/src/index_controller/uuid_resolver/store.rs @@ -21,7 +21,6 @@ pub trait UuidStore { async fn list(&self) -> Result>; async fn insert(&self, name: String, uuid: Uuid) -> Result<()>; async fn snapshot(&self, path: PathBuf) -> Result>; - async fn dump(&self, path: PathBuf) -> Result>; async fn get_size(&self) -> Result; } @@ -114,8 +113,6 @@ impl HeedUuidStore { Ok(()) } - // TODO: we should merge this function and the following function for the dump. it's exactly - // the same code pub fn snapshot(&self, mut path: PathBuf) -> Result> { let env = self.env.clone(); let db = self.db; @@ -138,28 +135,6 @@ impl HeedUuidStore { Ok(entries) } - pub fn dump(&self, mut path: PathBuf) -> Result> { - let env = self.env.clone(); - let db = self.db; - // Write transaction to acquire a lock on the database. - let txn = env.write_txn()?; - let mut entries = HashSet::new(); - for entry in db.iter(&txn)? { - let (_, uuid) = entry?; - let uuid = Uuid::from_slice(uuid)?; - entries.insert(uuid); - } - - // only perform dump if there are indexes - if !entries.is_empty() { - path.push("index_uuids"); - create_dir_all(&path).unwrap(); - path.push("data.mdb"); - env.copy_to_path(path, CompactionOption::Enabled)?; - } - Ok(entries) - } - pub fn get_size(&self) -> Result { Ok(self.env.size()) } @@ -197,11 +172,6 @@ impl UuidStore for HeedUuidStore { tokio::task::spawn_blocking(move || this.snapshot(path)).await? } - async fn dump(&self, path: PathBuf) -> Result> { - let this = self.clone(); - tokio::task::spawn_blocking(move || this.dump(path)).await? - } - async fn get_size(&self) -> Result { self.get_size() } diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index fd5cf6786..e19037482 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -46,8 +46,8 @@ macro_rules! create_app { .configure(synonym::services) .configure(health::services) .configure(stats::services) - .configure(key::services); - //.configure(routes::dump::services); + .configure(key::services) + .configure(dump::services); #[cfg(feature = "mini-dashboard")] let app = if $enable_frontend { let generated = dashboard::generate(); diff --git a/meilisearch-http/src/routes/dump.rs b/meilisearch-http/src/routes/dump.rs index c46b0e502..410b817b8 100644 --- a/meilisearch-http/src/routes/dump.rs +++ b/meilisearch-http/src/routes/dump.rs @@ -1,14 +1,10 @@ -use std::fs::File; -use std::path::Path; +use actix_web::{post, get, web}; +use actix_web::HttpResponse; +use serde::{Serialize, Deserialize}; -use actix_web::{get, post}; -use actix_web::{HttpResponse, web}; -use serde::{Deserialize, Serialize}; - -use crate::dump::{DumpInfo, DumpStatus, compressed_dumps_dir, init_dump_process}; -use crate::Data; -use crate::error::{Error, ResponseError}; +use crate::error::ResponseError; use crate::helpers::Authentication; +use crate::Data; pub fn services(cfg: &mut web::ServiceConfig) { cfg.service(trigger_dump) @@ -19,7 +15,10 @@ pub fn services(cfg: &mut web::ServiceConfig) { async fn trigger_dump( data: web::Data, ) -> Result { - todo!() + eprintln!("dump started"); + let res = data.dump().await?; + + Ok(HttpResponse::Ok().body(res)) } #[derive(Debug, Serialize)] @@ -30,13 +29,13 @@ struct DumpStatusResponse { #[derive(Deserialize)] struct DumpParam { - dump_uid: String, + _dump_uid: String, } #[get("/dumps/{dump_uid}/status", wrap = "Authentication::Private")] async fn get_dump_status( - data: web::Data, - path: web::Path, + _data: web::Data, + _path: web::Path, ) -> Result { todo!() } diff --git a/meilisearch-http/src/routes/mod.rs b/meilisearch-http/src/routes/mod.rs index aaf13613a..999c4f881 100644 --- a/meilisearch-http/src/routes/mod.rs +++ b/meilisearch-http/src/routes/mod.rs @@ -9,7 +9,7 @@ pub mod search; pub mod settings; pub mod stats; pub mod synonym; -//pub mod dump; +pub mod dump; #[derive(Deserialize)] pub struct IndexParam {