From a9844bd4f69630cd4cbab6f7dc93c555fa8bd05b Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 13 Sep 2022 15:35:57 +0200 Subject: [PATCH] move the update file store to another crate with as little dependencies as possible --- Cargo.lock | 10 +++ Cargo.toml | 1 + file-store/Cargo.toml | 14 ++++ file-store/src/lib.rs | 166 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+) create mode 100644 file-store/Cargo.toml create mode 100644 file-store/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 1687f3b66..739c71db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1169,6 +1169,16 @@ dependencies = [ "instant", ] +[[package]] +name = "file-store" +version = "0.1.0" +dependencies = [ + "nelson", + "tempfile", + "thiserror", + "uuid", +] + [[package]] name = "filetime" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index e4325adce..49122460d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "meilisearch-lib", "meilisearch-auth", "index-scheduler", + "file-store", "permissive-json-pointer", ] diff --git a/file-store/Cargo.toml b/file-store/Cargo.toml new file mode 100644 index 000000000..0bba339cb --- /dev/null +++ b/file-store/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "file-store" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +uuid = { version = "1.1.2", features = ["serde", "v4"] } +tempfile = "3.3.0" +thiserror = "1.0.30" + +[dev-dependencies] +nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"} diff --git a/file-store/src/lib.rs b/file-store/src/lib.rs new file mode 100644 index 000000000..4d27914f4 --- /dev/null +++ b/file-store/src/lib.rs @@ -0,0 +1,166 @@ +use std::fs::File; +use std::ops::{Deref, DerefMut}; +use std::path::{Path, PathBuf}; + +use tempfile::NamedTempFile; +use uuid::Uuid; + +#[cfg(not(test))] +pub use store::UpdateFileStore; +#[cfg(test)] +pub use test::MockUpdateFileStore as UpdateFileStore; + +const UPDATE_FILES_PATH: &str = "updates/updates_files"; + +pub struct UpdateFile { + path: PathBuf, + file: NamedTempFile, +} + +#[derive(Debug, thiserror::Error)] +pub enum UpdateFileStoreError { + #[error("Error while persisting update to disk")] + Error, + #[error(transparent)] + IoError(#[from] std::io::Error), + #[error(transparent)] + PersistError(#[from] tempfile::PersistError), +} + +pub type Result = std::result::Result; + +impl UpdateFile { + pub fn persist(self) -> Result<()> { + self.file.persist(&self.path)?; + Ok(()) + } +} + +impl Deref for UpdateFile { + type Target = NamedTempFile; + + fn deref(&self) -> &Self::Target { + &self.file + } +} + +impl DerefMut for UpdateFile { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.file + } +} + +mod store { + use super::*; + + #[derive(Clone, Debug)] + pub struct UpdateFileStore { + path: PathBuf, + } + + impl UpdateFileStore { + pub fn new(path: impl AsRef) -> Result { + let path = path.as_ref().join(UPDATE_FILES_PATH); + std::fs::create_dir_all(&path)?; + Ok(Self { path }) + } + + /// Creates a new temporary update file. + /// A call to `persist` is needed to persist the file in the database. + pub fn new_update(&self) -> Result<(Uuid, UpdateFile)> { + let file = NamedTempFile::new_in(&self.path)?; + let uuid = Uuid::new_v4(); + let path = self.path.join(uuid.to_string()); + let update_file = UpdateFile { file, path }; + + Ok((uuid, update_file)) + } + + /// Returns the file corresponding to the requested uuid. + pub fn get_update(&self, uuid: Uuid) -> Result { + let path = self.path.join(uuid.to_string()); + let file = File::open(path)?; + Ok(file) + } + + /// Copies the content of the update file pointed to by `uuid` to the `dst` directory. + pub fn snapshot(&self, uuid: Uuid, dst: impl AsRef) -> Result<()> { + let src = self.path.join(uuid.to_string()); + let mut dst = dst.as_ref().join(UPDATE_FILES_PATH); + std::fs::create_dir_all(&dst)?; + dst.push(uuid.to_string()); + std::fs::copy(src, dst)?; + Ok(()) + } + + pub fn get_size(&self, uuid: Uuid) -> Result { + Ok(self.get_update(uuid)?.metadata()?.len()) + } + + pub fn delete(&self, uuid: Uuid) -> Result<()> { + let path = self.path.join(uuid.to_string()); + std::fs::remove_file(path)?; + Ok(()) + } + } +} + +#[cfg(test)] +mod test { + use std::sync::Arc; + + use nelson::Mocker; + + use super::*; + + #[derive(Clone)] + pub enum MockUpdateFileStore { + Real(store::UpdateFileStore), + Mock(Arc), + } + + impl MockUpdateFileStore { + pub fn mock(mocker: Mocker) -> Self { + Self::Mock(Arc::new(mocker)) + } + + pub fn new(path: impl AsRef) -> Result { + store::UpdateFileStore::new(path).map(Self::Real) + } + + pub fn new_update(&self) -> Result<(Uuid, UpdateFile)> { + match self { + MockUpdateFileStore::Real(s) => s.new_update(), + MockUpdateFileStore::Mock(_) => todo!(), + } + } + + pub fn get_update(&self, uuid: Uuid) -> Result { + match self { + MockUpdateFileStore::Real(s) => s.get_update(uuid), + MockUpdateFileStore::Mock(_) => todo!(), + } + } + + pub fn snapshot(&self, uuid: Uuid, dst: impl AsRef) -> Result<()> { + match self { + MockUpdateFileStore::Real(s) => s.snapshot(uuid, dst), + MockUpdateFileStore::Mock(_) => todo!(), + } + } + + pub fn get_size(&self, uuid: Uuid) -> Result { + match self { + MockUpdateFileStore::Real(s) => s.get_size(uuid), + MockUpdateFileStore::Mock(_) => todo!(), + } + } + + pub fn delete(&self, uuid: Uuid) -> Result<()> { + match self { + MockUpdateFileStore::Real(s) => s.delete(uuid), + MockUpdateFileStore::Mock(mocker) => unsafe { mocker.get("delete").call(uuid) }, + } + } + } +}