From e8c4ea5de2a9ff069f5d94e807fb5b4b452a3831 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Wed, 29 Jan 2025 17:31:37 +0100 Subject: [PATCH] Support network in dumps --- crates/dump/src/reader/compat/v5_to_v6.rs | 4 ++++ crates/dump/src/reader/mod.rs | 7 ++++++ crates/dump/src/reader/v6/mod.rs | 24 +++++++++++++++++++ crates/dump/src/writer.rs | 6 ++++- .../src/scheduler/process_dump_creation.rs | 2 ++ crates/meilisearch/src/lib.rs | 5 +++- 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index 6b2655bdf..2dd4ed761 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -196,6 +196,10 @@ impl CompatV5ToV6 { pub fn features(&self) -> Result> { Ok(None) } + + pub fn network(&self) -> Result> { + Ok(None) + } } pub enum CompatIndexV5ToV6 { diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 151267378..50180fbc7 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -114,6 +114,13 @@ impl DumpReader { DumpReader::Compat(compat) => compat.features(), } } + + pub fn network(&self) -> Result> { + match self { + DumpReader::Current(current) => Ok(current.network()), + DumpReader::Compat(compat) => compat.network(), + } + } } impl From for DumpReader { diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 50b9751a2..4c05f16bf 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -20,6 +20,7 @@ pub type Unchecked = meilisearch_types::settings::Unchecked; pub type Task = crate::TaskDump; pub type Key = meilisearch_types::keys::Key; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; +pub type Network = meilisearch_types::features::Network; // ===== Other types to clarify the code of the compat module // everything related to the tasks @@ -50,6 +51,7 @@ pub struct V6Reader { tasks: BufReader, keys: BufReader, features: Option, + network: Option, } impl V6Reader { @@ -78,12 +80,30 @@ impl V6Reader { None }; + let network_file = match fs::read(dump.path().join("network.json")) { + Ok(network_file) => Some(network_file), + Err(error) => match error.kind() { + // Allows the file to be missing, this will only result in all experimental features disabled. + ErrorKind::NotFound => { + debug!("`network.json` not found in dump"); + None + } + _ => return Err(error.into()), + }, + }; + let network = if let Some(network_file) = network_file { + Some(serde_json::from_reader(&*network_file)?) + } else { + None + }; + Ok(V6Reader { metadata: serde_json::from_reader(&*meta_file)?, instance_uid, tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?), keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?), features, + network, dump, }) } @@ -154,6 +174,10 @@ impl V6Reader { pub fn features(&self) -> Option { self.features } + + pub fn network(&self) -> Option<&Network> { + self.network.as_ref() + } } pub struct UpdateFile { diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 3ee51cabf..880441445 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; -use meilisearch_types::features::RuntimeTogglableFeatures; +use meilisearch_types::features::{Network, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; use serde_json::{Map, Value}; @@ -61,6 +61,10 @@ impl DumpWriter { )?) } + pub fn create_network(&self, network: Network) -> Result<()> { + Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) + } + pub fn persist_to(self, mut writer: impl Write) -> Result<()> { let gz_encoder = GzEncoder::new(&mut writer, Compression::default()); let mut tar_encoder = tar::Builder::new(gz_encoder); diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index 09c1020ac..adf5a5b61 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -219,6 +219,8 @@ impl IndexScheduler { progress.update_progress(DumpCreationProgress::DumpTheExperimentalFeatures); let features = self.features().runtime_features(); dump.create_experimental_features(features)?; + let network = self.network(); + dump.create_network(network)?; let dump_uid = started_at.format(format_description!( "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 4d41c63ea..cbd299f26 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -431,10 +431,13 @@ fn import_dump( keys.push(key); } - // 3. Import the runtime features. + // 3. Import the runtime features and network let features = dump_reader.features()?.unwrap_or_default(); index_scheduler.put_runtime_features(features)?; + let network = dump_reader.network()?.cloned().unwrap_or_default(); + index_scheduler.put_network(network)?; + let indexer_config = index_scheduler.indexer_config(); // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might