From 331d28102f930570bd4d4872c2e17a6ce13b8602 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 7 Sep 2021 16:55:35 +0200 Subject: [PATCH 1/2] Change the format of custom ranking rules when importing v1 dumps --- .../index_controller/dump_actor/loaders/v1.rs | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 6a505c077..3c22a8236 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -138,6 +138,20 @@ fn load_index( Ok(()) } +/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. +fn asc_ranking_rule(text: &str) -> Option<&str> { + text.split_once("asc(") + .and_then(|(_, tail)| tail.rsplit_once(")")) + .map(|(field, _)| field) +} + +/// Parses the v1 version of the Desc ranking rules `asc(price)`and returns the field name. +fn desc_ranking_rule(text: &str) -> Option<&str> { + text.split_once("desc(") + .and_then(|(_, tail)| tail.rsplit_once(")")) + .map(|(field, _)| field) +} + /// we need to **always** be able to convert the old settings to the settings currently being used impl From for index_controller::Settings { fn from(settings: Settings) -> Self { @@ -164,19 +178,21 @@ impl From for index_controller::Settings { None => Setting::NotSet }, sortable_attributes: Setting::NotSet, - // we need to convert the old `Vec` into a `BTreeSet` ranking_rules: match settings.ranking_rules { - Some(Some(ranking_rules)) => Setting::Set(ranking_rules.into_iter().filter(|criterion| { + Some(Some(ranking_rules)) => Setting::Set(ranking_rules.into_iter().filter_map(|criterion| { match criterion.as_str() { - "words" | "typo" | "proximity" | "attribute" | "exactness" => true, - s if s.starts_with("asc") || s.starts_with("desc") => true, + "words" | "typo" | "proximity" | "attribute" | "exactness" => Some(criterion), + s if s.starts_with("asc") => asc_ranking_rule(s).map(|f| format!("{}:asc", f)), + s if s.starts_with("desc") => desc_ranking_rule(s).map(|f| format!("{}:desc", f)), "wordsPosition" => { - warn!("The criteria `attribute` and `wordsPosition` have been merged into a single criterion `attribute` so `wordsPositon` will be ignored"); - false + warn!("The criteria `attribute` and `wordsPosition` have been merged \ + into a single criterion `attribute` so `wordsPositon` will be \ + ignored"); + None } s => { error!("Unknown criterion found in the dump: `{}`, it will be ignored", s); - false + None } } }).collect()), From be50b2bec6e28ae97bdc4396f10da2619700e8e4 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 8 Sep 2021 11:43:10 +0200 Subject: [PATCH 2/2] Change the format of custom ranking rules when importing v2 dumps --- meilisearch-http/src/index/dump.rs | 38 ++++++++++++++++++- .../index_controller/dump_actor/loaders/v1.rs | 15 +------- meilisearch-http/src/index_controller/mod.rs | 14 +++++++ 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/meilisearch-http/src/index/dump.rs b/meilisearch-http/src/index/dump.rs index 3b10a1562..7df704339 100644 --- a/meilisearch-http/src/index/dump.rs +++ b/meilisearch-http/src/index/dump.rs @@ -8,7 +8,9 @@ use heed::RoTxn; use indexmap::IndexMap; use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream}; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use crate::index_controller::{asc_ranking_rule, desc_ranking_rule}; use crate::option::IndexerOpts; use super::error::Result; @@ -93,10 +95,22 @@ impl Index { let meta_path = src.as_ref().join(META_FILE_NAME); let mut meta_file = File::open(meta_path)?; + + // We first deserialize the dump meta into a serde_json::Value and change + // the custom ranking rules settings from the old format to the new format. + let mut meta: Value = serde_json::from_reader(&mut meta_file)?; + if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") { + convert_custom_ranking_rules(ranking_rules); + } + + // Then we serialize it back into a vec to deserialize it + // into a `DumpMeta` struct with the newly patched `rankingRules` format. + let patched_meta = serde_json::to_vec(&meta)?; + let DumpMeta { settings, primary_key, - } = serde_json::from_reader(&mut meta_file)?; + } = serde_json::from_slice(&patched_meta)?; let settings = settings.check(); let index = Self::open(&dst_dir_path, size)?; let mut txn = index.write_txn()?; @@ -132,3 +146,25 @@ impl Index { Ok(()) } } + +/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`. +/// +/// This is done for compatibility reasons, and to avoid a new dump version, +/// since the new syntax was introduced soon after the new dump version. +fn convert_custom_ranking_rules(ranking_rules: &mut Value) { + *ranking_rules = match ranking_rules.take() { + Value::Array(values) => values + .into_iter() + .filter_map(|value| match value { + Value::String(s) if s.starts_with("asc") => asc_ranking_rule(&s) + .map(|f| format!("{}:asc", f)) + .map(Value::String), + Value::String(s) if s.starts_with("desc") => desc_ranking_rule(&s) + .map(|f| format!("{}:desc", f)) + .map(Value::String), + otherwise => Some(otherwise), + }) + .collect(), + otherwise => otherwise, + } +} diff --git a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs index 3c22a8236..997fd2801 100644 --- a/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs +++ b/meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs @@ -12,6 +12,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use uuid::Uuid; use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata}; +use crate::index_controller::{asc_ranking_rule, desc_ranking_rule}; use crate::{ index::{update_handler::UpdateHandler, Index, Unchecked}, option::IndexerOpts, @@ -138,20 +139,6 @@ fn load_index( Ok(()) } -/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. -fn asc_ranking_rule(text: &str) -> Option<&str> { - text.split_once("asc(") - .and_then(|(_, tail)| tail.rsplit_once(")")) - .map(|(field, _)| field) -} - -/// Parses the v1 version of the Desc ranking rules `asc(price)`and returns the field name. -fn desc_ranking_rule(text: &str) -> Option<&str> { - text.split_once("desc(") - .and_then(|(_, tail)| tail.rsplit_once(")")) - .map(|(field, _)| field) -} - /// we need to **always** be able to convert the old settings to the settings currently being used impl From for index_controller::Settings { fn from(settings: Settings) -> Self { diff --git a/meilisearch-http/src/index_controller/mod.rs b/meilisearch-http/src/index_controller/mod.rs index a90498b9c..ca8c98f83 100644 --- a/meilisearch-http/src/index_controller/mod.rs +++ b/meilisearch-http/src/index_controller/mod.rs @@ -439,3 +439,17 @@ pub async fn get_arc_ownership_blocking(mut item: Arc) -> T { } } } + +/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name. +pub fn asc_ranking_rule(text: &str) -> Option<&str> { + text.split_once("asc(") + .and_then(|(_, tail)| tail.rsplit_once(")")) + .map(|(field, _)| field) +} + +/// Parses the v1 version of the Desc ranking rules `asc(price)`and returns the field name. +pub fn desc_ranking_rule(text: &str) -> Option<&str> { + text.split_once("desc(") + .and_then(|(_, tail)| tail.rsplit_once(")")) + .map(|(field, _)| field) +}