From 90afde435bb3fb5b391dcd4d95bb6dcc3d0bc14d Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 9 Jun 2022 16:03:49 +0200 Subject: [PATCH 1/2] fix escaped quotes in filter --- filter-parser/src/lib.rs | 13 ++---- milli/src/search/facet/filter.rs | 79 ++++++++++++++++++++++++++++++-- 2 files changed, 78 insertions(+), 14 deletions(-) diff --git a/filter-parser/src/lib.rs b/filter-parser/src/lib.rs index bad7dbc64..243d1a3f4 100644 --- a/filter-parser/src/lib.rs +++ b/filter-parser/src/lib.rs @@ -40,7 +40,6 @@ mod error; mod value; use std::fmt::Debug; -use std::ops::Deref; use std::str::FromStr; pub use condition::{parse_condition, parse_to, Condition}; @@ -70,14 +69,6 @@ pub struct Token<'a> { value: Option, } -impl<'a> Deref for Token<'a> { - type Target = &'a str; - - fn deref(&self) -> &Self::Target { - &self.span - } -} - impl<'a> PartialEq for Token<'a> { fn eq(&self, other: &Self) -> bool { self.span.fragment() == other.span.fragment() @@ -89,6 +80,10 @@ impl<'a> Token<'a> { Self { span, value } } + pub fn lexeme(&self) -> &str { + &self.span + } + pub fn value(&self) -> &str { self.value.as_ref().map_or(&self.span, |value| value) } diff --git a/milli/src/search/facet/filter.rs b/milli/src/search/facet/filter.rs index 8f1ee749f..a809aa5fb 100644 --- a/milli/src/search/facet/filter.rs +++ b/milli/src/search/facet/filter.rs @@ -1,7 +1,6 @@ use std::collections::HashSet; use std::fmt::{Debug, Display}; use std::ops::Bound::{self, Excluded, Included}; -use std::ops::Deref; use either::Either; pub use filter_parser::{Condition, Error as FPError, FilterCondition, Span, Token}; @@ -283,8 +282,9 @@ impl<'a> Filter<'a> { Condition::LowerThanOrEqual(val) => (Included(f64::MIN), Included(val.parse()?)), Condition::Between { from, to } => (Included(from.parse()?), Included(to.parse()?)), Condition::Equal(val) => { - let (_original_value, string_docids) = - strings_db.get(rtxn, &(field_id, &val.to_lowercase()))?.unwrap_or_default(); + let (_original_value, string_docids) = strings_db + .get(rtxn, &(field_id, &val.value().to_lowercase()))? + .unwrap_or_default(); let number = val.parse::().ok(); let number_docids = match number { Some(n) => { @@ -362,7 +362,7 @@ impl<'a> Filter<'a> { return Ok(RoaringBitmap::new()); } } else { - match *fid.deref() { + match fid.lexeme() { attribute @ "_geo" => { return Err(fid.as_external_error(FilterError::BadGeo(attribute)))?; } @@ -461,7 +461,7 @@ mod tests { use maplit::hashset; use super::*; - use crate::update::{IndexerConfig, Settings}; + use crate::update::{self, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings}; use crate::Index; #[test] @@ -598,6 +598,75 @@ mod tests { )); } + #[test] + fn escaped_quote_in_filter_value_2380() { + let path = tempfile::tempdir().unwrap(); + let mut options = EnvOpenOptions::new(); + options.map_size(10 * 1024 * 1024); // 10 MB + let index = Index::new(options, &path).unwrap(); + + let mut wtxn = index.write_txn().unwrap(); + let content = documents!([ + { + "id": "test_1", + "monitor_diagonal": "27' to 30'" + }, + { + "id": "test_2", + "monitor_diagonal": "27\" to 30\"" + }, + { + "id": "test_3", + "monitor_diagonal": "27\" to 30'" + }, + ]); + + let config = IndexerConfig::default(); + let indexing_config = IndexDocumentsConfig::default(); + let mut builder = + IndexDocuments::new(&mut wtxn, &index, &config, indexing_config.clone(), |_| ()) + .unwrap(); + builder.add_documents(content).unwrap(); + builder.execute().unwrap(); + + wtxn.commit().unwrap(); + + let mut wtxn = index.write_txn().unwrap(); + let mut builder = update::Settings::new(&mut wtxn, &index, &config); + + builder.set_filterable_fields(hashset!(S("monitor_diagonal"))); + builder.execute(|_| ()).unwrap(); + wtxn.commit().unwrap(); + + let rtxn = index.read_txn().unwrap(); + + let mut search = crate::Search::new(&rtxn, &index); + // this filter is copy pasted from #2380 with the exact same espace sequence + search.filter( + crate::Filter::from_str("monitor_diagonal = '27\" to 30\\''").unwrap().unwrap(), + ); + let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); + assert_eq!(documents_ids, vec![2]); + + search.filter( + crate::Filter::from_str(r#"monitor_diagonal = "27' to 30'" "#).unwrap().unwrap(), + ); + let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); + assert_eq!(documents_ids, vec![0]); + + search.filter( + crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30\"" "#).unwrap().unwrap(), + ); + let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); + assert_eq!(documents_ids, vec![1]); + + search.filter( + crate::Filter::from_str(r#"monitor_diagonal = "27\" to 30'" "#).unwrap().unwrap(), + ); + let crate::SearchResult { documents_ids, .. } = search.execute().unwrap(); + assert_eq!(documents_ids, vec![2]); + } + #[test] fn geo_radius_error() { let path = tempfile::tempdir().unwrap(); From 676187ba430019c2f86c7836332b592a100bcd5e Mon Sep 17 00:00:00 2001 From: Tamo Date: Thu, 9 Jun 2022 16:53:32 +0200 Subject: [PATCH 2/2] bump milli version --- benchmarks/Cargo.toml | 2 +- cli/Cargo.toml | 2 +- filter-parser/Cargo.toml | 2 +- flatten-serde-json/Cargo.toml | 2 +- helpers/Cargo.toml | 2 +- http-ui/Cargo.toml | 2 +- infos/Cargo.toml | 2 +- json-depth-checker/Cargo.toml | 2 +- milli/Cargo.toml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 44507d527..2c6c93bd8 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "benchmarks" -version = "0.29.2" +version = "0.29.3" edition = "2018" publish = false diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 01dc243b7..20dc25c28 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cli" -version = "0.29.2" +version = "0.29.3" edition = "2018" description = "A CLI to interact with a milli index" publish = false diff --git a/filter-parser/Cargo.toml b/filter-parser/Cargo.toml index d20407de2..e8d54ab4f 100644 --- a/filter-parser/Cargo.toml +++ b/filter-parser/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "filter-parser" -version = "0.29.2" +version = "0.29.3" edition = "2021" description = "The parser for the Meilisearch filter syntax" publish = false diff --git a/flatten-serde-json/Cargo.toml b/flatten-serde-json/Cargo.toml index 014d10ad6..ca6332a50 100644 --- a/flatten-serde-json/Cargo.toml +++ b/flatten-serde-json/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "flatten-serde-json" -version = "0.29.2" +version = "0.29.3" edition = "2021" description = "Flatten serde-json objects like elastic search" readme = "README.md" diff --git a/helpers/Cargo.toml b/helpers/Cargo.toml index 41573268f..cdf7c3b2e 100644 --- a/helpers/Cargo.toml +++ b/helpers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "helpers" -version = "0.29.2" +version = "0.29.3" authors = ["Clément Renault "] edition = "2018" description = "A small tool to do operations on the database" diff --git a/http-ui/Cargo.toml b/http-ui/Cargo.toml index 493d54e12..34e9df773 100644 --- a/http-ui/Cargo.toml +++ b/http-ui/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "http-ui" description = "The HTTP user interface of the milli search engine" -version = "0.29.2" +version = "0.29.3" authors = ["Clément Renault "] edition = "2018" publish = false diff --git a/infos/Cargo.toml b/infos/Cargo.toml index 69ae2e9de..169cc7e08 100644 --- a/infos/Cargo.toml +++ b/infos/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "infos" -version = "0.29.2" +version = "0.29.3" authors = ["Clément Renault "] edition = "2018" publish = false diff --git a/json-depth-checker/Cargo.toml b/json-depth-checker/Cargo.toml index 8383aad0a..8bc092ad0 100644 --- a/json-depth-checker/Cargo.toml +++ b/json-depth-checker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "json-depth-checker" -version = "0.29.2" +version = "0.29.3" edition = "2021" description = "A library that indicates if a JSON must be flattened" publish = false diff --git a/milli/Cargo.toml b/milli/Cargo.toml index 96f48707d..70bb8f585 100644 --- a/milli/Cargo.toml +++ b/milli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "milli" -version = "0.29.2" +version = "0.29.3" authors = ["Kerollmops "] edition = "2018"