searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-12-24 13:40:31 +01:00
Code Issues Releases Wiki Activity

Refine the facet condition to use both facet databases

Browse Source
This commit is contained in:
Clément Renault 2021-05-03 11:45:45 +02:00 committed by Kerollmops
parent e62b89a2ed
commit f7efde11d9
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
3 changed files with 111 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

304
milli/src/search/facet/facet_condition.rs
View File

@ -1,4 +1,4 @@
use std::collections::HashMap; use std::collections::{HashMap, HashSet};
use std::fmt::Debug; use std::fmt::Debug;
use std::ops::Bound::{self, Included, Excluded}; use std::ops::Bound::{self, Included, Excluded};
use std::str::FromStr; use std::str::FromStr;
@ -21,76 +21,52 @@ use super::parser::Rule;
use super::parser::{PREC_CLIMBER, FilterParser}; use super::parser::{PREC_CLIMBER, FilterParser};
use self::FacetCondition::*; use self::FacetCondition::*;
use self::FacetNumberOperator::*; use self::Operator::*;
#[derive(Debug, Copy, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum FacetNumberOperator { pub enum Operator {
GreaterThan(f64), GreaterThan(f64),
GreaterThanOrEqual(f64), GreaterThanOrEqual(f64),
Equal(f64), Equal(Option<f64>, String),
NotEqual(f64), NotEqual(Option<f64>, String),
LowerThan(f64), LowerThan(f64),
LowerThanOrEqual(f64), LowerThanOrEqual(f64),
Between(f64, f64), Between(f64, f64),
} }
impl FacetNumberOperator { impl Operator {
/// This method can return two operations in case it must express /// This method can return two operations in case it must express
/// an OR operation for the between case (i.e. `TO`). /// an OR operation for the between case (i.e. `TO`).
fn negate(self) -> (Self, Option<Self>) { fn negate(self) -> (Self, Option<Self>) {
match self { match self {
GreaterThan(x) => (LowerThanOrEqual(x), None), GreaterThan(n) => (LowerThanOrEqual(n), None),
GreaterThanOrEqual(x) => (LowerThan(x), None), GreaterThanOrEqual(n) => (LowerThan(n), None),
Equal(x) => (NotEqual(x), None), Equal(n, s) => (NotEqual(n, s), None),
NotEqual(x) => (Equal(x), None), NotEqual(n, s) => (Equal(n, s), None),
LowerThan(x) => (GreaterThanOrEqual(x), None), LowerThan(n) => (GreaterThanOrEqual(n), None),
LowerThanOrEqual(x) => (GreaterThan(x), None), LowerThanOrEqual(n) => (GreaterThan(n), None),
Between(x, y) => (LowerThan(x), Some(GreaterThan(y))), Between(n, m) => (LowerThan(n), Some(GreaterThan(m))),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub enum FacetStringOperator {
Equal(String),
NotEqual(String),
}
impl FacetStringOperator {
fn equal(s: &str) -> Self {
FacetStringOperator::Equal(s.to_lowercase())
}
#[allow(dead_code)]
fn not_equal(s: &str) -> Self {
FacetStringOperator::equal(s).negate()
}
fn negate(self) -> Self {
match self {
FacetStringOperator::Equal(x) => FacetStringOperator::NotEqual(x),
FacetStringOperator::NotEqual(x) => FacetStringOperator::Equal(x),
} }
} }
} }
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum FacetCondition { pub enum FacetCondition {
OperatorString(FieldId, FacetStringOperator), Operator(FieldId, Operator),
OperatorNumber(FieldId, FacetNumberOperator),
Or(Box<Self>, Box<Self>), Or(Box<Self>, Box<Self>),
And(Box<Self>, Box<Self>), And(Box<Self>, Box<Self>),
} }
fn get_field_id_facet_type<'a>( fn field_id<'a>(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
items: &mut Pairs<'a, Rule>, items: &mut Pairs<'a, Rule>,
) -> Result<(FieldId, FacetType), PestError<Rule>> ) -> Result<FieldId, PestError<Rule>>
{ {
// lexing ensures that we at least have a key // lexing ensures that we at least have a key
let key = items.next().unwrap(); let key = items.next().unwrap();
let field_id = fields_ids_map
fields_ids_map
.id(key.as_str()) .id(key.as_str())
.ok_or_else(|| { .ok_or_else(|| {
PestError::new_from_span( PestError::new_from_span(
@ -103,32 +79,14 @@ fn get_field_id_facet_type<'a>(
}, },
key.as_span(), key.as_span(),
) )
})?; })
let facet_type = faceted_fields
.get(&field_id)
.copied()
.ok_or_else(|| {
PestError::new_from_span(
ErrorVariant::CustomError {
message: format!(
"attribute `{}` is not faceted, available faceted attributes are: {}",
key.as_str(),
faceted_fields.keys().flat_map(|id| fields_ids_map.name(*id)).collect::<Vec<_>>().join(", ")
),
},
key.as_span(),
)
})?;
Ok((field_id, facet_type))
} }
fn pest_parse<T>(pair: Pair<Rule>) -> Result<T, pest::error::Error<Rule>> fn pest_parse<T>(pair: Pair<Rule>) -> (Result<T, pest::error::Error<Rule>>, String)
where T: FromStr, where T: FromStr,
T::Err: ToString, T::Err: ToString,
{ {
match pair.as_str().parse() { let result = match pair.as_str().parse::<T>() {
Ok(value) => Ok(value), Ok(value) => Ok(value),
Err(e) => { Err(e) => {
Err(PestError::<Rule>::new_from_span( Err(PestError::<Rule>::new_from_span(
@ -136,7 +94,9 @@ where T: FromStr,
pair.as_span(), pair.as_span(),
)) ))
} }
} };
(result, pair.as_str().to_string())
} }
impl FacetCondition { impl FacetCondition {
@ -232,7 +192,7 @@ impl FacetCondition {
fn from_pairs( fn from_pairs(
fim: &FieldsIdsMap, fim: &FieldsIdsMap,
ff: &HashMap<FieldId, FacetType>, ff: &HashSet<FieldId>,
expression: Pairs<Rule>, expression: Pairs<Rule>,
) -> anyhow::Result<Self> ) -> anyhow::Result<Self>
{ {
@ -263,10 +223,9 @@ impl FacetCondition {
fn negate(self) -> FacetCondition { fn negate(self) -> FacetCondition {
match self { match self {
OperatorString(fid, op) => OperatorString(fid, op.negate()), Operator(fid, op) => match op.negate() {
OperatorNumber(fid, op) => match op.negate() { (op, None) => Operator(fid, op),
(op, None) => OperatorNumber(fid, op), (a, Some(b)) => Or(Box::new(Operator(fid, a)), Box::new(Operator(fid, b))),
(a, Some(b)) => Or(Box::new(OperatorNumber(fid, a)), Box::new(OperatorNumber(fid, b))),
}, },
Or(a, b) => And(Box::new(a.negate()), Box::new(b.negate())), Or(a, b) => And(Box::new(a.negate()), Box::new(b.negate())),
And(a, b) => Or(Box::new(a.negate()), Box::new(b.negate())), And(a, b) => Or(Box::new(a.negate()), Box::new(b.negate())),
@ -275,137 +234,100 @@ impl FacetCondition {
fn between( fn between(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let item_span = item.as_span(); let item_span = item.as_span();
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let lvalue = items.next().unwrap();
let rvalue = items.next().unwrap(); let (lresult, _) = pest_parse(items.next().unwrap());
match ftype { let (rresult, _) = pest_parse(items.next().unwrap());
FacetType::String => {
Err(PestError::<Rule>::new_from_span( let lvalue = lresult?;
ErrorVariant::CustomError { let rvalue = rresult?;
message: "invalid operator on a faceted string".to_string(),
}, Ok(Operator(fid, Between(lvalue, rvalue)))
item_span,
).into())
},
FacetType::Number => {
let lvalue = pest_parse(lvalue)?;
let rvalue = pest_parse(rvalue)?;
Ok(OperatorNumber(fid, Between(lvalue, rvalue)))
},
}
} }
fn equal( fn equal(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let value = items.next().unwrap(); let value = items.next().unwrap();
match ftype { let (result, svalue) = pest_parse(value);
FacetType::String => Ok(OperatorString(fid, FacetStringOperator::equal(value.as_str()))),
FacetType::Number => Ok(OperatorNumber(fid, Equal(pest_parse(value)?))), Ok(Operator(fid, Equal(Some(result?), svalue)))
}
} }
fn greater_than( fn greater_than(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let item_span = item.as_span(); let item_span = item.as_span();
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let value = items.next().unwrap(); let value = items.next().unwrap();
match ftype { let (result, _svalue) = pest_parse(value);
FacetType::String => {
Err(PestError::<Rule>::new_from_span( Ok(Operator(fid, GreaterThan(result?)))
ErrorVariant::CustomError {
message: "invalid operator on a faceted string".to_string(),
},
item_span,
).into())
},
FacetType::Number => Ok(OperatorNumber(fid, GreaterThan(pest_parse(value)?))),
}
} }
fn greater_than_or_equal( fn greater_than_or_equal(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let item_span = item.as_span(); let item_span = item.as_span();
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let value = items.next().unwrap(); let value = items.next().unwrap();
match ftype { let (result, _svalue) = pest_parse(value);
FacetType::String => {
Err(PestError::<Rule>::new_from_span( Ok(Operator(fid, GreaterThanOrEqual(result?)))
ErrorVariant::CustomError {
message: "invalid operator on a faceted string".to_string(),
},
item_span,
).into())
},
FacetType::Number => Ok(OperatorNumber(fid, GreaterThanOrEqual(pest_parse(value)?))),
}
} }
fn lower_than( fn lower_than(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let item_span = item.as_span(); let item_span = item.as_span();
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let value = items.next().unwrap(); let value = items.next().unwrap();
match ftype { let (result, _svalue) = pest_parse(value);
FacetType::String => {
Err(PestError::<Rule>::new_from_span( Ok(Operator(fid, LowerThan(result?)))
ErrorVariant::CustomError {
message: "invalid operator on a faceted string".to_string(),
},
item_span,
).into())
},
FacetType::Number => Ok(OperatorNumber(fid, LowerThan(pest_parse(value)?))),
}
} }
fn lower_than_or_equal( fn lower_than_or_equal(
fields_ids_map: &FieldsIdsMap, fields_ids_map: &FieldsIdsMap,
faceted_fields: &HashMap<FieldId, FacetType>, faceted_fields: &HashSet<FieldId>,
item: Pair<Rule>, item: Pair<Rule>,
) -> anyhow::Result<FacetCondition> ) -> anyhow::Result<FacetCondition>
{ {
let item_span = item.as_span(); let item_span = item.as_span();
let mut items = item.into_inner(); let mut items = item.into_inner();
let (fid, ftype) = get_field_id_facet_type(fields_ids_map, faceted_fields, &mut items)?; let fid = field_id(fields_ids_map, faceted_fields, &mut items)?;
let value = items.next().unwrap(); let value = items.next().unwrap();
match ftype { let (result, _svalue) = pest_parse(value);
FacetType::String => {
Err(PestError::<Rule>::new_from_span( Ok(Operator(fid, LowerThanOrEqual(result?)))
ErrorVariant::CustomError {
message: "invalid operator on a faceted string".to_string(),
},
item_span,
).into())
},
FacetType::Number => Ok(OperatorNumber(fid, LowerThanOrEqual(pest_parse(value)?))),
}
} }
} }
@ -485,34 +407,53 @@ impl FacetCondition {
Ok(()) Ok(())
} }
fn evaluate_number_operator<>( fn evaluate_operator(
rtxn: &heed::RoTxn, rtxn: &heed::RoTxn,
index: &Index, index: &Index,
db: heed::Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>, numbers_db: heed::Database<FacetLevelValueF64Codec, CboRoaringBitmapCodec>,
strings_db: heed::Database<FacetValueStringCodec, CboRoaringBitmapCodec>,
field_id: FieldId, field_id: FieldId,
operator: FacetNumberOperator, operator: &Operator,
) -> anyhow::Result<RoaringBitmap> ) -> anyhow::Result<RoaringBitmap>
{ {
// Make sure we always bound the ranges with the field id and the level, // Make sure we always bound the ranges with the field id and the level,
// as the facets values are all in the same database and prefixed by the // as the facets values are all in the same database and prefixed by the
// field id and the level. // field id and the level.
let (left, right) = match operator { let (left, right) = match operator {
GreaterThan(val) => (Excluded(val), Included(f64::MAX)), GreaterThan(val) => (Excluded(*val), Included(f64::MAX)),
GreaterThanOrEqual(val) => (Included(val), Included(f64::MAX)), GreaterThanOrEqual(val) => (Included(*val), Included(f64::MAX)),
Equal(val) => (Included(val), Included(val)), Equal(number, string) => {
NotEqual(val) => { let string_docids = strings_db.get(rtxn, &(field_id, &string))?.unwrap_or_default();
let all_documents_ids = index.faceted_documents_ids(rtxn, field_id)?; let number_docids = match number {
let docids = Self::evaluate_number_operator(rtxn, index, db, field_id, Equal(val))?; Some(n) => {
return Ok(all_documents_ids - docids); let n = Included(*n);
let mut output = RoaringBitmap::new();
Self::explore_facet_number_levels(rtxn, numbers_db, field_id, 0, n, n, &mut output)?;
output
},
None => RoaringBitmap::new(),
};
return Ok(string_docids | number_docids);
}, },
LowerThan(val) => (Included(f64::MIN), Excluded(val)), NotEqual(number, string) => {
LowerThanOrEqual(val) => (Included(f64::MIN), Included(val)), let all_numbers_ids = if number.is_some() {
Between(left, right) => (Included(left), Included(right)), index.number_faceted_documents_ids(rtxn, field_id)?
} else {
RoaringBitmap::new()
};
let all_strings_ids = index.string_faceted_documents_ids(rtxn, field_id)?;
let operator = Equal(*number, string.clone());
let docids = Self::evaluate_operator(rtxn, index, numbers_db, strings_db, field_id, &operator)?;
return Ok((all_numbers_ids | all_strings_ids) - docids);
},
LowerThan(val) => (Included(f64::MIN), Excluded(*val)),
LowerThanOrEqual(val) => (Included(f64::MIN), Included(*val)),
Between(left, right) => (Included(*left), Included(*right)),
}; };
// Ask for the biggest value that can exist for this specific field, if it exists // Ask for the biggest value that can exist for this specific field, if it exists
// that's fine if it don't, the value just before will be returned instead. // that's fine if it don't, the value just before will be returned instead.
let biggest_level = db let biggest_level = numbers_db
.remap_data_type::<DecodeIgnore>() .remap_data_type::<DecodeIgnore>()
.get_lower_than_or_equal_to(rtxn, &(field_id, u8::MAX, f64::MAX, f64::MAX))? .get_lower_than_or_equal_to(rtxn, &(field_id, u8::MAX, f64::MAX, f64::MAX))?
.and_then(|((id, level, _, _), _)| if id == field_id { Some(level) } else { None }); .and_then(|((id, level, _, _), _)| if id == field_id { Some(level) } else { None });
@ -520,52 +461,25 @@ impl FacetCondition {
match biggest_level { match biggest_level {
Some(level) => { Some(level) => {
let mut output = RoaringBitmap::new(); let mut output = RoaringBitmap::new();
Self::explore_facet_number_levels(rtxn, db, field_id, level, left, right, &mut output)?; Self::explore_facet_number_levels(rtxn, numbers_db, field_id, level, left, right, &mut output)?;
Ok(output) Ok(output)
}, },
None => Ok(RoaringBitmap::new()), None => Ok(RoaringBitmap::new()),
} }
} }
fn evaluate_string_operator(
rtxn: &heed::RoTxn,
index: &Index,
db: heed::Database<FacetValueStringCodec, CboRoaringBitmapCodec>,
field_id: FieldId,
operator: &FacetStringOperator,
) -> anyhow::Result<RoaringBitmap>
{
match operator {
FacetStringOperator::Equal(string) => {
match db.get(rtxn, &(field_id, string))? {
Some(docids) => Ok(docids),
None => Ok(RoaringBitmap::new())
}
},
FacetStringOperator::NotEqual(string) => {
let all_documents_ids = index.faceted_documents_ids(rtxn, field_id)?;
let op = FacetStringOperator::Equal(string.clone());
let docids = Self::evaluate_string_operator(rtxn, index, db, field_id, &op)?;
Ok(all_documents_ids - docids)
},
}
}
pub fn evaluate( pub fn evaluate(
&self, &self,
rtxn: &heed::RoTxn, rtxn: &heed::RoTxn,
index: &Index, index: &Index,
) -> anyhow::Result<RoaringBitmap> ) -> anyhow::Result<RoaringBitmap>
{ {
let db = index.facet_field_id_value_docids; let numbers_db = index.facet_id_f64_docids;
let strings_db = index.facet_id_string_docids;
match self { match self {
OperatorString(fid, op) => { Operator(fid, op) => {
let db = db.remap_key_type::<FacetValueStringCodec>(); Self::evaluate_operator(rtxn, index, numbers_db, strings_db, *fid, op)
Self::evaluate_string_operator(rtxn, index, db, *fid, op)
},
OperatorNumber(fid, op) => {
let db = db.remap_key_type::<FacetLevelValueF64Codec>();
Self::evaluate_number_operator(rtxn, index, db, *fid, *op)
}, },
Or(lhs, rhs) => { Or(lhs, rhs) => {
let lhs = lhs.evaluate(rtxn, index)?; let lhs = lhs.evaluate(rtxn, index)?;

2
milli/src/search/facet/mod.rs
View File

@ -9,7 +9,7 @@ use crate::heed_codec::CboRoaringBitmapCodec;
use crate::heed_codec::facet::FacetLevelValueF64Codec; use crate::heed_codec::facet::FacetLevelValueF64Codec;
use crate::{Index, FieldId}; use crate::{Index, FieldId};
pub use self::facet_condition::{FacetCondition, FacetNumberOperator, FacetStringOperator}; pub use self::facet_condition::{FacetCondition, Operator};
pub use self::facet_distribution::FacetDistribution; pub use self::facet_distribution::FacetDistribution;
mod facet_condition; mod facet_condition;

4
milli/src/search/mod.rs
View File

@ -16,9 +16,7 @@ use distinct::{Distinct, DocIter, FacetDistinct, MapDistinct, NoopDistinct};
use crate::search::criteria::r#final::{Final, FinalResult}; use crate::search::criteria::r#final::{Final, FinalResult};
use crate::{Index, DocumentId}; use crate::{Index, DocumentId};
pub use self::facet::{ pub use self::facet::{FacetCondition, FacetDistribution, FacetIter, Operator};
FacetCondition, FacetDistribution, FacetIter, FacetNumberOperator, FacetStringOperator,
};
pub use self::query_tree::MatchingWords; pub use self::query_tree::MatchingWords;
use self::query_tree::QueryTreeBuilder; use self::query_tree::QueryTreeBuilder;