mirror of
https://github.com/meilisearch/MeiliSearch
synced 2025-07-03 11:57:07 +02:00
setting up facets
This commit is contained in:
parent
dd08cfc6a3
commit
59c67f6bc8
11 changed files with 633 additions and 139 deletions
|
@ -11,8 +11,10 @@ bincode = "1.2.1"
|
|||
byteorder = "1.3.4"
|
||||
chrono = { version = "0.4.11", features = ["serde"] }
|
||||
compact_arena = "0.4.0"
|
||||
cow-utils = "0.1.2"
|
||||
crossbeam-channel = "0.4.2"
|
||||
deunicode = "1.1.0"
|
||||
either = "1.5.3"
|
||||
env_logger = "0.7.1"
|
||||
fst = { version = "0.3.5", default-features = false }
|
||||
hashbrown = { version = "0.7.1", features = ["serde"] }
|
||||
|
|
|
@ -28,7 +28,8 @@ pub enum Error {
|
|||
Serializer(SerializerError),
|
||||
Deserializer(DeserializerError),
|
||||
UnsupportedOperation(UnsupportedOperation),
|
||||
FilterParseError(PestError<Rule>)
|
||||
FilterParseError(PestError<Rule>),
|
||||
FacetError(FacetError),
|
||||
}
|
||||
|
||||
impl From<io::Error> for Error {
|
||||
|
@ -57,7 +58,13 @@ impl From<PestError<Rule>> for Error {
|
|||
s.to_string()
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FacetError> for Error {
|
||||
fn from(error: FacetError) -> Error {
|
||||
Error::FacetError(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<meilisearch_schema::Error> for Error {
|
||||
fn from(error: meilisearch_schema::Error) -> Error {
|
||||
|
@ -127,6 +134,7 @@ impl fmt::Display for Error {
|
|||
Deserializer(e) => write!(f, "deserializer error; {}", e),
|
||||
UnsupportedOperation(op) => write!(f, "unsupported operation; {}", op),
|
||||
FilterParseError(e) => write!(f, "error parsing filter; {}", e),
|
||||
FacetError(e) => write!(f, "error processing facet filter: {}", e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -156,3 +164,40 @@ impl fmt::Display for UnsupportedOperation {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum FacetError {
|
||||
EmptyArray,
|
||||
ParsingError(String),
|
||||
UnexpectedToken { expected: &'static [&'static str], found: String },
|
||||
InvalidFormat(String),
|
||||
AttributeNotFound(String),
|
||||
AttributeNotSet { expected: Vec<String>, found: String },
|
||||
InvalidDocumentAttribute(String),
|
||||
}
|
||||
|
||||
impl FacetError {
|
||||
pub fn unexpected_token(expected: &'static [&'static str], found: impl ToString) -> FacetError {
|
||||
FacetError::UnexpectedToken{ expected, found: found.to_string() }
|
||||
}
|
||||
|
||||
pub fn attribute_not_set(expected: Vec<String>, found: impl ToString) -> FacetError {
|
||||
FacetError::AttributeNotSet{ expected, found: found.to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FacetError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
use FacetError::*;
|
||||
|
||||
match self {
|
||||
EmptyArray => write!(f, "empty array in facet filter is unspecified behavior"),
|
||||
ParsingError(msg) => write!(f, "parsing error: {}", msg),
|
||||
UnexpectedToken { expected, found } => write!(f, "unexpected token {}, expected {}", found, expected.join("or")),
|
||||
InvalidFormat(found) => write!(f, "invalid facet: {}, facets should be \"facetName:facetValue\"", found),
|
||||
AttributeNotFound(attr) => write!(f, "unknown {:?} attribute", attr),
|
||||
AttributeNotSet { found, expected } => write!(f, "`{}` is not set as a faceted attribute. available facet attributes: {}", found, expected.join(", ")),
|
||||
InvalidDocumentAttribute(attr) => write!(f, "invalid document attribute {}, accepted types: string and [string]", attr),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
353
meilisearch-core/src/facets.rs
Normal file
353
meilisearch-core/src/facets.rs
Normal file
|
@ -0,0 +1,353 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::hash::Hash;
|
||||
use std::ops::Deref;
|
||||
|
||||
use cow_utils::CowUtils;
|
||||
use either::Either;
|
||||
use heed::types::{Str, OwnedType};
|
||||
use indexmap::IndexMap;
|
||||
use serde_json::Value;
|
||||
|
||||
use meilisearch_schema::{FieldId, Schema};
|
||||
use meilisearch_types::DocumentId;
|
||||
|
||||
use crate::database::MainT;
|
||||
use crate::error::{FacetError, Error};
|
||||
use crate::store::BEU16;
|
||||
|
||||
/// Data structure used to represent a boolean expression in the form of nested arrays.
|
||||
/// Values in the outer array are and-ed together, values in the inner arrays are or-ed together.
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct FacetFilter(Vec<Either<Vec<FacetKey>, FacetKey>>);
|
||||
|
||||
impl Deref for FacetFilter {
|
||||
type Target = Vec<Either<Vec<FacetKey>, FacetKey>>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl FacetFilter {
|
||||
pub fn from_str(
|
||||
s: &str,
|
||||
schema: &Schema,
|
||||
attributes_for_faceting: &[FieldId],
|
||||
) -> Result<Self, FacetError> {
|
||||
|
||||
let parsed = serde_json::from_str::<Value>(s).map_err(|e| FacetError::ParsingError(e.to_string()))?;
|
||||
let mut filter = Vec::new();
|
||||
match parsed {
|
||||
Value::Array(and_exprs) => {
|
||||
if and_exprs.is_empty() {
|
||||
return Err(FacetError::EmptyArray);
|
||||
}
|
||||
for expr in and_exprs {
|
||||
match expr {
|
||||
Value::String(s) => {
|
||||
let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
|
||||
filter.push(Either::Right(key));
|
||||
}
|
||||
Value::Array(or_exprs) => {
|
||||
if or_exprs.is_empty() {
|
||||
return Err(FacetError::EmptyArray);
|
||||
}
|
||||
let mut inner = Vec::new();
|
||||
for expr in or_exprs {
|
||||
match expr {
|
||||
Value::String(s) => {
|
||||
let key = FacetKey::from_str( &s, schema, attributes_for_faceting)?;
|
||||
inner.push(key);
|
||||
}
|
||||
bad_value => return Err(FacetError::unexpected_token(&["String"], bad_value)),
|
||||
}
|
||||
}
|
||||
filter.push(Either::Left(inner));
|
||||
}
|
||||
bad_value => return Err(FacetError::unexpected_token(&["Array", "String"], bad_value)),
|
||||
}
|
||||
}
|
||||
return Ok(Self(filter));
|
||||
}
|
||||
bad_value => Err(FacetError::unexpected_token(&["String"], bad_value)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq, Hash)]
|
||||
#[repr(C)]
|
||||
pub struct FacetKey(FieldId, String);
|
||||
|
||||
impl FacetKey {
|
||||
pub fn new(field_id: FieldId, value: String) -> Self {
|
||||
let value = match value.cow_to_lowercase() {
|
||||
Cow::Borrowed(_) => value,
|
||||
Cow::Owned(s) => s,
|
||||
};
|
||||
Self(field_id, value)
|
||||
}
|
||||
|
||||
pub fn key(&self) -> FieldId {
|
||||
self.0
|
||||
}
|
||||
|
||||
pub fn value(&self) -> &str {
|
||||
&self.1
|
||||
}
|
||||
|
||||
// TODO improve parser
|
||||
fn from_str(
|
||||
s: &str,
|
||||
schema: &Schema,
|
||||
attributes_for_faceting: &[FieldId],
|
||||
) -> Result<Self, FacetError> {
|
||||
let mut split = s.splitn(2, ':');
|
||||
let key = split
|
||||
.next()
|
||||
.ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
|
||||
.trim();
|
||||
let field_id = schema
|
||||
.id(key)
|
||||
.ok_or_else(|| FacetError::AttributeNotFound(key.to_string()))?;
|
||||
|
||||
if !attributes_for_faceting.contains(&field_id) {
|
||||
return Err(FacetError::attribute_not_set(
|
||||
attributes_for_faceting
|
||||
.iter()
|
||||
.filter_map(|&id| schema.name(id))
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>(),
|
||||
key))
|
||||
}
|
||||
let value = split
|
||||
.next()
|
||||
.ok_or_else(|| FacetError::InvalidFormat(s.to_string()))?
|
||||
.trim();
|
||||
// unquoting the string if need be:
|
||||
let mut indices = value.char_indices();
|
||||
let value = match (indices.next(), indices.last()) {
|
||||
(Some((s, '\'')), Some((e, '\''))) |
|
||||
(Some((s, '\"')), Some((e, '\"'))) => value[s + 1..e].to_string(),
|
||||
_ => value.to_string(),
|
||||
};
|
||||
Ok(Self::new(field_id, value))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> heed::BytesEncode<'a> for FacetKey {
|
||||
type EItem = FacetKey;
|
||||
|
||||
fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
|
||||
let mut buffer = Vec::with_capacity(2 + item.1.len());
|
||||
let id = BEU16::new(item.key().into());
|
||||
let id_bytes = OwnedType::bytes_encode(&id)?;
|
||||
let value_bytes = Str::bytes_encode(item.value())?;
|
||||
buffer.extend_from_slice(id_bytes.as_ref());
|
||||
buffer.extend_from_slice(value_bytes.as_ref());
|
||||
Some(Cow::Owned(buffer))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> heed::BytesDecode<'a> for FacetKey {
|
||||
type DItem = FacetKey;
|
||||
|
||||
fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
|
||||
let (id_bytes, value_bytes) = bytes.split_at(2);
|
||||
let id = OwnedType::<BEU16>::bytes_decode(id_bytes)?;
|
||||
let id = id.get().into();
|
||||
let string = Str::bytes_decode(&value_bytes)?;
|
||||
Some(FacetKey(id, string.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_to_facet_map(
|
||||
facet_map: &mut HashMap<FacetKey, Vec<DocumentId>>,
|
||||
field_id: FieldId,
|
||||
value: Value,
|
||||
document_id: DocumentId,
|
||||
) -> Result<(), FacetError> {
|
||||
let value = match value {
|
||||
Value::String(s) => s,
|
||||
// ignore null
|
||||
Value::Null => return Ok(()),
|
||||
value => return Err(FacetError::InvalidDocumentAttribute(value.to_string())),
|
||||
};
|
||||
let key = FacetKey::new(field_id, value);
|
||||
facet_map.entry(key).or_insert_with(Vec::new).push(document_id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn facet_map_from_docids(
|
||||
rtxn: &heed::RoTxn<MainT>,
|
||||
index: &crate::Index,
|
||||
document_ids: &[DocumentId],
|
||||
attributes_for_facetting: &[FieldId],
|
||||
) -> Result<HashMap<FacetKey, Vec<DocumentId>>, Error> {
|
||||
let mut facet_map = HashMap::new();
|
||||
for document_id in document_ids {
|
||||
for result in index
|
||||
.documents_fields
|
||||
.document_fields(rtxn, *document_id)?
|
||||
{
|
||||
let (field_id, bytes) = result?;
|
||||
if attributes_for_facetting.contains(&field_id) {
|
||||
match serde_json::from_slice(bytes)? {
|
||||
Value::Array(values) => {
|
||||
for v in values {
|
||||
add_to_facet_map(&mut facet_map, field_id, v, *document_id)?;
|
||||
}
|
||||
}
|
||||
v => add_to_facet_map(&mut facet_map, field_id, v, *document_id)?,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(facet_map)
|
||||
}
|
||||
|
||||
pub fn facet_map_from_docs(
|
||||
schema: &Schema,
|
||||
documents: &HashMap<DocumentId, IndexMap<String, Value>>,
|
||||
attributes_for_facetting: &[FieldId],
|
||||
) -> Result<HashMap<FacetKey, Vec<DocumentId>>, Error> {
|
||||
let mut facet_map = HashMap::new();
|
||||
let attributes_for_facetting = attributes_for_facetting
|
||||
.iter()
|
||||
.filter_map(|&id| schema.name(id).map(|name| (id, name)))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for (id, document) in documents {
|
||||
for (field_id, name) in &attributes_for_facetting {
|
||||
if let Some(value) = document.get(*name) {
|
||||
match value {
|
||||
Value::Array(values) => {
|
||||
for v in values {
|
||||
add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?;
|
||||
}
|
||||
}
|
||||
v => add_to_facet_map(&mut facet_map, *field_id, v.clone(), *id)?,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(facet_map)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use meilisearch_schema::Schema;
|
||||
|
||||
#[test]
|
||||
fn test_facet_key() {
|
||||
let mut schema = Schema::new();
|
||||
let id = schema.insert_and_index("hello").unwrap();
|
||||
let facet_list = [schema.id("hello").unwrap()];
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:12", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "12".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:\"foo bar\"", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "foo bar".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:'foo bar'", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "foo bar".to_string())
|
||||
);
|
||||
// weird case
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:blabla:machin", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "blabla:machin".to_string())
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:\"\"", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "".to_string())
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:'", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "'".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
FacetKey::from_str("hello:''", &schema, &facet_list).unwrap(),
|
||||
FacetKey::new(id, "".to_string())
|
||||
);
|
||||
assert!(FacetKey::from_str("hello", &schema, &facet_list).is_err());
|
||||
assert!(FacetKey::from_str("toto:12", &schema, &facet_list).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_facet_array() {
|
||||
use either::Either::{Left, Right};
|
||||
let mut schema = Schema::new();
|
||||
let _id = schema.insert_and_index("hello").unwrap();
|
||||
let facet_list = [schema.id("hello").unwrap()];
|
||||
assert_eq!(
|
||||
FacetFilter::from_str("[[\"hello:12\"]]", &schema, &facet_list).unwrap(),
|
||||
FacetFilter(vec![Left(vec![FacetKey(FieldId(0), "12".to_string())])])
|
||||
);
|
||||
assert_eq!(
|
||||
FacetFilter::from_str("[\"hello:12\"]", &schema, &facet_list).unwrap(),
|
||||
FacetFilter(vec![Right(FacetKey(FieldId(0), "12".to_string()))])
|
||||
);
|
||||
assert_eq!(
|
||||
FacetFilter::from_str("[\"hello:12\", \"hello:13\"]", &schema, &facet_list).unwrap(),
|
||||
FacetFilter(vec![
|
||||
Right(FacetKey(FieldId(0), "12".to_string())),
|
||||
Right(FacetKey(FieldId(0), "13".to_string()))
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
FacetFilter::from_str("[[\"hello:12\", \"hello:13\"]]", &schema, &facet_list).unwrap(),
|
||||
FacetFilter(vec![Left(vec![
|
||||
FacetKey(FieldId(0), "12".to_string()),
|
||||
FacetKey(FieldId(0), "13".to_string())
|
||||
])])
|
||||
);
|
||||
assert_eq!(
|
||||
FacetFilter::from_str(
|
||||
"[[\"hello:12\", \"hello:13\"], \"hello:14\"]",
|
||||
&schema,
|
||||
&facet_list
|
||||
)
|
||||
.unwrap(),
|
||||
FacetFilter(vec![
|
||||
Left(vec![
|
||||
FacetKey(FieldId(0), "12".to_string()),
|
||||
FacetKey(FieldId(0), "13".to_string())
|
||||
]),
|
||||
Right(FacetKey(FieldId(0), "14".to_string()))
|
||||
])
|
||||
);
|
||||
|
||||
// invalid array depths
|
||||
assert!(FacetFilter::from_str(
|
||||
"[[[\"hello:12\", \"hello:13\"], \"hello:14\"]]",
|
||||
&schema,
|
||||
&facet_list
|
||||
)
|
||||
.is_err());
|
||||
assert!(FacetFilter::from_str(
|
||||
"[[[\"hello:12\", \"hello:13\"]], \"hello:14\"]]",
|
||||
&schema,
|
||||
&facet_list
|
||||
)
|
||||
.is_err());
|
||||
assert!(FacetFilter::from_str("\"hello:14\"", &schema, &facet_list).is_err());
|
||||
|
||||
// unexisting key
|
||||
assert!(FacetFilter::from_str("[\"foo:12\"]", &schema, &facet_list).is_err());
|
||||
|
||||
// invalid facet key
|
||||
assert!(FacetFilter::from_str("[\"foo=12\"]", &schema, &facet_list).is_err());
|
||||
assert!(FacetFilter::from_str("[\"foo12\"]", &schema, &facet_list).is_err());
|
||||
assert!(FacetFilter::from_str("[\"\"]", &schema, &facet_list).is_err());
|
||||
|
||||
// empty array error
|
||||
assert!(FacetFilter::from_str("[]", &schema, &facet_list).is_err());
|
||||
assert!(FacetFilter::from_str("[\"hello:12\", []]", &schema, &facet_list).is_err());
|
||||
}
|
||||
}
|
|
@ -19,14 +19,15 @@ mod ranked_map;
|
|||
mod raw_document;
|
||||
mod reordered_attrs;
|
||||
mod update;
|
||||
pub mod settings;
|
||||
pub mod criterion;
|
||||
pub mod facets;
|
||||
pub mod raw_indexer;
|
||||
pub mod settings;
|
||||
pub mod serde;
|
||||
pub mod store;
|
||||
|
||||
pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT};
|
||||
pub use self::error::{Error, HeedError, FstError, MResult, pest_error};
|
||||
pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError};
|
||||
pub use self::filters::Filter;
|
||||
pub use self::number::{Number, ParseNumberError};
|
||||
pub use self::ranked_map::RankedMap;
|
||||
|
|
32
meilisearch-core/src/store/cow_set.rs
Normal file
32
meilisearch-core/src/store/cow_set.rs
Normal file
|
@ -0,0 +1,32 @@
|
|||
use std::borrow::Cow;
|
||||
|
||||
use heed::{types::CowSlice, BytesEncode, BytesDecode};
|
||||
use sdset::{Set, SetBuf};
|
||||
use zerocopy::{AsBytes, FromBytes};
|
||||
|
||||
pub struct CowSet<T>(std::marker::PhantomData<T>);
|
||||
|
||||
impl<'a, T: 'a> BytesEncode<'a> for CowSet<T>
|
||||
where
|
||||
T: AsBytes,
|
||||
{
|
||||
type EItem = Set<T>;
|
||||
|
||||
fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<[u8]>> {
|
||||
CowSlice::bytes_encode(item.as_slice())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, T: 'a> BytesDecode<'a> for CowSet<T>
|
||||
where
|
||||
T: FromBytes + Copy,
|
||||
{
|
||||
type DItem = Cow<'a, Set<T>>;
|
||||
|
||||
fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
|
||||
match CowSlice::<T>::bytes_decode(bytes)? {
|
||||
Cow::Owned(vec) => Some(Cow::Owned(SetBuf::new_unchecked(vec))),
|
||||
Cow::Borrowed(slice) => Some(Cow::Borrowed(Set::new_unchecked(slice))),
|
||||
}
|
||||
}
|
||||
}
|
53
meilisearch-core/src/store/facets.rs
Normal file
53
meilisearch-core/src/store/facets.rs
Normal file
|
@ -0,0 +1,53 @@
|
|||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use heed::{RwTxn, RoTxn, Result as ZResult};
|
||||
use sdset::{SetBuf, Set, SetOperation};
|
||||
|
||||
use meilisearch_types::DocumentId;
|
||||
|
||||
use crate::database::MainT;
|
||||
use crate::facets::FacetKey;
|
||||
use super::cow_set::CowSet;
|
||||
|
||||
/// contains facet info
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Facets {
|
||||
pub(crate) facets: heed::Database<FacetKey, CowSet<DocumentId>>,
|
||||
}
|
||||
|
||||
impl Facets {
|
||||
// we use sdset::SetBuf to ensure the docids are sorted.
|
||||
pub fn put_facet_document_ids(&self, writer: &mut RwTxn<MainT>, facet_key: FacetKey, doc_ids: &Set<DocumentId>) -> ZResult<()> {
|
||||
self.facets.put(writer, &facet_key, doc_ids)
|
||||
}
|
||||
|
||||
pub fn facet_document_ids<'txn>(&self, reader: &'txn RoTxn<MainT>, facet_key: &FacetKey) -> ZResult<Option<Cow<'txn, Set<DocumentId>>>> {
|
||||
self.facets.get(reader, &facet_key)
|
||||
}
|
||||
|
||||
/// updates the facets store, revmoving the documents from the facets provided in the
|
||||
/// `facet_map` argument
|
||||
pub fn remove(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, Vec<DocumentId>>) -> ZResult<()> {
|
||||
for (key, document_ids) in facet_map {
|
||||
if let Some(old) = self.facets.get(writer, &key)? {
|
||||
let to_remove = SetBuf::from_dirty(document_ids);
|
||||
let new = sdset::duo::OpBuilder::new(old.as_ref(), to_remove.as_set()).difference().into_set_buf();
|
||||
self.facets.put(writer, &key, new.as_set())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add(&self, writer: &mut RwTxn<MainT>, facet_map: HashMap<FacetKey, Vec<DocumentId>>) -> ZResult<()> {
|
||||
for (key, document_ids) in facet_map {
|
||||
let set = SetBuf::from_dirty(document_ids);
|
||||
self.put_facet_document_ids(writer, key, set.as_set())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<()> {
|
||||
self.facets.clear(writer)
|
||||
}
|
||||
}
|
|
@ -4,13 +4,16 @@ use std::collections::HashMap;
|
|||
use chrono::{DateTime, Utc};
|
||||
use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str};
|
||||
use heed::Result as ZResult;
|
||||
use meilisearch_schema::Schema;
|
||||
use meilisearch_schema::{FieldId, Schema};
|
||||
use sdset::Set;
|
||||
|
||||
use crate::database::MainT;
|
||||
use crate::RankedMap;
|
||||
use crate::settings::RankingRule;
|
||||
use super::cow_set::CowSet;
|
||||
|
||||
const CREATED_AT_KEY: &str = "created-at";
|
||||
const ATTRIBUTES_FOR_FACETING: &str = "attributes-for-faceting";
|
||||
const RANKING_RULES_KEY: &str = "ranking-rules";
|
||||
const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute";
|
||||
const STOP_WORDS_KEY: &str = "stop-words";
|
||||
|
@ -188,6 +191,18 @@ impl Main {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn attributes_for_faceting<'txn>(&self, reader: &'txn heed::RoTxn<MainT>) -> ZResult<Option<Cow<'txn, Set<FieldId>>>> {
|
||||
self.main.get::<_, Str, CowSet<FieldId>>(reader, ATTRIBUTES_FOR_FACETING)
|
||||
}
|
||||
|
||||
pub fn put_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>, attributes: &Set<FieldId>) -> ZResult<()> {
|
||||
self.main.put::<_, Str, CowSet<FieldId>>(writer, ATTRIBUTES_FOR_FACETING, attributes)
|
||||
}
|
||||
|
||||
pub fn delete_attributes_for_faceting(self, writer: &mut heed::RwTxn<MainT>) -> ZResult<bool> {
|
||||
self.main.delete::<_, Str>(writer, ATTRIBUTES_FOR_FACETING)
|
||||
}
|
||||
|
||||
pub fn ranking_rules(&self, reader: &heed::RoTxn<MainT>) -> ZResult<Option<Vec<RankingRule>>> {
|
||||
self.main.get::<_, Str, SerdeBincode<Vec<RankingRule>>>(reader, RANKING_RULES_KEY)
|
||||
}
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
mod cow_set;
|
||||
mod docs_words;
|
||||
mod prefix_documents_cache;
|
||||
mod prefix_postings_lists_cache;
|
||||
|
@ -8,8 +9,10 @@ mod postings_lists;
|
|||
mod synonyms;
|
||||
mod updates;
|
||||
mod updates_results;
|
||||
mod facets;
|
||||
|
||||
pub use self::docs_words::DocsWords;
|
||||
pub use self::facets::Facets;
|
||||
pub use self::prefix_documents_cache::PrefixDocumentsCache;
|
||||
pub use self::prefix_postings_lists_cache::PrefixPostingsListsCache;
|
||||
pub use self::documents_fields::{DocumentFieldsIter, DocumentsFields};
|
||||
|
@ -42,7 +45,7 @@ use crate::settings::SettingsUpdate;
|
|||
use crate::{query_builder::QueryBuilder, update, DocIndex, DocumentId, Error, MResult};
|
||||
|
||||
type BEU64 = zerocopy::U64<byteorder::BigEndian>;
|
||||
type BEU16 = zerocopy::U16<byteorder::BigEndian>;
|
||||
pub type BEU16 = zerocopy::U16<byteorder::BigEndian>;
|
||||
|
||||
#[derive(Debug, Copy, Clone, AsBytes, FromBytes)]
|
||||
#[repr(C)]
|
||||
|
@ -197,12 +200,17 @@ fn updates_results_name(name: &str) -> String {
|
|||
format!("store-{}-updates-results", name)
|
||||
}
|
||||
|
||||
fn facets_name(name: &str) -> String {
|
||||
format!("store-{}-facets", name)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Index {
|
||||
pub main: Main,
|
||||
pub postings_lists: PostingsLists,
|
||||
pub documents_fields: DocumentsFields,
|
||||
pub documents_fields_counts: DocumentsFieldsCounts,
|
||||
pub facets: Facets,
|
||||
pub synonyms: Synonyms,
|
||||
pub docs_words: DocsWords,
|
||||
pub prefix_documents_cache: PrefixDocumentsCache,
|
||||
|
@ -352,29 +360,14 @@ impl Index {
|
|||
}
|
||||
|
||||
pub fn query_builder(&self) -> QueryBuilder {
|
||||
QueryBuilder::new(
|
||||
self.main,
|
||||
self.postings_lists,
|
||||
self.documents_fields_counts,
|
||||
self.synonyms,
|
||||
self.prefix_documents_cache,
|
||||
self.prefix_postings_lists_cache,
|
||||
)
|
||||
QueryBuilder::new(self)
|
||||
}
|
||||
|
||||
pub fn query_builder_with_criteria<'c, 'f, 'd>(
|
||||
&self,
|
||||
pub fn query_builder_with_criteria<'c, 'f, 'd, 'fa, 'i>(
|
||||
&'i self,
|
||||
criteria: Criteria<'c>,
|
||||
) -> QueryBuilder<'c, 'f, 'd> {
|
||||
QueryBuilder::with_criteria(
|
||||
self.main,
|
||||
self.postings_lists,
|
||||
self.documents_fields_counts,
|
||||
self.synonyms,
|
||||
self.prefix_documents_cache,
|
||||
self.prefix_postings_lists_cache,
|
||||
criteria,
|
||||
)
|
||||
) -> QueryBuilder<'c, 'f, 'd, 'fa, 'i> {
|
||||
QueryBuilder::with_criteria(self, criteria)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -395,12 +388,14 @@ pub fn create(
|
|||
let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
|
||||
let updates_name = updates_name(name);
|
||||
let updates_results_name = updates_results_name(name);
|
||||
let facets_name = facets_name(name);
|
||||
|
||||
// open all the stores
|
||||
let main = env.create_poly_database(Some(&main_name))?;
|
||||
let postings_lists = env.create_database(Some(&postings_lists_name))?;
|
||||
let documents_fields = env.create_database(Some(&documents_fields_name))?;
|
||||
let documents_fields_counts = env.create_database(Some(&documents_fields_counts_name))?;
|
||||
let facets = env.create_database(Some(&facets_name))?;
|
||||
let synonyms = env.create_database(Some(&synonyms_name))?;
|
||||
let docs_words = env.create_database(Some(&docs_words_name))?;
|
||||
let prefix_documents_cache = env.create_database(Some(&prefix_documents_cache_name))?;
|
||||
|
@ -417,6 +412,8 @@ pub fn create(
|
|||
docs_words: DocsWords { docs_words },
|
||||
prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
|
||||
prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
|
||||
facets: Facets { facets },
|
||||
|
||||
updates: Updates { updates },
|
||||
updates_results: UpdatesResults { updates_results },
|
||||
updates_notifier,
|
||||
|
@ -437,6 +434,7 @@ pub fn open(
|
|||
let synonyms_name = synonyms_name(name);
|
||||
let docs_words_name = docs_words_name(name);
|
||||
let prefix_documents_cache_name = prefix_documents_cache_name(name);
|
||||
let facets_name = facets_name(name);
|
||||
let prefix_postings_lists_cache_name = prefix_postings_lists_cache_name(name);
|
||||
let updates_name = updates_name(name);
|
||||
let updates_results_name = updates_results_name(name);
|
||||
|
@ -470,6 +468,10 @@ pub fn open(
|
|||
Some(prefix_documents_cache) => prefix_documents_cache,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let facets = match env.open_database(Some(&facets_name))? {
|
||||
Some(facets) => facets,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let prefix_postings_lists_cache = match env.open_database(Some(&prefix_postings_lists_cache_name))? {
|
||||
Some(prefix_postings_lists_cache) => prefix_postings_lists_cache,
|
||||
None => return Ok(None),
|
||||
|
@ -491,6 +493,7 @@ pub fn open(
|
|||
synonyms: Synonyms { synonyms },
|
||||
docs_words: DocsWords { docs_words },
|
||||
prefix_documents_cache: PrefixDocumentsCache { prefix_documents_cache },
|
||||
facets: Facets { facets },
|
||||
prefix_postings_lists_cache: PrefixPostingsListsCache { prefix_postings_lists_cache },
|
||||
updates: Updates { updates },
|
||||
updates_results: UpdatesResults { updates_results },
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue