MeiliSearch/meilisearch-core/src/lib.rs

203 lines
5.9 KiB
Rust
Raw Normal View History

2020-06-27 15:10:39 +02:00
#![allow(clippy::type_complexity)]
#[cfg(test)]
2019-10-18 13:05:28 +02:00
#[macro_use]
extern crate assert_matches;
2020-04-06 20:05:02 +02:00
#[macro_use]
extern crate pest_derive;
mod automaton;
2019-12-13 11:49:56 +01:00
mod bucket_sort;
mod database;
mod distinct_map;
2019-10-03 15:04:11 +02:00
mod error;
2020-04-06 20:05:02 +02:00
mod filters;
2019-10-30 17:25:42 +01:00
mod levenshtein;
2019-10-03 11:49:13 +02:00
mod number;
mod query_builder;
mod query_tree;
mod query_words_mapper;
2019-10-03 15:04:11 +02:00
mod ranked_map;
mod raw_document;
2019-10-18 13:05:28 +02:00
mod reordered_attrs;
2019-12-13 11:49:56 +01:00
pub mod criterion;
2020-05-05 22:19:34 +02:00
pub mod facets;
2019-12-13 11:49:56 +01:00
pub mod raw_indexer;
2019-10-03 11:49:13 +02:00
pub mod serde;
pub mod settings;
pub mod store;
pub mod update;
2020-05-27 12:04:35 +02:00
pub use self::database::{BoxUpdateFn, Database, DatabaseOptions, MainT, UpdateT, MainWriter, MainReader, UpdateWriter, UpdateReader};
2020-05-05 22:19:34 +02:00
pub use self::error::{Error, HeedError, FstError, MResult, pest_error, FacetError};
2020-04-06 20:05:02 +02:00
pub use self::filters::Filter;
2019-10-04 13:26:33 +02:00
pub use self::number::{Number, ParseNumberError};
pub use self::ranked_map::RankedMap;
pub use self::raw_document::RawDocument;
2019-10-04 13:26:33 +02:00
pub use self::store::Index;
pub use self::update::{EnqueuedUpdateResult, ProcessedUpdateResult, UpdateStatus, UpdateType};
pub use meilisearch_types::{DocIndex, DocumentId, Highlight};
2020-02-02 22:59:19 +01:00
pub use meilisearch_schema::Schema;
pub use query_words_mapper::QueryWordsMapper;
use compact_arena::SmallArena;
2020-02-02 22:59:19 +01:00
use log::{error, trace};
2020-05-22 15:00:50 +02:00
use std::borrow::Cow;
use std::collections::HashMap;
use std::convert::TryFrom;
2020-01-16 14:56:16 +01:00
2020-01-16 14:24:45 +01:00
use crate::bucket_sort::PostingsListView;
use crate::levenshtein::prefix_damerau_levenshtein;
2020-01-16 14:56:16 +01:00
use crate::query_tree::{QueryId, QueryKind};
use crate::reordered_attrs::ReorderedAttrs;
2020-05-22 15:00:50 +02:00
type FstSetCow<'a> = fst::Set<Cow<'a, [u8]>>;
type FstMapCow<'a> = fst::Map<Cow<'a, [u8]>>;
2019-12-13 11:14:12 +01:00
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct Document {
pub id: DocumentId,
pub highlights: Vec<Highlight>,
2019-12-13 11:14:12 +01:00
#[cfg(test)]
pub matches: Vec<crate::bucket_sort::SimpleMatch>,
}
fn highlights_from_raw_document<'a, 'tag, 'txn>(
raw_document: &RawDocument<'a, 'tag>,
2020-01-16 14:56:16 +01:00
queries_kinds: &HashMap<QueryId, &QueryKind>,
arena: &SmallArena<'tag, PostingsListView<'txn>>,
searchable_attrs: Option<&ReorderedAttrs>,
schema: &Schema,
) -> Vec<Highlight>
{
let mut highlights = Vec::new();
for bm in raw_document.bare_matches.iter() {
let postings_list = &arena[bm.postings_list];
let input = postings_list.input();
2020-01-16 14:56:16 +01:00
let kind = &queries_kinds.get(&bm.query_index);
for di in postings_list.iter() {
2020-01-16 14:56:16 +01:00
let covered_area = match kind {
Some(QueryKind::NonTolerant(query)) | Some(QueryKind::Tolerant(query)) => {
2020-01-16 14:56:16 +01:00
let len = if query.len() > input.len() {
input.len()
} else {
prefix_damerau_levenshtein(query.as_bytes(), input).1
};
u16::try_from(len).unwrap_or(u16::max_value())
},
_ => di.char_length,
};
let attribute = searchable_attrs
.and_then(|sa| sa.reverse(di.attribute))
.unwrap_or(di.attribute);
let attribute = match schema.indexed_pos_to_field_id(attribute) {
Some(field_id) => field_id.0,
None => {
error!("Cannot convert indexed_pos {} to field_id", attribute);
2020-01-29 18:30:21 +01:00
trace!("Schema is compromized; {:?}", schema);
continue
}
};
let highlight = Highlight {
2020-02-02 22:59:19 +01:00
attribute,
char_index: di.char_index,
2020-01-16 14:56:16 +01:00
char_length: covered_area,
};
highlights.push(highlight);
}
}
highlights
}
impl Document {
#[cfg(not(test))]
pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
Document { id, highlights: highlights.to_owned() }
}
#[cfg(test)]
pub fn from_highlights(id: DocumentId, highlights: &[Highlight]) -> Document {
Document { id, highlights: highlights.to_owned(), matches: Vec::new() }
}
#[cfg(not(test))]
pub fn from_raw<'a, 'tag, 'txn>(
raw_document: RawDocument<'a, 'tag>,
2020-01-16 14:56:16 +01:00
queries_kinds: &HashMap<QueryId, &QueryKind>,
arena: &SmallArena<'tag, PostingsListView<'txn>>,
searchable_attrs: Option<&ReorderedAttrs>,
schema: &Schema,
) -> Document
{
2020-01-16 14:24:45 +01:00
let highlights = highlights_from_raw_document(
&raw_document,
2020-01-16 14:56:16 +01:00
queries_kinds,
2020-01-16 14:24:45 +01:00
arena,
searchable_attrs,
schema,
2020-01-16 14:24:45 +01:00
);
Document { id: raw_document.id, highlights }
}
#[cfg(test)]
pub fn from_raw<'a, 'tag, 'txn>(
raw_document: RawDocument<'a, 'tag>,
2020-01-16 14:56:16 +01:00
queries_kinds: &HashMap<QueryId, &QueryKind>,
arena: &SmallArena<'tag, PostingsListView<'txn>>,
searchable_attrs: Option<&ReorderedAttrs>,
schema: &Schema,
) -> Document
{
use crate::bucket_sort::SimpleMatch;
2019-12-13 11:14:12 +01:00
2020-01-16 14:24:45 +01:00
let highlights = highlights_from_raw_document(
&raw_document,
2020-01-16 14:56:16 +01:00
queries_kinds,
2020-01-16 14:24:45 +01:00
arena,
searchable_attrs,
schema,
2020-01-16 14:24:45 +01:00
);
let mut matches = Vec::new();
for sm in raw_document.processed_matches {
let attribute = searchable_attrs
.and_then(|sa| sa.reverse(sm.attribute))
.unwrap_or(sm.attribute);
let attribute = match schema.indexed_pos_to_field_id(attribute) {
Some(field_id) => field_id.0,
None => {
error!("Cannot convert indexed_pos {} to field_id", attribute);
2020-01-29 18:30:21 +01:00
trace!("Schema is compromized; {:?}", schema);
continue
}
};
matches.push(SimpleMatch { attribute, ..sm });
2019-12-13 11:14:12 +01:00
}
matches.sort_unstable();
Document { id: raw_document.id, highlights, matches }
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::mem;
#[test]
fn docindex_mem_size() {
assert_eq!(mem::size_of::<DocIndex>(), 12);
}
}