searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-23 05:14:27 +01:00
Code Issues Releases Wiki Activity

Restrict field ids in search context

Browse Source
This commit is contained in:
ManyTheFish 2023-06-13 17:37:35 +02:00
parent 0ccf1e2e40
commit fb8fa07169
2 changed files with 60 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
milli/src/search/new/db_cache.rs
View File

@ -10,7 +10,7 @@ use roaring::RoaringBitmap;
use super::interner::Interned; use super::interner::Interned;
use super::Word; use super::Word;
use crate::heed_codec::{BytesDecodeOwned, StrBEU16Codec}; use crate::heed_codec::{BytesDecodeOwned, StrBEU16Codec};
use crate::update::MergeFn; use crate::update::{merge_cbo_roaring_bitmaps, MergeFn};
use crate::{ use crate::{
CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec, SearchContext, CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec, SearchContext,
}; };
@ -79,7 +79,7 @@ impl<'ctx> DatabaseCache<'ctx> {
fn get_value_from_keys<'v, K1, KC, DC>( fn get_value_from_keys<'v, K1, KC, DC>(
txn: &'ctx RoTxn, txn: &'ctx RoTxn,
cache_key: K1, cache_key: K1,
db_keys: &[&'v KC::EItem], db_keys: &'v [KC::EItem],
cache: &mut FxHashMap<K1, Option<Cow<'ctx, [u8]>>>, cache: &mut FxHashMap<K1, Option<Cow<'ctx, [u8]>>>,
db: Database<KC, ByteSlice>, db: Database<KC, ByteSlice>,
merger: MergeFn, merger: MergeFn,
@ -88,6 +88,7 @@ impl<'ctx> DatabaseCache<'ctx> {
K1: Copy + Eq + Hash, K1: Copy + Eq + Hash,
KC: BytesEncode<'v>, KC: BytesEncode<'v>,
DC: BytesDecodeOwned, DC: BytesDecodeOwned,
KC::EItem: Sized,
{ {
match cache.entry(cache_key) { match cache.entry(cache_key) {
Entry::Occupied(_) => {} Entry::Occupied(_) => {}
@ -125,6 +126,7 @@ impl<'ctx> DatabaseCache<'ctx> {
} }
} }
} }
impl<'ctx> SearchContext<'ctx> { impl<'ctx> SearchContext<'ctx> {
pub fn get_words_fst(&mut self) -> Result<fst::Set<Cow<'ctx, [u8]>>> { pub fn get_words_fst(&mut self) -> Result<fst::Set<Cow<'ctx, [u8]>>> {
if let Some(fst) = self.db_cache.words_fst.clone() { if let Some(fst) = self.db_cache.words_fst.clone() {
@ -158,13 +160,28 @@ impl<'ctx> SearchContext<'ctx> {
/// Retrieve or insert the given value in the `word_docids` database. /// Retrieve or insert the given value in the `word_docids` database.
fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> { fn get_db_word_docids(&mut self, word: Interned<String>) -> Result<Option<RoaringBitmap>> {
DatabaseCache::get_value::<_, _, RoaringBitmapCodec>( match &self.restricted_fids {
self.txn, Some(restricted_fids) => {
word, let interned = self.word_interner.get(word).as_str();
self.word_interner.get(word).as_str(), let keys: Vec<_> = restricted_fids.iter().map(|fid| (interned, *fid)).collect();
&mut self.db_cache.word_docids,
self.index.word_docids.remap_data_type::<ByteSlice>(), DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
) self.txn,
word,
&keys[..],
&mut self.db_cache.word_docids,
self.index.word_fid_docids.remap_data_type::<ByteSlice>(),
merge_cbo_roaring_bitmaps,
)
}
None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
self.txn,
word,
self.word_interner.get(word).as_str(),
&mut self.db_cache.word_docids,
self.index.word_docids.remap_data_type::<ByteSlice>(),
),
}
} }
fn get_db_exact_word_docids( fn get_db_exact_word_docids(
@ -205,13 +222,28 @@ impl<'ctx> SearchContext<'ctx> {
&mut self, &mut self,
prefix: Interned<String>, prefix: Interned<String>,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
DatabaseCache::get_value::<_, _, RoaringBitmapCodec>( match &self.restricted_fids {
self.txn, Some(restricted_fids) => {
prefix, let interned = self.word_interner.get(prefix).as_str();
self.word_interner.get(prefix).as_str(), let keys: Vec<_> = restricted_fids.iter().map(|fid| (interned, *fid)).collect();
&mut self.db_cache.word_prefix_docids,
self.index.word_prefix_docids.remap_data_type::<ByteSlice>(), DatabaseCache::get_value_from_keys::<_, _, CboRoaringBitmapCodec>(
) self.txn,
prefix,
&keys[..],
&mut self.db_cache.word_prefix_docids,
self.index.word_prefix_fid_docids.remap_data_type::<ByteSlice>(),
merge_cbo_roaring_bitmaps,
)
}
None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
self.txn,
prefix,
self.word_interner.get(prefix).as_str(),
&mut self.db_cache.word_prefix_docids,
self.index.word_prefix_docids.remap_data_type::<ByteSlice>(),
),
}
} }
fn get_db_exact_word_prefix_docids( fn get_db_exact_word_prefix_docids(
@ -307,6 +339,11 @@ impl<'ctx> SearchContext<'ctx> {
word: Interned<String>, word: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the restricted list, return None.
if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None);
}
DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
(word, fid), (word, fid),
@ -321,6 +358,11 @@ impl<'ctx> SearchContext<'ctx> {
word_prefix: Interned<String>, word_prefix: Interned<String>,
fid: u16, fid: u16,
) -> Result<Option<RoaringBitmap>> { ) -> Result<Option<RoaringBitmap>> {
// if the requested fid isn't in the restricted list, return None.
if self.restricted_fids.as_ref().map_or(false, |fids| !fids.contains(&fid)) {
return Ok(None);
}
DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>( DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
self.txn, self.txn,
(word_prefix, fid), (word_prefix, fid),

4
milli/src/update/mod.rs
View File

@ -4,8 +4,8 @@ pub use self::delete_documents::{DeleteDocuments, DeletionStrategy, DocumentDele
pub use self::facet::bulk::FacetsUpdateBulk; pub use self::facet::bulk::FacetsUpdateBulk;
pub use self::facet::incremental::FacetsUpdateIncrementalInner; pub use self::facet::incremental::FacetsUpdateIncrementalInner;
pub use self::index_documents::{ pub use self::index_documents::{
merge_roaring_bitmaps, DocumentAdditionResult, DocumentId, IndexDocuments, merge_cbo_roaring_bitmaps, merge_roaring_bitmaps, DocumentAdditionResult, DocumentId,
IndexDocumentsConfig, IndexDocumentsMethod, MergeFn, IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, MergeFn,
}; };
pub use self::indexer_config::IndexerConfig; pub use self::indexer_config::IndexerConfig;
pub use self::prefix_word_pairs::{ pub use self::prefix_word_pairs::{