searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 00:34:26 +01:00
Code Issues Releases Wiki Activity

Use an LRU only with 1000 entries

Browse Source
This commit is contained in:
Clément Renault 2024-07-22 13:39:20 +02:00
parent 6264dbf326
commit ca332883cc
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
9 changed files with 23 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
milli/src/update/index_documents/cache.rs
View File

@ -14,7 +14,7 @@ use crate::CboRoaringBitmapCodec;
const ENABLED: bool = true;
pub struct SorterCacheDelAddCboRoaringBitmap<const N: usize, MF> {
cache: ArcCache<SmallVec<[u8; N]>, DelAddRoaringBitmap>,
cache: LruCache<SmallVec<[u8; N]>, DelAddRoaringBitmap>,
prefix: &'static [u8; 3],
sorter: grenad::Sorter<MF>,
deladd_buffer: Vec<u8>,
@ -30,7 +30,7 @@ impl<const N: usize, MF> SorterCacheDelAddCboRoaringBitmap<N, MF> {
conn: sled::Db,
) -> Self {
SorterCacheDelAddCboRoaringBitmap {
cache: ArcCache::new(cap),
cache: LruCache::new(cap),
prefix,
sorter,
deladd_buffer: Vec::new(),
@ -49,7 +49,7 @@ where
return self.write_entry_to_sorter(key, DelAddRoaringBitmap::new_del_u32(n));
}
let (cache, evicted) = self.cache.get_mut(key);
let cache = self.cache.get_mut(key);
match cache {
Some(DelAddRoaringBitmap { del, add: _ }) => {
del.get_or_insert_with(RoaringBitmap::new).insert(n);
@ -62,10 +62,7 @@ where
}
}
match evicted {
Some((key, value)) => self.write_entry_to_sorter(key, value),
None => Ok(()),
}
Ok(())
}
pub fn insert_del(
@ -77,7 +74,7 @@ where
return self.write_entry_to_sorter(key, DelAddRoaringBitmap::new_del(bitmap));
}
let (cache, evicted) = self.cache.get_mut(key);
let cache = self.cache.get_mut(key);
match cache {
Some(DelAddRoaringBitmap { del, add: _ }) => {
*del.get_or_insert_with(RoaringBitmap::new) |= bitmap;
@ -90,10 +87,7 @@ where
}
}
match evicted {
Some((key, value)) => self.write_entry_to_sorter(key, value),
None => Ok(()),
}
Ok(())
}
pub fn insert_add_u32(&mut self, key: &[u8], n: u32) -> Result<(), grenad::Error<U>> {
@ -101,7 +95,7 @@ where
return self.write_entry_to_sorter(key, DelAddRoaringBitmap::new_add_u32(n));
}
let (cache, evicted) = self.cache.get_mut(key);
let cache = self.cache.get_mut(key);
match cache {
Some(DelAddRoaringBitmap { del: _, add }) => {
add.get_or_insert_with(RoaringBitmap::new).insert(n);
@ -114,10 +108,7 @@ where
}
}
match evicted {
Some((key, value)) => self.write_entry_to_sorter(key, value),
None => Ok(()),
}
Ok(())
}
pub fn insert_add(
@ -129,7 +120,7 @@ where
return self.write_entry_to_sorter(key, DelAddRoaringBitmap::new_add(bitmap));
}
let (cache, evicted) = self.cache.get_mut(key);
let cache = self.cache.get_mut(key);
match cache {
Some(DelAddRoaringBitmap { del: _, add }) => {
*add.get_or_insert_with(RoaringBitmap::new) |= bitmap;
@ -142,10 +133,7 @@ where
}
}
match evicted {
Some((key, value)) => self.write_entry_to_sorter(key, value),
None => Ok(()),
}
Ok(())
}
pub fn insert_del_add_u32(&mut self, key: &[u8], n: u32) -> Result<(), grenad::Error<U>> {
@ -153,7 +141,7 @@ where
return self.write_entry_to_sorter(key, DelAddRoaringBitmap::new_del_add_u32(n));
}
let (cache, evicted) = self.cache.get_mut(key);
let cache = self.cache.get_mut(key);
match cache {
Some(DelAddRoaringBitmap { del, add }) => {
del.get_or_insert_with(RoaringBitmap::new).insert(n);
@ -167,10 +155,7 @@ where
}
}
match evicted {
Some((key, value)) => self.write_entry_to_sorter(key, value),
None => Ok(()),
}
Ok(())
}
fn write_entry_to_sorter<A: AsRef<[u8]>>(
@ -218,7 +203,7 @@ where
}
pub fn into_sorter(mut self) -> Result<grenad::Sorter<MF>, grenad::Error<U>> {
let default_arc = ArcCache::new(NonZeroUsize::MIN);
let default_arc = LruCache::new(NonZeroUsize::MIN);
for (key, deladd) in mem::replace(&mut self.cache, default_arc) {
self.write_entry_to_sorter(key, deladd)?;
}

2
milli/src/update/index_documents/extract/extract_facet_number_docids.rs
View File

@ -38,7 +38,7 @@ pub fn extract_facet_number_docids<R: io::Read + io::Seek>(
);
let mut cached_facet_number_docids_sorter =
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
facet_number_docids_sorter,
b"fnd",
super::SLED_DB.clone(),

2
milli/src/update/index_documents/extract/extract_facet_string_docids.rs
View File

@ -46,7 +46,7 @@ pub fn extract_facet_string_docids<R: io::Read + io::Seek>(
);
let mut cached_facet_string_docids_sorter =
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
facet_string_docids_sorter,
b"fsd",
SLED_DB.clone(),

2
milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs
View File

@ -41,7 +41,7 @@ pub fn extract_fid_word_count_docids<R: io::Read + io::Seek>(
);
let mut cached_fid_word_count_docids_sorter =
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
fid_word_count_docids_sorter,
b"fwc",
super::SLED_DB.clone(),

6
milli/src/update/index_documents/extract/extract_word_docids.rs
View File

@ -50,7 +50,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
max_memory.map(|m| m / 3),
);
let mut cached_word_fid_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, _>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
word_fid_docids_sorter,
b"wfd",
SLED_DB.clone(),
@ -111,7 +111,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
max_memory.map(|m| m / 3),
);
let mut cached_word_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
word_docids_sorter,
b"wdi",
SLED_DB.clone(),
@ -126,7 +126,7 @@ pub fn extract_word_docids<R: io::Read + io::Seek>(
max_memory.map(|m| m / 3),
);
let mut cached_exact_word_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
exact_word_docids_sorter,
b"ewd",
SLED_DB.clone(),

2
milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs
View File

@ -53,7 +53,7 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
max_memory.map(|m| m / MAX_DISTANCE as usize),
);
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
sorter,
b"wpp",
super::SLED_DB.clone(),

2
milli/src/update/index_documents/extract/extract_word_position_docids.rs
View File

@ -39,7 +39,7 @@ pub fn extract_word_position_docids<R: io::Read + io::Seek>(
);
let mut cached_word_position_docids_sorter =
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
word_position_docids_sorter,
b"wpd",
super::SLED_DB.clone(),

2
milli/src/update/word_prefix_docids.rs
View File

@ -66,7 +66,7 @@ impl<'t, 'i> WordPrefixDocids<'t, 'i> {
self.max_memory,
);
let mut cached_prefix_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
prefix_docids_sorter,
b"pdi",
SLED_DB.clone(),

2
milli/src/update/words_prefix_integer_docids.rs
View File

@ -71,7 +71,7 @@ impl<'t, 'i> WordPrefixIntegerDocids<'t, 'i> {
);
let mut cached_prefix_integer_docids_sorter =
SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new(
NonZeroUsize::new(500).unwrap(),
NonZeroUsize::new(1000).unwrap(),
prefix_integer_docids_sorter,
b"pid",
SLED_DB.clone(),