From 16b4545d23185efc5ac077eb47585ba9db9504bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 18 Jul 2024 15:55:37 +0200 Subject: [PATCH] Prefix Redis keys to avoid false negative excessive writes --- milli/src/update/index_documents/cache.rs | 14 ++++++++++++-- .../extract/extract_facet_number_docids.rs | 1 + .../extract/extract_facet_string_docids.rs | 1 + .../extract/extract_fid_word_count_docids.rs | 1 + .../index_documents/extract/extract_word_docids.rs | 3 +++ .../extract/extract_word_pair_proximity_docids.rs | 1 + .../extract/extract_word_position_docids.rs | 1 + milli/src/update/word_prefix_docids.rs | 1 + milli/src/update/words_prefix_integer_docids.rs | 1 + 9 files changed, 22 insertions(+), 2 deletions(-) diff --git a/milli/src/update/index_documents/cache.rs b/milli/src/update/index_documents/cache.rs index cd633670b..51adf4ee5 100644 --- a/milli/src/update/index_documents/cache.rs +++ b/milli/src/update/index_documents/cache.rs @@ -11,6 +11,7 @@ use crate::CboRoaringBitmapCodec; pub struct SorterCacheDelAddCboRoaringBitmap { cache: LruCache, DelAddRoaringBitmap>, + prefix: &'static [u8; 3], sorter: grenad::Sorter, deladd_buffer: Vec, cbo_buffer: Vec, @@ -18,9 +19,15 @@ pub struct SorterCacheDelAddCboRoaringBitmap { } impl SorterCacheDelAddCboRoaringBitmap { - pub fn new(cap: NonZeroUsize, sorter: grenad::Sorter, conn: redis::Connection) -> Self { + pub fn new( + cap: NonZeroUsize, + sorter: grenad::Sorter, + prefix: &'static [u8; 3], + conn: redis::Connection, + ) -> Self { SorterCacheDelAddCboRoaringBitmap { cache: LruCache::new(cap), + prefix, sorter, deladd_buffer: Vec::new(), cbo_buffer: Vec::new(), @@ -136,7 +143,10 @@ where } DelAddRoaringBitmap { del: None, add: None } => return Ok(()), } - redis::cmd("INCR").arg(key.as_ref()).query::(&mut self.conn).unwrap(); + self.cbo_buffer.clear(); + self.cbo_buffer.extend_from_slice(self.prefix); + self.cbo_buffer.extend_from_slice(&key); + redis::cmd("INCR").arg(&self.cbo_buffer).query::(&mut self.conn).unwrap(); self.sorter.insert(key, value_writer.into_inner().unwrap()) } diff --git a/milli/src/update/index_documents/extract/extract_facet_number_docids.rs b/milli/src/update/index_documents/extract/extract_facet_number_docids.rs index 6c6243663..f36903b8f 100644 --- a/milli/src/update/index_documents/extract/extract_facet_number_docids.rs +++ b/milli/src/update/index_documents/extract/extract_facet_number_docids.rs @@ -40,6 +40,7 @@ pub fn extract_facet_number_docids( SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(20).unwrap(), facet_number_docids_sorter, + b"fnd", super::REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/index_documents/extract/extract_facet_string_docids.rs b/milli/src/update/index_documents/extract/extract_facet_string_docids.rs index a618b065f..da6b8fd82 100644 --- a/milli/src/update/index_documents/extract/extract_facet_string_docids.rs +++ b/milli/src/update/index_documents/extract/extract_facet_string_docids.rs @@ -48,6 +48,7 @@ pub fn extract_facet_string_docids( SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(200).unwrap(), facet_string_docids_sorter, + b"fsd", REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs b/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs index cb1dcea51..49de09eee 100644 --- a/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs +++ b/milli/src/update/index_documents/extract/extract_fid_word_count_docids.rs @@ -43,6 +43,7 @@ pub fn extract_fid_word_count_docids( SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(300).unwrap(), fid_word_count_docids_sorter, + b"fwc", super::REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/index_documents/extract/extract_word_docids.rs b/milli/src/update/index_documents/extract/extract_word_docids.rs index dbe52e426..fca9ca3b1 100644 --- a/milli/src/update/index_documents/extract/extract_word_docids.rs +++ b/milli/src/update/index_documents/extract/extract_word_docids.rs @@ -52,6 +52,7 @@ pub fn extract_word_docids( let mut cached_word_fid_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, _>::new( NonZeroUsize::new(300).unwrap(), word_fid_docids_sorter, + b"wfd", REDIS_CLIENT.get_connection().unwrap(), ); @@ -112,6 +113,7 @@ pub fn extract_word_docids( let mut cached_word_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(100).unwrap(), word_docids_sorter, + b"wdi", REDIS_CLIENT.get_connection().unwrap(), ); @@ -126,6 +128,7 @@ pub fn extract_word_docids( let mut cached_exact_word_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(100).unwrap(), exact_word_docids_sorter, + b"ewd", REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs b/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs index 881a17818..6b7a2496c 100644 --- a/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs +++ b/milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs @@ -55,6 +55,7 @@ pub fn extract_word_pair_proximity_docids( SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(100).unwrap(), sorter, + b"wpp", super::REDIS_CLIENT.get_connection().unwrap(), ) }) diff --git a/milli/src/update/index_documents/extract/extract_word_position_docids.rs b/milli/src/update/index_documents/extract/extract_word_position_docids.rs index ecdd62df0..7343b6e6b 100644 --- a/milli/src/update/index_documents/extract/extract_word_position_docids.rs +++ b/milli/src/update/index_documents/extract/extract_word_position_docids.rs @@ -41,6 +41,7 @@ pub fn extract_word_position_docids( SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(300).unwrap(), word_position_docids_sorter, + b"wpd", super::REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/word_prefix_docids.rs b/milli/src/update/word_prefix_docids.rs index 4326a380c..c551642ba 100644 --- a/milli/src/update/word_prefix_docids.rs +++ b/milli/src/update/word_prefix_docids.rs @@ -68,6 +68,7 @@ impl<'t, 'i> WordPrefixDocids<'t, 'i> { let mut cached_prefix_docids_sorter = SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(200).unwrap(), prefix_docids_sorter, + b"pdi", REDIS_CLIENT.get_connection().unwrap(), ); diff --git a/milli/src/update/words_prefix_integer_docids.rs b/milli/src/update/words_prefix_integer_docids.rs index 8e6e957ce..038882530 100644 --- a/milli/src/update/words_prefix_integer_docids.rs +++ b/milli/src/update/words_prefix_integer_docids.rs @@ -73,6 +73,7 @@ impl<'t, 'i> WordPrefixIntegerDocids<'t, 'i> { SorterCacheDelAddCboRoaringBitmap::<20, MergeFn>::new( NonZeroUsize::new(200).unwrap(), prefix_integer_docids_sorter, + b"pid", REDIS_CLIENT.get_connection().unwrap(), );