From 2e32d0474ccc846bbe86c0bbafd88368f82e8a3e Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 4 Dec 2024 17:05:07 +0100 Subject: [PATCH] Lexicographically sort all the map to merge --- crates/milli/src/update/new/extract/cache.rs | 38 +++++++------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/crates/milli/src/update/new/extract/cache.rs b/crates/milli/src/update/new/extract/cache.rs index b57ba6b9b..325a72280 100644 --- a/crates/milli/src/update/new/extract/cache.rs +++ b/crates/milli/src/update/new/extract/cache.rs @@ -640,36 +640,24 @@ where } // Then manage the content on the HashMap entries that weren't taken (mem::take). - let order_count = 1000; while let Some(mut map) = maps.pop() { - let mut iter = map.iter_mut(); + // Make sure we don't try to work with entries already managed by the spilled + let mut ordered_entries: Vec<_> = + map.iter_mut().filter(|(_, bbbul)| !bbbul.is_empty()).collect(); + ordered_entries.sort_unstable_by_key(|(key, _)| *key); - loop { - let mut ordered_buffer: Vec<_> = iter.by_ref().take(order_count).collect(); - ordered_buffer.sort_unstable_by_key(|(key, _)| *key); + for (key, bbbul) in ordered_entries { + let mut output = DelAddRoaringBitmap::empty(); + output.union_and_clear_bbbul(bbbul); - if ordered_buffer.is_empty() { - break; + for rhs in maps.iter_mut() { + if let Some(new) = rhs.get_mut(key) { + output.union_and_clear_bbbul(new); + } } - for (key, bbbul) in ordered_buffer { - // Make sure we don't try to work with entries already managed by the spilled - if bbbul.is_empty() { - continue; - } - - let mut output = DelAddRoaringBitmap::empty(); - output.union_and_clear_bbbul(bbbul); - - for rhs in maps.iter_mut() { - if let Some(new) = rhs.get_mut(key) { - output.union_and_clear_bbbul(new); - } - } - - // We send the merged entry outside. - (f)(key, output)?; - } + // We send the merged entry outside. + (f)(key, output)?; } }