searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 00:34:26 +01:00
Code Issues Releases Wiki Activity

Use deladd merging function in chunks mergers

Browse Source
This commit is contained in:
ManyTheFish 2023-10-25 14:15:06 +02:00 committed by Louis Dureuil
parent 01d5eedf2f
commit 762b0b47e6
No known key found for this signature in database
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
milli/src/update/index_documents/extract/mod.rs
View File

@ -28,8 +28,8 @@ use self::extract_word_docids::extract_word_docids;
use self::extract_word_pair_proximity_docids::extract_word_pair_proximity_docids; use self::extract_word_pair_proximity_docids::extract_word_pair_proximity_docids;
use self::extract_word_position_docids::extract_word_position_docids; use self::extract_word_position_docids::extract_word_position_docids;
use super::helpers::{ use super::helpers::{
as_cloneable_grenad, merge_cbo_roaring_bitmaps, CursorClonableMmap, GrenadParameters, MergeFn, as_cloneable_grenad, merge_deladd_cbo_roaring_bitmaps, CursorClonableMmap, GrenadParameters,
MergeableReader, MergeFn, MergeableReader,
}; };
use super::{helpers, TypedChunk}; use super::{helpers, TypedChunk};
use crate::{FieldId, Result}; use crate::{FieldId, Result};
@ -108,7 +108,7 @@ pub(crate) fn data_from_obkv_documents(
let lmdb_writer_sx = lmdb_writer_sx.clone(); let lmdb_writer_sx = lmdb_writer_sx.clone();
rayon::spawn(move || { rayon::spawn(move || {
debug!("merge {} database", "facet-id-exists-docids"); debug!("merge {} database", "facet-id-exists-docids");
match facet_exists_docids_chunks.merge(merge_cbo_roaring_bitmaps, &indexer) { match facet_exists_docids_chunks.merge(merge_deladd_cbo_roaring_bitmaps, &indexer) {
Ok(reader) => { Ok(reader) => {
let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetExistsDocids(reader))); let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetExistsDocids(reader)));
} }
@ -124,7 +124,7 @@ pub(crate) fn data_from_obkv_documents(
let lmdb_writer_sx = lmdb_writer_sx.clone(); let lmdb_writer_sx = lmdb_writer_sx.clone();
rayon::spawn(move || { rayon::spawn(move || {
debug!("merge {} database", "facet-id-is-null-docids"); debug!("merge {} database", "facet-id-is-null-docids");
match facet_is_null_docids_chunks.merge(merge_cbo_roaring_bitmaps, &indexer) { match facet_is_null_docids_chunks.merge(merge_deladd_cbo_roaring_bitmaps, &indexer) {
Ok(reader) => { Ok(reader) => {
let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetIsNullDocids(reader))); let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetIsNullDocids(reader)));
} }
@ -140,7 +140,7 @@ pub(crate) fn data_from_obkv_documents(
let lmdb_writer_sx = lmdb_writer_sx.clone(); let lmdb_writer_sx = lmdb_writer_sx.clone();
rayon::spawn(move || { rayon::spawn(move || {
debug!("merge {} database", "facet-id-is-empty-docids"); debug!("merge {} database", "facet-id-is-empty-docids");
match facet_is_empty_docids_chunks.merge(merge_cbo_roaring_bitmaps, &indexer) { match facet_is_empty_docids_chunks.merge(merge_deladd_cbo_roaring_bitmaps, &indexer) {
Ok(reader) => { Ok(reader) => {
let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetIsEmptyDocids(reader))); let _ = lmdb_writer_sx.send(Ok(TypedChunk::FieldIdFacetIsEmptyDocids(reader)));
} }
@ -156,7 +156,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx.clone(), lmdb_writer_sx.clone(),
extract_word_pair_proximity_docids, extract_word_pair_proximity_docids,
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
TypedChunk::WordPairProximityDocids, TypedChunk::WordPairProximityDocids,
"word-pair-proximity-docids", "word-pair-proximity-docids",
); );
@ -166,7 +166,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx.clone(), lmdb_writer_sx.clone(),
extract_fid_word_count_docids, extract_fid_word_count_docids,
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
TypedChunk::FieldIdWordCountDocids, TypedChunk::FieldIdWordCountDocids,
"field-id-wordcount-docids", "field-id-wordcount-docids",
); );
@ -184,7 +184,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx.clone(), lmdb_writer_sx.clone(),
move |doc_word_pos, indexer| extract_word_docids(doc_word_pos, indexer, &exact_attributes), move |doc_word_pos, indexer| extract_word_docids(doc_word_pos, indexer, &exact_attributes),
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
|(word_docids_reader, exact_word_docids_reader, word_fid_docids_reader)| { |(word_docids_reader, exact_word_docids_reader, word_fid_docids_reader)| {
TypedChunk::WordDocids { TypedChunk::WordDocids {
word_docids_reader, word_docids_reader,
@ -200,7 +200,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx.clone(), lmdb_writer_sx.clone(),
extract_word_position_docids, extract_word_position_docids,
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
TypedChunk::WordPositionDocids, TypedChunk::WordPositionDocids,
"word-position-docids", "word-position-docids",
); );
@ -210,7 +210,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx.clone(), lmdb_writer_sx.clone(),
extract_facet_string_docids, extract_facet_string_docids,
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
TypedChunk::FieldIdFacetStringDocids, TypedChunk::FieldIdFacetStringDocids,
"field-id-facet-string-docids", "field-id-facet-string-docids",
); );
@ -220,7 +220,7 @@ pub(crate) fn data_from_obkv_documents(
indexer, indexer,
lmdb_writer_sx, lmdb_writer_sx,
extract_facet_number_docids, extract_facet_number_docids,
merge_cbo_roaring_bitmaps, merge_deladd_cbo_roaring_bitmaps,
TypedChunk::FieldIdFacetNumberDocids, TypedChunk::FieldIdFacetNumberDocids,
"field-id-facet-number-docids", "field-id-facet-number-docids",
); );