searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-26 23:04:26 +01:00
Code Issues Releases Wiki Activity

Renaming the indexers

Browse Source
This commit is contained in:
Clément Renault 2024-09-02 14:42:27 +02:00
parent 6526ce1208
commit 72e7b7846e
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
6 changed files with 35 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
milli/src/update/new/indexer/document_deletion.rs
View File

@ -8,11 +8,11 @@ use crate::documents::PrimaryKey;
use crate::update::new::{Deletion, DocumentChange, ItemsPool};
use crate::{FieldsIdsMap, Index, InternalError, Result};
pub struct DocumentDeletionIndexer {
pub struct DocumentDeletion {
pub to_delete: RoaringBitmap,
}
impl DocumentDeletionIndexer {
impl DocumentDeletion {
pub fn new() -> Self {
Self { to_delete: Default::default() }
}
@ -22,7 +22,7 @@ impl DocumentDeletionIndexer {
}
}
impl<'p> Indexer<'p> for DocumentDeletionIndexer {
impl<'p> Indexer<'p> for DocumentDeletion {
type Parameter = (&'p Index, &'p FieldsIdsMap, &'p PrimaryKey<'p>);
fn document_changes(

40
milli/src/update/new/indexer/document_operation.rs
View File

@ -19,9 +19,9 @@ use crate::update::new::{Deletion, Insertion, KvReaderFieldId, KvWriterFieldId,
use crate::update::{AvailableIds, IndexDocumentsMethod};
use crate::{DocumentId, Error, FieldsIdsMap, Index, Result, UserError};
pub struct DocumentOperationIndexer {
pub(crate) operations: Vec<Payload>,
pub(crate) index_documents_method: IndexDocumentsMethod,
pub struct DocumentOperation {
operations: Vec<Payload>,
index_documents_method: IndexDocumentsMethod,
}
pub enum Payload {
@ -34,7 +34,7 @@ pub struct PayloadStats {
pub bytes: u64,
}
pub enum DocumentOperation {
enum InnerDocOp {
Addition(DocumentOffset),
Deletion,
}
@ -48,7 +48,7 @@ pub struct DocumentOffset {
pub offset: u32,
}
impl DocumentOperationIndexer {
impl DocumentOperation {
pub fn new(method: IndexDocumentsMethod) -> Self {
Self { operations: Default::default(), index_documents_method: method }
}
@ -70,7 +70,7 @@ impl DocumentOperationIndexer {
}
}
impl<'p> Indexer<'p> for DocumentOperationIndexer {
impl<'p> Indexer<'p> for DocumentOperation {
type Parameter = (&'p Index, &'p RoTxn<'static>, &'p mut FieldsIdsMap, &'p PrimaryKey<'p>);
fn document_changes(
@ -120,7 +120,7 @@ impl<'p> Indexer<'p> for DocumentOperationIndexer {
let content = content.clone();
let document_offset = DocumentOffset { content, offset };
let document_operation = DocumentOperation::Addition(document_offset);
let document_operation = InnerDocOp::Addition(document_offset);
match docids_version_offsets.get_mut(&external_document_id) {
None => {
@ -160,10 +160,10 @@ impl<'p> Indexer<'p> for DocumentOperationIndexer {
docids_version_offsets.insert(
external_document_id,
(docid, vec![DocumentOperation::Deletion]),
(docid, vec![InnerDocOp::Deletion]),
);
}
Some((_, offsets)) => offsets.push(DocumentOperation::Deletion),
Some((_, offsets)) => offsets.push(InnerDocOp::Deletion),
}
}
}
@ -204,7 +204,7 @@ fn merge_document_for_updates(
fields_ids_map: &FieldsIdsMap,
docid: DocumentId,
external_docid: String,
operations: &[DocumentOperation],
operations: &[InnerDocOp],
) -> Result<Option<DocumentChange>> {
let mut document = BTreeMap::<_, Cow<_>>::new();
let current = index.documents.remap_data_type::<Bytes>().get(rtxn, &docid)?;
@ -226,14 +226,12 @@ fn merge_document_for_updates(
});
}
}
if operations.is_empty() {
match current {
Some(current) => {
return Ok(Some(DocumentChange::Deletion(Deletion::create(
docid,
external_docid,
current.boxed(),
))));
let deletion = Deletion::create(docid, external_docid, current.boxed());
return Ok(Some(DocumentChange::Deletion(deletion)));
}
None => return Ok(None),
}
@ -241,8 +239,8 @@ fn merge_document_for_updates(
for operation in operations {
let DocumentOffset { content, offset } = match operation {
DocumentOperation::Addition(offset) => offset,
DocumentOperation::Deletion => {
InnerDocOp::Addition(offset) => offset,
InnerDocOp::Deletion => {
unreachable!("Deletion in document operations")
}
};
@ -283,13 +281,13 @@ fn merge_document_for_replacements(
fields_ids_map: &FieldsIdsMap,
docid: DocumentId,
external_docid: String,
operations: &[DocumentOperation],
operations: &[InnerDocOp],
) -> Result<Option<DocumentChange>> {
let current = index.documents.remap_data_type::<Bytes>().get(rtxn, &docid)?;
let current: Option<&KvReaderFieldId> = current.map(Into::into);
match operations.last() {
Some(DocumentOperation::Addition(DocumentOffset { content, offset })) => {
Some(InnerDocOp::Addition(DocumentOffset { content, offset })) => {
let reader = DocumentsBatchReader::from_reader(Cursor::new(content.as_ref()))?;
let (mut cursor, batch_index) = reader.into_cursor_and_fields_index();
let update = cursor.get(*offset)?.expect("must exists");
@ -318,13 +316,13 @@ fn merge_document_for_replacements(
}
}
}
Some(DocumentOperation::Deletion) => match current {
Some(InnerDocOp::Deletion) => match current {
Some(current) => {
let deletion = Deletion::create(docid, external_docid, current.boxed());
Ok(Some(DocumentChange::Deletion(deletion)))
}
None => Ok(None),
},
None => Ok(None),
None => Ok(None), // but it's strange
}
}

8
milli/src/update/new/indexer/mod.rs
View File

@ -1,13 +1,13 @@
use std::thread;
use big_s::S;
pub use document_deletion::DocumentDeletionIndexer;
pub use document_operation::DocumentOperationIndexer;
pub use document_deletion::DocumentDeletion;
pub use document_operation::DocumentOperation;
use heed::RwTxn;
pub use partial_dump::PartialDumpIndexer;
pub use partial_dump::PartialDump;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use rayon::ThreadPool;
pub use update_by_function::UpdateByFunctionIndexer;
pub use update_by_function::UpdateByFunction;
use super::channel::{
extractors_merger_channels, merger_writer_channels, EntryOperation, ExtractorsMergerChannels,

9
milli/src/update/new/indexer/partial_dump.rs
View File

@ -6,17 +6,17 @@ use crate::update::concurrent_available_ids::ConcurrentAvailableIds;
use crate::update::new::{DocumentChange, Insertion, KvWriterFieldId};
use crate::{all_obkv_to_json, Error, FieldsIdsMap, Object, Result, UserError};
pub struct PartialDumpIndexer<I> {
pub struct PartialDump<I> {
pub iter: I,
}
impl<I> PartialDumpIndexer<I> {
impl<I> PartialDump<I> {
pub fn new_from_jsonlines(iter: I) -> Self {
PartialDumpIndexer { iter }
PartialDump { iter }
}
}
impl<'p, I> Indexer<'p> for PartialDumpIndexer<I>
impl<'p, I> Indexer<'p> for PartialDump<I>
where
I: IntoIterator<Item = Object>,
I::IntoIter: Send + 'p,
@ -45,6 +45,7 @@ where
let key = fields_ids_map.id(key).unwrap();
/// TODO better error management
let value = serde_json::to_vec(&value).unwrap();
/// TODO it is not ordered
writer.insert(key, value).unwrap();
});

5
milli/src/update/new/indexer/update_by_function.rs
View File

@ -4,15 +4,16 @@ use super::Indexer;
use crate::update::new::DocumentChange;
use crate::Result;
pub struct UpdateByFunctionIndexer;
pub struct UpdateByFunction;
impl<'p> Indexer<'p> for UpdateByFunctionIndexer {
impl<'p> Indexer<'p> for UpdateByFunction {
type Parameter = ();
fn document_changes(
self,
_param: Self::Parameter,
) -> Result<impl ParallelIterator<Item = Result<Option<DocumentChange>>> + 'p> {
todo!();
Ok(vec![].into_par_iter())
}
}

6
milli/src/update/new/mod.rs
View File

@ -1,8 +1,4 @@
pub use document_change::{Deletion, DocumentChange, Insertion, Update};
pub use indexer::{
index, DocumentDeletionIndexer, DocumentOperationIndexer, PartialDumpIndexer,
UpdateByFunctionIndexer,
};
pub use items_pool::ItemsPool;
use super::del_add::DelAdd;
@ -13,7 +9,7 @@ mod merger;
// mod extract;
// mod global_fields_ids_map;
mod channel;
mod indexer;
pub mod indexer;
mod items_pool;
/// TODO move them elsewhere