searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 08:44:27 +01:00
Code Issues Releases Wiki Activity

Plug the deletion pipeline

Browse Source
This commit is contained in:
Clément Renault 2024-10-08 16:04:19 +02:00
parent 470c2272dd
commit 39b27e42be
No known key found for this signature in database
GPG Key ID: F250A4C4E3AE5F5F
2 changed files with 27 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
index-scheduler/src/batch.rs
View File

@ -1500,26 +1500,35 @@ impl IndexScheduler {
} }
} }
let config = IndexDocumentsConfig { let rtxn = index.read_txn()?;
update_method: IndexDocumentsMethod::ReplaceDocuments, let mut fields_ids_map = index.fields_ids_map(&rtxn)?;
..Default::default()
};
let must_stop_processing = self.must_stop_processing.clone(); let primary_key =
let mut builder = milli::update::IndexDocuments::new( retrieve_or_guess_primary_key(&rtxn, index, &mut fields_ids_map, None)?
index_wtxn, .unwrap();
index,
self.index_mapper.indexer_config(),
config,
|indexing_step| tracing::debug!(update = ?indexing_step),
|| must_stop_processing.get(),
)?;
let (new_builder, _count) = if !tasks.iter().all(|res| res.error.is_some()) {
builder.remove_documents_from_db_no_batch(&to_delete)?; /// TODO create a pool if needed
builder = new_builder; // let pool = indexer_config.thread_pool.unwrap();
let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
let _ = builder.execute()?; let param = (index, &fields_ids_map, &primary_key);
let mut indexer = indexer::DocumentDeletion::new();
indexer.delete_documents_by_docids(to_delete);
/// TODO remove this fields-ids-map, it's useless for the deletion pipeline (the &mut cloned one).
let document_changes =
indexer.document_changes(&mut fields_ids_map.clone(), param)?;
/// TODO pass/write the FieldsIdsMap
indexer::index(
index_wtxn,
index,
fields_ids_map.clone(),
&pool,
document_changes,
)?;
// tracing::info!(indexing_result = ?addition, processed_in = ?started_processing_at.elapsed(), "document indexing done");
}
Ok(tasks) Ok(tasks)
} }

2
milli/src/update/new/indexer/document_deletion.rs
View File

@ -43,7 +43,7 @@ impl<'p> DocumentChanges<'p> for DocumentDeletion {
|rtxn, docid| { |rtxn, docid| {
let current = index.document(rtxn, docid)?; let current = index.document(rtxn, docid)?;
let external_document_id = primary_key let external_document_id = primary_key
.document_id(&current, fields_ids_map)? .document_id(current, fields_ids_map)?
.map_err(|_| InternalError::DatabaseMissingEntry { .map_err(|_| InternalError::DatabaseMissingEntry {
db_name: EXTERNAL_DOCUMENTS_IDS, db_name: EXTERNAL_DOCUMENTS_IDS,
key: None, key: None,