searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-30 00:34:26 +01:00
Code Issues Releases Wiki Activity

Span to measure the part of db writes that is after the merge/extraction

Browse Source
This commit is contained in:
Louis Dureuil 2024-11-26 14:10:52 +01:00
parent fb66fec398
commit 8f57b4fdf4
No known key found for this signature in database
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
crates/milli/src/update/new/indexer/mod.rs
View File

@ -1,4 +1,5 @@
use std::cmp::Ordering; use std::cmp::Ordering;
use std::sync::atomic::AtomicBool;
use std::sync::{OnceLock, RwLock}; use std::sync::{OnceLock, RwLock};
use std::thread::{self, Builder}; use std::thread::{self, Builder};
@ -76,6 +77,7 @@ where
SP: Fn(Progress) + Sync, SP: Fn(Progress) + Sync,
{ {
let (extractor_sender, writer_receiver) = extractor_writer_channel(10_000); let (extractor_sender, writer_receiver) = extractor_writer_channel(10_000);
let finished_extraction = AtomicBool::new(false);
let metadata_builder = MetadataBuilder::from_index(index, wtxn)?; let metadata_builder = MetadataBuilder::from_index(index, wtxn)?;
let new_fields_ids_map = FieldIdMapWithMetadata::new(new_fields_ids_map, metadata_builder); let new_fields_ids_map = FieldIdMapWithMetadata::new(new_fields_ids_map, metadata_builder);
@ -100,6 +102,7 @@ where
thread::scope(|s| -> Result<()> { thread::scope(|s| -> Result<()> {
let indexer_span = tracing::Span::current(); let indexer_span = tracing::Span::current();
let embedders = &embedders; let embedders = &embedders;
let finished_extraction = &finished_extraction;
// prevent moving the field_distribution and document_ids in the inner closure... // prevent moving the field_distribution and document_ids in the inner closure...
let field_distribution = &mut field_distribution; let field_distribution = &mut field_distribution;
let document_ids = &mut document_ids; let document_ids = &mut document_ids;
@ -350,6 +353,8 @@ where
(indexing_context.send_progress)(Progress::from_step(Step::WritingToDatabase)); (indexing_context.send_progress)(Progress::from_step(Step::WritingToDatabase));
finished_extraction.store(true, std::sync::atomic::Ordering::Relaxed);
Result::Ok(facet_field_ids_delta) Result::Ok(facet_field_ids_delta)
})?; })?;
@ -384,7 +389,15 @@ where
let span = tracing::trace_span!(target: "indexing::write_db", "all"); let span = tracing::trace_span!(target: "indexing::write_db", "all");
let _entered = span.enter(); let _entered = span.enter();
let span = tracing::trace_span!(target: "indexing::write_db", "post_merge");
let mut _entered_post_merge = None;
for operation in writer_receiver { for operation in writer_receiver {
if _entered_post_merge.is_none()
&& finished_extraction.load(std::sync::atomic::Ordering::Relaxed)
{
_entered_post_merge = Some(span.enter());
}
match operation { match operation {
WriterOperation::DbOperation(db_operation) => { WriterOperation::DbOperation(db_operation) => {
let database = db_operation.database(index); let database = db_operation.database(index);