From 4a5923a55e1146dad0f0660615e271b1cc2c6c13 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 27 Jan 2025 14:17:43 +0100 Subject: [PATCH 1/6] log the time arroy took to insert embeddings --- crates/milli/src/update/new/indexer/write.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/update/new/indexer/write.rs b/crates/milli/src/update/new/indexer/write.rs index d1cc2038c..ffbe7728d 100644 --- a/crates/milli/src/update/new/indexer/write.rs +++ b/crates/milli/src/update/new/indexer/write.rs @@ -76,7 +76,7 @@ pub(super) fn write_to_db( Ok(()) } -#[tracing::instrument(level = "trace", skip_all, target = "indexing::vectors")] +#[tracing::instrument(level = "debug", skip_all, target = "indexing::vectors")] pub(super) fn build_vectors( index: &Index, wtxn: &mut RwTxn<'_>, From 0f8eb3b5066642c80cee237feab34e5a4257a55a Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 13 Jan 2025 14:51:14 +0100 Subject: [PATCH 2/6] Improve the logs of the search with AI --- crates/milli/src/search/new/mod.rs | 2 +- crates/milli/src/search/new/vector_sort.rs | 2 +- crates/milli/src/vector/mod.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index 49f08b521..b9161b417 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -563,7 +563,7 @@ fn resolve_sort_criteria<'ctx, Query: RankingRuleQueryTrait>( Ok(()) } -#[tracing::instrument(level = "trace", skip_all, target = "search::universe")] +#[tracing::instrument(level = "debug", skip_all, target = "search::universe")] pub fn filtered_universe( index: &Index, txn: &RoTxn<'_>, diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index a25605cfc..f6502a908 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -83,7 +83,7 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for VectorSort { } #[allow(clippy::only_used_in_recursion)] - #[tracing::instrument(level = "trace", skip_all, target = "search::vector_sort")] + #[tracing::instrument(level = "debug", skip_all, target = "search::vector_sort")] fn next_bucket( &mut self, ctx: &mut SearchContext<'ctx>, diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 0be698027..9ccd7341c 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -611,6 +611,7 @@ impl Embedder { } } + #[tracing::instrument(level = "debug", skip_all, target = "search")] pub fn embed_one( &self, text: String, From 47f70e3d79d84aa88bde280902b19e09c5341d1b Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 14 Jan 2025 16:13:18 +0100 Subject: [PATCH 3/6] Debug the first vector sort fill buffer --- crates/milli/src/search/new/vector_sort.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index f6502a908..230fb08a4 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -66,7 +66,7 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for VectorSort { "vector_sort".to_owned() } - #[tracing::instrument(level = "trace", skip_all, target = "search::vector_sort")] + #[tracing::instrument(level = "debug", skip_all, target = "search::vector_sort")] fn start_iteration( &mut self, ctx: &mut SearchContext<'ctx>, From 19bc885b07618942793437b7e47df915a3b2a1f1 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 27 Jan 2025 14:30:59 +0100 Subject: [PATCH 4/6] Fix the milli logo --- crates/milli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/README.md b/crates/milli/README.md index 8f04d04dc..101da0684 100644 --- a/crates/milli/README.md +++ b/crates/milli/README.md @@ -1,5 +1,5 @@

- the milli logo + the milli logo

a concurrent indexer combined with fast and relevant search algorithms

From 424c5bde4006ffc887ed5aa20fda08cc0906d0ef Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Wed, 29 Jan 2025 16:40:36 +0100 Subject: [PATCH 5/6] Move the embedding computation and extraction log to debug --- crates/milli/src/update/new/indexer/extract.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/update/new/indexer/extract.rs b/crates/milli/src/update/new/indexer/extract.rs index 63536c559..53478f029 100644 --- a/crates/milli/src/update/new/indexer/extract.rs +++ b/crates/milli/src/update/new/indexer/extract.rs @@ -234,7 +234,7 @@ where ); let mut datastore = ThreadLocal::with_capacity(rayon::current_num_threads()); { - let span = tracing::trace_span!(target: "indexing::documents::extract", "vectors"); + let span = tracing::debug_span!(target: "indexing::documents::extract", "vectors"); let _entered = span.enter(); extract( @@ -247,7 +247,7 @@ where )?; } { - let span = tracing::trace_span!(target: "indexing::documents::merge", "vectors"); + let span = tracing::debug_span!(target: "indexing::documents::merge", "vectors"); let _entered = span.enter(); for config in &mut index_embeddings { From cc8df5e11f2853dc641ea39f2ae9f06b73d51f85 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 3 Feb 2025 13:52:00 +0100 Subject: [PATCH 6/6] Move back the search-side logging to tracing --- crates/milli/src/search/new/vector_sort.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/new/vector_sort.rs b/crates/milli/src/search/new/vector_sort.rs index 230fb08a4..a25605cfc 100644 --- a/crates/milli/src/search/new/vector_sort.rs +++ b/crates/milli/src/search/new/vector_sort.rs @@ -66,7 +66,7 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for VectorSort { "vector_sort".to_owned() } - #[tracing::instrument(level = "debug", skip_all, target = "search::vector_sort")] + #[tracing::instrument(level = "trace", skip_all, target = "search::vector_sort")] fn start_iteration( &mut self, ctx: &mut SearchContext<'ctx>, @@ -83,7 +83,7 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for VectorSort { } #[allow(clippy::only_used_in_recursion)] - #[tracing::instrument(level = "debug", skip_all, target = "search::vector_sort")] + #[tracing::instrument(level = "trace", skip_all, target = "search::vector_sort")] fn next_bucket( &mut self, ctx: &mut SearchContext<'ctx>,