From 2b75072b0976f6068511dd00fb5e2252ad08280f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 3 Jul 2025 14:04:27 +0200 Subject: [PATCH 1/5] Expose the number of internal chat searches on the /metrics route --- crates/meilisearch/src/metrics.rs | 6 ++++++ crates/meilisearch/src/routes/chats/chat_completions.rs | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 29c1aeae8..1c7d0c3a4 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -15,6 +15,12 @@ lazy_static! { "Meilisearch number of degraded search requests" )) .expect("Can't create a metric"); + pub static ref MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS: IntGauge = + register_int_gauge!(opts!( + "meilisearch_chat_internal_search_requests", + "Meilisearch number of search requests performed by the chat route itself" + )) + .expect("Can't create a metric"); pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge = register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch DB Size In Bytes")) .expect("Can't create a metric"); diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index ccbdccbbc..f6030f2bc 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -48,7 +48,9 @@ use crate::analytics::Analytics; use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _}; -use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS; +use crate::metrics::{ + MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, MEILISEARCH_DEGRADED_SEARCH_REQUESTS, +}; use crate::routes::chats::utils::SseEventSender; use crate::routes::indexes::search::search_kind; use crate::search::{add_search_rules, prepare_search, search_from_kind, SearchQuery}; @@ -286,7 +288,7 @@ async fn process_search_request( let output = output?; let mut documents = Vec::new(); if let Ok((ref rtxn, ref search_result)) = output { - // aggregate.succeed(search_result); + MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS.inc(); if search_result.degraded { MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc(); } From 9f0d33ec999920dfdec917aff14604df9f30e6b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 3 Jul 2025 15:05:15 +0200 Subject: [PATCH 2/5] Expose the number of tokens on the chat completions routes --- crates/meilisearch/src/metrics.rs | 5 +++ .../src/routes/chats/chat_completions.rs | 35 ++++++++++++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 1c7d0c3a4..9941bacae 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -21,6 +21,11 @@ lazy_static! { "Meilisearch number of search requests performed by the chat route itself" )) .expect("Can't create a metric"); + pub static ref MEILISEARCH_CHAT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( + opts!("meilisearch_chat_tokens_usage", "Meilisearch Chat Tokens Usage"), + &["chat", "model", "type"] + ) + .expect("Can't create a metric"); pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge = register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch DB Size In Bytes")) .expect("Can't create a metric"); diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index f6030f2bc..a7d878c6e 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -13,9 +13,9 @@ use async_openai::types::{ ChatCompletionRequestDeveloperMessageContent, ChatCompletionRequestMessage, ChatCompletionRequestSystemMessage, ChatCompletionRequestSystemMessageContent, ChatCompletionRequestToolMessage, ChatCompletionRequestToolMessageContent, - ChatCompletionStreamResponseDelta, ChatCompletionToolArgs, ChatCompletionToolType, - CreateChatCompletionRequest, CreateChatCompletionStreamResponse, FinishReason, FunctionCall, - FunctionCallStream, FunctionObjectArgs, + ChatCompletionStreamOptions, ChatCompletionStreamResponseDelta, ChatCompletionToolArgs, + ChatCompletionToolType, CreateChatCompletionRequest, CreateChatCompletionStreamResponse, + FinishReason, FunctionCall, FunctionCallStream, FunctionObjectArgs, }; use async_openai::Client; use bumpalo::Bump; @@ -49,7 +49,8 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _}; use crate::metrics::{ - MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, MEILISEARCH_DEGRADED_SEARCH_REQUESTS, + MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, MEILISEARCH_CHAT_TOKENS_USAGE, + MEILISEARCH_DEGRADED_SEARCH_REQUESTS, }; use crate::routes::chats::utils::SseEventSender; use crate::routes::indexes::search::search_kind; @@ -490,6 +491,7 @@ async fn streamed_chat( let (tx, rx) = tokio::sync::mpsc::channel(10); let tx = SseEventSender::new(tx); + let workspace_uid = workspace_uid.to_string(); let _join_handle = Handle::current().spawn(async move { let client = Client::with_config(config.clone()); let mut global_tool_calls = HashMap::::new(); @@ -499,6 +501,7 @@ async fn streamed_chat( let output = run_conversation( &index_scheduler, &auth_ctrl, + &workspace_uid, &search_queue, &auth_token, &client, @@ -536,6 +539,7 @@ async fn run_conversation( Data, >, auth_ctrl: &web::Data, + workspace_uid: &str, search_queue: &web::Data, auth_token: &str, client: &Client, @@ -546,12 +550,33 @@ async fn run_conversation( function_support: FunctionSupport, ) -> Result, ()>, SendError> { let mut finish_reason = None; + chat_completion.stream_options = Some(ChatCompletionStreamOptions { include_usage: true }); // safety: unwrap: can only happens if `stream` was set to `false` let mut response = client.chat().create_stream(chat_completion.clone()).await.unwrap(); while let Some(result) = response.next().await { match result { Ok(resp) => { - let choice = &resp.choices[0]; + let choice = match resp.choices.get(0) { + Some(choice) => choice, + None => { + if let Some(usage) = resp.usage.as_ref() { + for (r#type, value) in &[ + ("prompt", usage.prompt_tokens), + ("completion", usage.completion_tokens), + ("total", usage.total_tokens), + ] { + MEILISEARCH_CHAT_TOKENS_USAGE + .with_label_values(&[ + workspace_uid, + &chat_completion.model, + r#type, + ]) + .inc_by(*value as u64); + } + } + break; + } + }; finish_reason = choice.finish_reason; let ChatCompletionStreamResponseDelta { ref tool_calls, .. } = &choice.delta; From b5e41f0e4612eb4c665994a6a33064a2afac8c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 3 Jul 2025 15:18:16 +0200 Subject: [PATCH 3/5] Fix the Mistral uncompatibility with the usage of OpenAI --- .../src/routes/chats/chat_completions.rs | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index a7d878c6e..ea3077e99 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -549,33 +549,33 @@ async fn run_conversation( global_tool_calls: &mut HashMap, function_support: FunctionSupport, ) -> Result, ()>, SendError> { + use DbChatCompletionSource::*; + let mut finish_reason = None; - chat_completion.stream_options = Some(ChatCompletionStreamOptions { include_usage: true }); + chat_completion.stream_options = match source { + OpenAi | AzureOpenAi => Some(ChatCompletionStreamOptions { include_usage: true }), + Mistral | VLlm => None, + }; + // safety: unwrap: can only happens if `stream` was set to `false` let mut response = client.chat().create_stream(chat_completion.clone()).await.unwrap(); while let Some(result) = response.next().await { match result { Ok(resp) => { - let choice = match resp.choices.get(0) { - Some(choice) => choice, - None => { - if let Some(usage) = resp.usage.as_ref() { - for (r#type, value) in &[ - ("prompt", usage.prompt_tokens), - ("completion", usage.completion_tokens), - ("total", usage.total_tokens), - ] { - MEILISEARCH_CHAT_TOKENS_USAGE - .with_label_values(&[ - workspace_uid, - &chat_completion.model, - r#type, - ]) - .inc_by(*value as u64); - } - } - break; + if let Some(usage) = resp.usage.as_ref() { + for (r#type, value) in &[ + ("prompt", usage.prompt_tokens), + ("completion", usage.completion_tokens), + ("total", usage.total_tokens), + ] { + MEILISEARCH_CHAT_TOKENS_USAGE + .with_label_values(&[workspace_uid, &chat_completion.model, r#type]) + .inc_by(*value as u64); } + } + let choice = match resp.choices.first() { + Some(choice) => choice, + None => break, }; finish_reason = choice.finish_reason; From 6397ef12a0f42aee7255b046109ed2a63f3f34d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 3 Jul 2025 15:56:56 +0200 Subject: [PATCH 4/5] Use three metrics for the three different tokens --- crates/meilisearch/src/metrics.rs | 20 +++++++++++++++--- .../src/routes/chats/chat_completions.rs | 21 ++++++++++--------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 9941bacae..2207e69ff 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -21,9 +21,23 @@ lazy_static! { "Meilisearch number of search requests performed by the chat route itself" )) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( - opts!("meilisearch_chat_tokens_usage", "Meilisearch Chat Tokens Usage"), - &["chat", "model", "type"] + pub static ref MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( + opts!("meilisearch_chat_prompt_tokens_usage", "Meilisearch Chat Prompt Tokens Usage"), + &["workspace", "model"] + ) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE: IntCounterVec = + register_int_counter_vec!( + opts!( + "meilisearch_chat_completion_tokens_usage", + "Meilisearch Chat Completion Tokens Usage" + ), + &["workspace", "model"] + ) + .expect("Can't create a metric"); + pub static ref MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( + opts!("meilisearch_chat_total_tokens_usage", "Meilisearch Chat Total Tokens Usage"), + &["workspace", "model"] ) .expect("Can't create a metric"); pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge = diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index ea3077e99..9d132a96f 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -49,7 +49,8 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _}; use crate::metrics::{ - MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, MEILISEARCH_CHAT_TOKENS_USAGE, + MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE, MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, + MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE, MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE, MEILISEARCH_DEGRADED_SEARCH_REQUESTS, }; use crate::routes::chats::utils::SseEventSender; @@ -563,15 +564,15 @@ async fn run_conversation( match result { Ok(resp) => { if let Some(usage) = resp.usage.as_ref() { - for (r#type, value) in &[ - ("prompt", usage.prompt_tokens), - ("completion", usage.completion_tokens), - ("total", usage.total_tokens), - ] { - MEILISEARCH_CHAT_TOKENS_USAGE - .with_label_values(&[workspace_uid, &chat_completion.model, r#type]) - .inc_by(*value as u64); - } + MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE + .with_label_values(&[workspace_uid, &chat_completion.model]) + .inc_by(usage.prompt_tokens as u64); + MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE + .with_label_values(&[workspace_uid, &chat_completion.model]) + .inc_by(usage.completion_tokens as u64); + MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE + .with_label_values(&[workspace_uid, &chat_completion.model]) + .inc_by(usage.total_tokens as u64); } let choice = match resp.choices.first() { Some(choice) => choice, From a76a3e8f118a48d1bd57775cf9d509e8374305e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 3 Jul 2025 16:01:31 +0200 Subject: [PATCH 5/5] Change the metric name for the search to use a label --- crates/meilisearch/src/metrics.rs | 12 +++++++----- .../meilisearch/src/routes/chats/chat_completions.rs | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 2207e69ff..d52e04cc6 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -15,12 +15,14 @@ lazy_static! { "Meilisearch number of degraded search requests" )) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS: IntGauge = - register_int_gauge!(opts!( - "meilisearch_chat_internal_search_requests", + pub static ref MEILISEARCH_CHAT_SEARCH_REQUESTS: IntCounterVec = register_int_counter_vec!( + opts!( + "meilisearch_chat_search_requests", "Meilisearch number of search requests performed by the chat route itself" - )) - .expect("Can't create a metric"); + ), + &["type"] + ) + .expect("Can't create a metric"); pub static ref MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( opts!("meilisearch_chat_prompt_tokens_usage", "Meilisearch Chat Prompt Tokens Usage"), &["workspace", "model"] diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 9d132a96f..4f7087ae8 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -49,8 +49,8 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _}; use crate::metrics::{ - MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE, MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS, - MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE, MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE, + MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE, MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE, + MEILISEARCH_CHAT_SEARCH_REQUESTS, MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE, MEILISEARCH_DEGRADED_SEARCH_REQUESTS, }; use crate::routes::chats::utils::SseEventSender; @@ -290,7 +290,7 @@ async fn process_search_request( let output = output?; let mut documents = Vec::new(); if let Ok((ref rtxn, ref search_result)) = output { - MEILISEARCH_CHAT_INTERNAL_SEARCH_REQUESTS.inc(); + MEILISEARCH_CHAT_SEARCH_REQUESTS.with_label_values(&["internal"]).inc(); if search_result.degraded { MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc(); }