Merge pull request #5707 from Mubelotix/last_embedder_message

Add last embedder error in batches

crates/meilisearch-types/src/batch_view.rs

@@ -3,7 +3,7 @@ use serde::Serialize;
 use time::{Duration, OffsetDateTime};
 use utoipa::ToSchema;
 
-use crate::batches::{Batch, BatchId, BatchStats};
+use crate::batches::{Batch, BatchId, BatchStats, EmbedderStatsView};
 use crate::task_view::DetailsView;
 use crate::tasks::serialize_duration;
 
@@ -14,7 +14,7 @@ pub struct BatchView {
     pub uid: BatchId,
     pub progress: Option<ProgressView>,
     pub details: DetailsView,
-    pub stats: BatchStats,
+    pub stats: BatchStatsView,
     #[serde(serialize_with = "serialize_duration", default)]
     pub duration: Option<Duration>,
     #[serde(with = "time::serde::rfc3339", default)]
@@ -25,13 +25,26 @@ pub struct BatchView {
     pub batch_strategy: String,
 }
 
+#[derive(Debug, Clone, Serialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct BatchStatsView {
+    #[serde(flatten)]
+    pub stats: BatchStats,
+    #[serde(skip_serializing_if = "EmbedderStatsView::skip_serializing", default)]
+    pub embedder_requests: EmbedderStatsView,
+}
+
 impl BatchView {
     pub fn from_batch(batch: &Batch) -> Self {
         Self {
             uid: batch.uid,
             progress: batch.progress.clone(),
             details: batch.details.clone(),
-            stats: batch.stats.clone(),
+            stats: BatchStatsView {
+                stats: batch.stats.clone(),
+                embedder_requests: batch.embedder_stats.clone(),
+            },
             duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at),
             started_at: batch.started_at,
             finished_at: batch.finished_at,

crates/meilisearch-types/src/batches.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use milli::progress::ProgressView;
+use milli::progress::{EmbedderStats, ProgressView};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use utoipa::ToSchema;
@@ -19,6 +19,8 @@ pub struct Batch {
     pub progress: Option<ProgressView>,
     pub details: DetailsView,
     pub stats: BatchStats,
+    #[serde(skip_serializing_if = "EmbedderStatsView::skip_serializing", default)]
+    pub embedder_stats: EmbedderStatsView,
 
     #[serde(with = "time::serde::rfc3339")]
     pub started_at: OffsetDateTime,
@@ -43,6 +45,7 @@ impl PartialEq for Batch {
             progress,
             details,
             stats,
+            embedder_stats,
             started_at,
             finished_at,
             enqueued_at,
@@ -53,6 +56,7 @@ impl PartialEq for Batch {
             && progress.is_none() == other.progress.is_none()
             && details == &other.details
             && stats == &other.stats
+            && embedder_stats == &other.embedder_stats
             && started_at == &other.started_at
             && finished_at == &other.finished_at
             && enqueued_at == &other.enqueued_at
@@ -83,3 +87,30 @@ pub struct BatchStats {
     #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
     pub internal_database_sizes: serde_json::Map<String, serde_json::Value>,
 }
+
+#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct EmbedderStatsView {
+    pub total: usize,
+    pub failed: usize,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub last_error: Option<String>,
+}
+
+impl From<&EmbedderStats> for EmbedderStatsView {
+    fn from(stats: &EmbedderStats) -> Self {
+        let errors = stats.errors.read().unwrap_or_else(|p| p.into_inner());
+        Self {
+            total: stats.total_count.load(std::sync::atomic::Ordering::Relaxed),
+            failed: errors.1 as usize,
+            last_error: errors.0.clone(),
+        }
+    }
+}
+
+impl EmbedderStatsView {
+    pub fn skip_serializing(&self) -> bool {
+        self.total == 0 && self.failed == 0 && self.last_error.is_none()
+    }
+}