Add media to search

crates/meilisearch/src/routes/indexes/facet_search.rs

@@ -56,6 +56,8 @@ pub struct FacetSearchQuery {
     pub q: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchVector>)]
     pub vector: Option<Vec<f32>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchMedia>)]
+    pub media: Option<Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHybridQuery>)]
     pub hybrid: Option<HybridQuery>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchFilter>)]
@@ -94,6 +96,7 @@ impl FacetSearchAggregator {
             facet_name,
             vector,
             q,
+            media,
             filter,
             matching_strategy,
             attributes_to_search_on,
@@ -108,6 +111,7 @@ impl FacetSearchAggregator {
             facet_names: Some(facet_name.clone()).into_iter().collect(),
             additional_search_parameters_provided: q.is_some()
                 || vector.is_some()
+                || media.is_some()
                 || filter.is_some()
                 || *matching_strategy != MatchingStrategy::default()
                 || attributes_to_search_on.is_some()
@@ -291,6 +295,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             facet_name: _,
             q,
             vector,
+            media,
             filter,
             matching_strategy,
             attributes_to_search_on,
@@ -312,6 +317,7 @@ impl From<FacetSearchQuery> for SearchQuery {
 
         SearchQuery {
             q,
+            media,
             offset: DEFAULT_SEARCH_OFFSET(),
             limit: DEFAULT_SEARCH_LIMIT(),
             page,

crates/meilisearch/src/routes/indexes/search.rs

@@ -205,6 +205,8 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
 
         Ok(Self {
             q: other.q,
+            // `media` not supported for `GET`
+            media: None,
             vector: other.vector.map(CS::into_inner),
             offset: other.offset.0,
             limit: other.limit.0,

crates/meilisearch/src/search/mod.rs

@@ -64,6 +64,8 @@ pub struct SearchQuery {
     pub q: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchVector>)]
     pub vector: Option<Vec<f32>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchMedia>)]
+    pub media: Option<serde_json::Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHybridQuery>)]
     pub hybrid: Option<HybridQuery>,
     #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
@@ -147,6 +149,7 @@ impl From<SearchParameters> for SearchQuery {
             ranking_score_threshold: ranking_score_threshold.map(RankingScoreThreshold::from),
             q: None,
             vector: None,
+            media: None,
             offset: DEFAULT_SEARCH_OFFSET(),
             page: None,
             hits_per_page: None,
@@ -220,6 +223,7 @@ impl fmt::Debug for SearchQuery {
         let Self {
             q,
             vector,
+            media,
             hybrid,
             offset,
             limit,
@@ -274,6 +278,9 @@ impl fmt::Debug for SearchQuery {
                 );
             }
         }
+        if let Some(media) = media {
+            debug.field("media", media);
+        }
         if let Some(hybrid) = hybrid {
             debug.field("hybrid", &hybrid);
         }
@@ -482,8 +489,10 @@ pub struct SearchQueryWithIndex {
     pub index_uid: IndexUid,
     #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
     pub q: Option<String>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchQ>)]
+    #[deserr(default, error = DeserrJsonError<InvalidSearchVector>)]
     pub vector: Option<Vec<f32>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchMedia>)]
+    pub media: Option<serde_json::Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHybridQuery>)]
     pub hybrid: Option<HybridQuery>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchOffset>)]
@@ -564,6 +573,7 @@ impl SearchQueryWithIndex {
         let SearchQuery {
             q,
             vector,
+            media,
             hybrid,
             offset,
             limit,
@@ -594,6 +604,7 @@ impl SearchQueryWithIndex {
             index_uid,
             q,
             vector,
+            media,
             hybrid,
             offset: if offset == DEFAULT_SEARCH_OFFSET() { None } else { Some(offset) },
             limit: if limit == DEFAULT_SEARCH_LIMIT() { None } else { Some(limit) },
@@ -628,6 +639,7 @@ impl SearchQueryWithIndex {
             federation_options,
             q,
             vector,
+            media,
             offset,
             limit,
             page,
@@ -658,6 +670,7 @@ impl SearchQueryWithIndex {
             SearchQuery {
                 q,
                 vector,
+                media,
                 offset: offset.unwrap_or(DEFAULT_SEARCH_OFFSET()),
                 limit: limit.unwrap_or(DEFAULT_SEARCH_LIMIT()),
                 page,
@@ -984,14 +997,27 @@ pub fn prepare_search<'t>(
 
                     let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
 
+                    let q = query.q.as_deref();
+                    let media = query.media.as_ref();
+
+                    let search_query = match (q, media) {
+                        (Some(text), None) => milli::vector::SearchQuery::Text(text),
+                        (q, media) => milli::vector::SearchQuery::Media { q, media },
+                    };
+
                     embedder
-                        .embed_search(query.q.as_ref().unwrap(), Some(deadline))
+                        .embed_search(search_query, Some(deadline))
                         .map_err(milli::vector::Error::from)
                         .map_err(milli::Error::from)?
                 }
             };
-
-            search.semantic(embedder_name.clone(), embedder.clone(), *quantized, Some(vector));
+            search.semantic(
+                embedder_name.clone(),
+                embedder.clone(),
+                *quantized,
+                Some(vector),
+                query.media.clone(),
+            );
         }
         SearchKind::Hybrid { embedder_name, embedder, quantized, semantic_ratio: _ } => {
             if let Some(q) = &query.q {
@@ -1003,6 +1029,7 @@ pub fn prepare_search<'t>(
                 embedder.clone(),
                 *quantized,
                 query.vector.clone(),
+                query.media.clone(),
             );
         }
     }
@@ -1127,6 +1154,7 @@ pub fn perform_search(
         locales,
         // already used in prepare_search
         vector: _,
+        media: _,
         hybrid: _,
         offset: _,
         ranking_score_threshold: _,

crates/milli/src/search/hybrid.rs

@@ -7,6 +7,7 @@ use roaring::RoaringBitmap;
 use crate::score_details::{ScoreDetails, ScoreValue, ScoringStrategy};
 use crate::search::new::{distinct_fid, distinct_single_docid};
 use crate::search::SemanticSearch;
+use crate::vector::SearchQuery;
 use crate::{Index, MatchingWords, Result, Search, SearchResult};
 
 struct ScoreWithRatioResult {
@@ -225,12 +226,9 @@ impl Search<'_> {
             return Ok(return_keyword_results(self.limit, self.offset, keyword_results));
         }
 
-        // no vector search against placeholder search
-        let Some(query) = search.query.take() else {
-            return Ok(return_keyword_results(self.limit, self.offset, keyword_results));
-        };
         // no embedder, no semantic search
-        let Some(SemanticSearch { vector, embedder_name, embedder, quantized }) = semantic else {
+        let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic
+        else {
             return Ok(return_keyword_results(self.limit, self.offset, keyword_results));
         };
 
@@ -241,9 +239,17 @@ impl Search<'_> {
                 let span = tracing::trace_span!(target: "search::hybrid", "embed_one");
                 let _entered = span.enter();
 
+                let q = search.query.as_deref();
+                let media = media.as_ref();
+
+                let query = match (q, media) {
+                    (Some(text), None) => SearchQuery::Text(text),
+                    (q, media) => SearchQuery::Media { q, media },
+                };
+
                 let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3);
 
-                match embedder.embed_search(&query, Some(deadline)) {
+                match embedder.embed_search(query, Some(deadline)) {
                     Ok(embedding) => embedding,
                     Err(error) => {
                         tracing::error!(error=%error, "Embedding failed");
@@ -257,8 +263,13 @@ impl Search<'_> {
             }
         };
 
-        search.semantic =
-            Some(SemanticSearch { vector: Some(vector_query), embedder_name, embedder, quantized });
+        search.semantic = Some(SemanticSearch {
+            vector: Some(vector_query),
+            embedder_name,
+            embedder,
+            quantized,
+            media,
+        });
 
         // TODO: would be better to have two distinct functions at this point
         let vector_results = search.execute()?;

crates/milli/src/search/mod.rs

@@ -12,7 +12,7 @@ use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats};
 use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features};
 use crate::index::MatchingStrategy;
 use crate::score_details::{ScoreDetails, ScoringStrategy};
-use crate::vector::Embedder;
+use crate::vector::{Embedder, Embedding};
 use crate::{
     execute_search, filtered_universe, AscDesc, DefaultSearchLogger, DocumentId, Error, Index,
     Result, SearchContext, TimeBudget, UserError,
@@ -32,6 +32,7 @@ pub mod similar;
 #[derive(Debug, Clone)]
 pub struct SemanticSearch {
     vector: Option<Vec<f32>>,
+    media: Option<serde_json::Value>,
     embedder_name: String,
     embedder: Arc<Embedder>,
     quantized: bool,
@@ -93,9 +94,10 @@ impl<'a> Search<'a> {
         embedder_name: String,
         embedder: Arc<Embedder>,
         quantized: bool,
-        vector: Option<Vec<f32>>,
+        vector: Option<Embedding>,
+        media: Option<serde_json::Value>,
     ) -> &mut Search<'a> {
-        self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector });
+        self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media });
         self
     }
 
@@ -231,24 +233,28 @@ impl<'a> Search<'a> {
             degraded,
             used_negative_operator,
         } = match self.semantic.as_ref() {
-            Some(SemanticSearch { vector: Some(vector), embedder_name, embedder, quantized }) => {
-                execute_vector_search(
-                    &mut ctx,
-                    vector,
-                    self.scoring_strategy,
-                    universe,
-                    &self.sort_criteria,
-                    &self.distinct,
-                    self.geo_param,
-                    self.offset,
-                    self.limit,
-                    embedder_name,
-                    embedder,
-                    *quantized,
-                    self.time_budget.clone(),
-                    self.ranking_score_threshold,
-                )?
-            }
+            Some(SemanticSearch {
+                vector: Some(vector),
+                embedder_name,
+                embedder,
+                quantized,
+                media: _,
+            }) => execute_vector_search(
+                &mut ctx,
+                vector,
+                self.scoring_strategy,
+                universe,
+                &self.sort_criteria,
+                &self.distinct,
+                self.geo_param,
+                self.offset,
+                self.limit,
+                embedder_name,
+                embedder,
+                *quantized,
+                self.time_budget.clone(),
+                self.ranking_score_threshold,
+            )?,
             _ => execute_search(
                 &mut ctx,
                 self.query.as_deref(),