diff --git a/crates/meilisearch/src/routes/indexes/facet_search.rs b/crates/meilisearch/src/routes/indexes/facet_search.rs index 41f306746..18ad54ccf 100644 --- a/crates/meilisearch/src/routes/indexes/facet_search.rs +++ b/crates/meilisearch/src/routes/indexes/facet_search.rs @@ -56,6 +56,8 @@ pub struct FacetSearchQuery { pub q: Option, #[deserr(default, error = DeserrJsonError)] pub vector: Option>, + #[deserr(default, error = DeserrJsonError)] + pub media: Option, #[deserr(default, error = DeserrJsonError)] pub hybrid: Option, #[deserr(default, error = DeserrJsonError)] @@ -94,6 +96,7 @@ impl FacetSearchAggregator { facet_name, vector, q, + media, filter, matching_strategy, attributes_to_search_on, @@ -108,6 +111,7 @@ impl FacetSearchAggregator { facet_names: Some(facet_name.clone()).into_iter().collect(), additional_search_parameters_provided: q.is_some() || vector.is_some() + || media.is_some() || filter.is_some() || *matching_strategy != MatchingStrategy::default() || attributes_to_search_on.is_some() @@ -291,6 +295,7 @@ impl From for SearchQuery { facet_name: _, q, vector, + media, filter, matching_strategy, attributes_to_search_on, @@ -312,6 +317,7 @@ impl From for SearchQuery { SearchQuery { q, + media, offset: DEFAULT_SEARCH_OFFSET(), limit: DEFAULT_SEARCH_LIMIT(), page, diff --git a/crates/meilisearch/src/routes/indexes/search.rs b/crates/meilisearch/src/routes/indexes/search.rs index 333ae1944..035ba71d8 100644 --- a/crates/meilisearch/src/routes/indexes/search.rs +++ b/crates/meilisearch/src/routes/indexes/search.rs @@ -205,6 +205,8 @@ impl TryFrom for SearchQuery { Ok(Self { q: other.q, + // `media` not supported for `GET` + media: None, vector: other.vector.map(CS::into_inner), offset: other.offset.0, limit: other.limit.0, diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 61ef3f813..6d8639504 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -64,6 +64,8 @@ pub struct SearchQuery { pub q: Option, #[deserr(default, error = DeserrJsonError)] pub vector: Option>, + #[deserr(default, error = DeserrJsonError)] + pub media: Option, #[deserr(default, error = DeserrJsonError)] pub hybrid: Option, #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError)] @@ -147,6 +149,7 @@ impl From for SearchQuery { ranking_score_threshold: ranking_score_threshold.map(RankingScoreThreshold::from), q: None, vector: None, + media: None, offset: DEFAULT_SEARCH_OFFSET(), page: None, hits_per_page: None, @@ -220,6 +223,7 @@ impl fmt::Debug for SearchQuery { let Self { q, vector, + media, hybrid, offset, limit, @@ -274,6 +278,9 @@ impl fmt::Debug for SearchQuery { ); } } + if let Some(media) = media { + debug.field("media", media); + } if let Some(hybrid) = hybrid { debug.field("hybrid", &hybrid); } @@ -482,8 +489,10 @@ pub struct SearchQueryWithIndex { pub index_uid: IndexUid, #[deserr(default, error = DeserrJsonError)] pub q: Option, - #[deserr(default, error = DeserrJsonError)] + #[deserr(default, error = DeserrJsonError)] pub vector: Option>, + #[deserr(default, error = DeserrJsonError)] + pub media: Option, #[deserr(default, error = DeserrJsonError)] pub hybrid: Option, #[deserr(default, error = DeserrJsonError)] @@ -564,6 +573,7 @@ impl SearchQueryWithIndex { let SearchQuery { q, vector, + media, hybrid, offset, limit, @@ -594,6 +604,7 @@ impl SearchQueryWithIndex { index_uid, q, vector, + media, hybrid, offset: if offset == DEFAULT_SEARCH_OFFSET() { None } else { Some(offset) }, limit: if limit == DEFAULT_SEARCH_LIMIT() { None } else { Some(limit) }, @@ -628,6 +639,7 @@ impl SearchQueryWithIndex { federation_options, q, vector, + media, offset, limit, page, @@ -658,6 +670,7 @@ impl SearchQueryWithIndex { SearchQuery { q, vector, + media, offset: offset.unwrap_or(DEFAULT_SEARCH_OFFSET()), limit: limit.unwrap_or(DEFAULT_SEARCH_LIMIT()), page, @@ -984,14 +997,27 @@ pub fn prepare_search<'t>( let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let q = query.q.as_deref(); + let media = query.media.as_ref(); + + let search_query = match (q, media) { + (Some(text), None) => milli::vector::SearchQuery::Text(text), + (q, media) => milli::vector::SearchQuery::Media { q, media }, + }; + embedder - .embed_search(query.q.as_ref().unwrap(), Some(deadline)) + .embed_search(search_query, Some(deadline)) .map_err(milli::vector::Error::from) .map_err(milli::Error::from)? } }; - - search.semantic(embedder_name.clone(), embedder.clone(), *quantized, Some(vector)); + search.semantic( + embedder_name.clone(), + embedder.clone(), + *quantized, + Some(vector), + query.media.clone(), + ); } SearchKind::Hybrid { embedder_name, embedder, quantized, semantic_ratio: _ } => { if let Some(q) = &query.q { @@ -1003,6 +1029,7 @@ pub fn prepare_search<'t>( embedder.clone(), *quantized, query.vector.clone(), + query.media.clone(), ); } } @@ -1127,6 +1154,7 @@ pub fn perform_search( locales, // already used in prepare_search vector: _, + media: _, hybrid: _, offset: _, ranking_score_threshold: _, diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index b63f6288f..c906e1eb7 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -7,6 +7,7 @@ use roaring::RoaringBitmap; use crate::score_details::{ScoreDetails, ScoreValue, ScoringStrategy}; use crate::search::new::{distinct_fid, distinct_single_docid}; use crate::search::SemanticSearch; +use crate::vector::SearchQuery; use crate::{Index, MatchingWords, Result, Search, SearchResult}; struct ScoreWithRatioResult { @@ -225,12 +226,9 @@ impl Search<'_> { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); } - // no vector search against placeholder search - let Some(query) = search.query.take() else { - return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); - }; // no embedder, no semantic search - let Some(SemanticSearch { vector, embedder_name, embedder, quantized }) = semantic else { + let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic + else { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); }; @@ -241,9 +239,17 @@ impl Search<'_> { let span = tracing::trace_span!(target: "search::hybrid", "embed_one"); let _entered = span.enter(); + let q = search.query.as_deref(); + let media = media.as_ref(); + + let query = match (q, media) { + (Some(text), None) => SearchQuery::Text(text), + (q, media) => SearchQuery::Media { q, media }, + }; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); - match embedder.embed_search(&query, Some(deadline)) { + match embedder.embed_search(query, Some(deadline)) { Ok(embedding) => embedding, Err(error) => { tracing::error!(error=%error, "Embedding failed"); @@ -257,8 +263,13 @@ impl Search<'_> { } }; - search.semantic = - Some(SemanticSearch { vector: Some(vector_query), embedder_name, embedder, quantized }); + search.semantic = Some(SemanticSearch { + vector: Some(vector_query), + embedder_name, + embedder, + quantized, + media, + }); // TODO: would be better to have two distinct functions at this point let vector_results = search.execute()?; diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 62183afc3..97d542524 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -12,7 +12,7 @@ use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::index::MatchingStrategy; use crate::score_details::{ScoreDetails, ScoringStrategy}; -use crate::vector::Embedder; +use crate::vector::{Embedder, Embedding}; use crate::{ execute_search, filtered_universe, AscDesc, DefaultSearchLogger, DocumentId, Error, Index, Result, SearchContext, TimeBudget, UserError, @@ -32,6 +32,7 @@ pub mod similar; #[derive(Debug, Clone)] pub struct SemanticSearch { vector: Option>, + media: Option, embedder_name: String, embedder: Arc, quantized: bool, @@ -93,9 +94,10 @@ impl<'a> Search<'a> { embedder_name: String, embedder: Arc, quantized: bool, - vector: Option>, + vector: Option, + media: Option, ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector }); + self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media }); self } @@ -231,24 +233,28 @@ impl<'a> Search<'a> { degraded, used_negative_operator, } = match self.semantic.as_ref() { - Some(SemanticSearch { vector: Some(vector), embedder_name, embedder, quantized }) => { - execute_vector_search( - &mut ctx, - vector, - self.scoring_strategy, - universe, - &self.sort_criteria, - &self.distinct, - self.geo_param, - self.offset, - self.limit, - embedder_name, - embedder, - *quantized, - self.time_budget.clone(), - self.ranking_score_threshold, - )? - } + Some(SemanticSearch { + vector: Some(vector), + embedder_name, + embedder, + quantized, + media: _, + }) => execute_vector_search( + &mut ctx, + vector, + self.scoring_strategy, + universe, + &self.sort_criteria, + &self.distinct, + self.geo_param, + self.offset, + self.limit, + embedder_name, + embedder, + *quantized, + self.time_budget.clone(), + self.ranking_score_threshold, + )?, _ => execute_search( &mut ctx, self.query.as_deref(),