Merge #4938

4938: Remove default embedder r=ManyTheFish a=dureuill

# Pull Request

## Related issue
Fixes #4738 

## What does this PR do?

[See public usage](https://meilisearch.notion.site/v1-11-AI-search-changes-0e37727193884a70999f254fa953ce6e#1044b06b651f80edb9d4ef6dc367bad0)

- Remove `hybrid.embedder` boolean from analytics because embedder is now mandatory and so the boolean would always be `true`
- Rework search kind so that a search without query but with vector is a vector search regardless of (non-zero) semantic ratio


Co-authored-by: Louis Dureuil <louis@meilisearch.com>

meilisearch/src/analytics/segment_analytics.rs

@@ -646,8 +646,6 @@ pub struct SearchAggregator {
     max_vector_size: usize,
     // Whether the semantic ratio passed to a hybrid search equals the default ratio.
     semantic_ratio: bool,
-    // Whether a non-default embedder was specified
-    embedder: bool,
     hybrid: bool,
     retrieve_vectors: bool,
 
@@ -795,7 +793,6 @@ impl SearchAggregator {
 
         if let Some(hybrid) = hybrid {
             ret.semantic_ratio = hybrid.semantic_ratio != DEFAULT_SEMANTIC_RATIO();
-            ret.embedder = hybrid.embedder.is_some();
             ret.hybrid = true;
         }
 
@@ -863,7 +860,6 @@ impl SearchAggregator {
             show_ranking_score,
             show_ranking_score_details,
             semantic_ratio,
-            embedder,
             hybrid,
             total_degraded,
             total_used_negative_operator,
@@ -923,7 +919,6 @@ impl SearchAggregator {
         self.retrieve_vectors |= retrieve_vectors;
         self.semantic_ratio |= semantic_ratio;
         self.hybrid |= hybrid;
-        self.embedder |= embedder;
 
         // pagination
         self.max_limit = self.max_limit.max(max_limit);
@@ -999,7 +994,6 @@ impl SearchAggregator {
             show_ranking_score,
             show_ranking_score_details,
             semantic_ratio,
-            embedder,
             hybrid,
             total_degraded,
             total_used_negative_operator,
@@ -1051,7 +1045,6 @@ impl SearchAggregator {
                 "hybrid": {
                     "enabled": hybrid,
                     "semantic_ratio": semantic_ratio,
-                    "embedder": embedder,
                 },
                 "pagination": {
                    "max_limit": max_limit,
@@ -1782,7 +1775,6 @@ pub struct SimilarAggregator {
     used_syntax: HashMap<String, usize>,
 
     // Whether a non-default embedder was specified
-    embedder: bool,
     retrieve_vectors: bool,
 
     // pagination
@@ -1803,7 +1795,7 @@ impl SimilarAggregator {
     pub fn from_query(query: &SimilarQuery, request: &HttpRequest) -> Self {
         let SimilarQuery {
             id: _,
-            embedder,
+            embedder: _,
             offset,
             limit,
             attributes_to_retrieve: _,
@@ -1851,7 +1843,6 @@ impl SimilarAggregator {
         ret.show_ranking_score_details = *show_ranking_score_details;
         ret.ranking_score_threshold = ranking_score_threshold.is_some();
 
-        ret.embedder = embedder.is_some();
         ret.retrieve_vectors = *retrieve_vectors;
 
         ret
@@ -1883,7 +1874,6 @@ impl SimilarAggregator {
             max_attributes_to_retrieve,
             show_ranking_score,
             show_ranking_score_details,
-            embedder,
             ranking_score_threshold,
             retrieve_vectors,
         } = other;
@@ -1914,7 +1904,6 @@ impl SimilarAggregator {
             *used_syntax = used_syntax.saturating_add(value);
         }
 
-        self.embedder |= embedder;
         self.retrieve_vectors |= retrieve_vectors;
 
         // pagination
@@ -1948,7 +1937,6 @@ impl SimilarAggregator {
             max_attributes_to_retrieve,
             show_ranking_score,
             show_ranking_score_details,
-            embedder,
             ranking_score_threshold,
             retrieve_vectors,
         } = self;
@@ -1980,9 +1968,6 @@ impl SimilarAggregator {
                 "vector": {
                     "retrieve_vectors": retrieve_vectors,
                 },
-                "hybrid": {
-                    "embedder": embedder,
-                },
                 "pagination": {
                    "max_limit": max_limit,
                    "max_offset": max_offset,

meilisearch/src/error.rs

@@ -72,7 +72,7 @@ pub enum MeilisearchHttpError {
     DocumentFormat(#[from] DocumentFormatError),
     #[error(transparent)]
     Join(#[from] JoinError),
-    #[error("Invalid request: missing `hybrid` parameter when both `q` and `vector` are present.")]
+    #[error("Invalid request: missing `hybrid` parameter when `vector` is present.")]
     MissingSearchHybrid,
 }
 

meilisearch/src/routes/indexes/search.rs

@@ -128,8 +128,10 @@ impl std::ops::Deref for SemanticRatioGet {
     }
 }
 
-impl From<SearchQueryGet> for SearchQuery {
-    fn from(other: SearchQueryGet) -> Self {
+impl TryFrom<SearchQueryGet> for SearchQuery {
+    type Error = ResponseError;
+
+    fn try_from(other: SearchQueryGet) -> Result<Self, Self::Error> {
         let filter = match other.filter {
             Some(f) => match serde_json::from_str(&f) {
                 Ok(v) => Some(v),
@@ -140,19 +142,28 @@ impl From<SearchQueryGet> for SearchQuery {
 
         let hybrid = match (other.hybrid_embedder, other.hybrid_semantic_ratio) {
             (None, None) => None,
-            (None, Some(semantic_ratio)) => {
-                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder: None })
+            (None, Some(_)) => {
+                return Err(ResponseError::from_msg(
+                    "`hybridEmbedder` is mandatory when `hybridSemanticRatio` is present".into(),
+                    meilisearch_types::error::Code::InvalidHybridQuery,
+                ));
+            }
+            (Some(embedder), None) => {
+                Some(HybridQuery { semantic_ratio: DEFAULT_SEMANTIC_RATIO(), embedder })
             }
-            (Some(embedder), None) => Some(HybridQuery {
-                semantic_ratio: DEFAULT_SEMANTIC_RATIO(),
-                embedder: Some(embedder),
-            }),
             (Some(embedder), Some(semantic_ratio)) => {
-                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder: Some(embedder) })
+                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder })
             }
         };
 
-        Self {
+        if other.vector.is_some() && hybrid.is_none() {
+            return Err(ResponseError::from_msg(
+                "`hybridEmbedder` is mandatory when `vector` is present".into(),
+                meilisearch_types::error::Code::MissingSearchHybrid,
+            ));
+        }
+
+        Ok(Self {
             q: other.q,
             vector: other.vector.map(CS::into_inner),
             offset: other.offset.0,
@@ -179,7 +190,7 @@ impl From<SearchQueryGet> for SearchQuery {
             hybrid,
             ranking_score_threshold: other.ranking_score_threshold.map(|o| o.0),
             locales: other.locales.map(|o| o.into_iter().collect()),
-        }
+        })
     }
 }
 
@@ -219,7 +230,7 @@ pub async fn search_with_url_query(
     debug!(parameters = ?params, "Search get");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
-    let mut query: SearchQuery = params.into_inner().into();
+    let mut query: SearchQuery = params.into_inner().try_into()?;
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
@@ -312,44 +323,36 @@ pub fn search_kind(
         features.check_vector("Passing `hybrid` as a parameter")?;
     }
 
-    // regardless of anything, always do a keyword search when we don't have a vector and the query is whitespace or missing
-    if query.vector.is_none() {
-        match &query.q {
-            Some(q) if q.trim().is_empty() => return Ok(SearchKind::KeywordOnly),
-            None => return Ok(SearchKind::KeywordOnly),
-            _ => {}
+    // handle with care, the order of cases matters, the semantics is subtle
+    match (query.q.as_deref(), &query.hybrid, query.vector.as_deref()) {
+        // empty query, no vector => placeholder search
+        (Some(q), _, None) if q.trim().is_empty() => Ok(SearchKind::KeywordOnly),
+        // no query, no vector => placeholder search
+        (None, _, None) => Ok(SearchKind::KeywordOnly),
+        // hybrid.semantic_ratio == 1.0 => vector
+        (_, Some(HybridQuery { semantic_ratio, embedder }), v) if **semantic_ratio == 1.0 => {
+            SearchKind::semantic(index_scheduler, index, embedder, v.map(|v| v.len()))
         }
-    }
-
-    match &query.hybrid {
-        Some(HybridQuery { semantic_ratio, embedder }) if **semantic_ratio == 1.0 => {
-            Ok(SearchKind::semantic(
-                index_scheduler,
-                index,
-                embedder.as_deref(),
-                query.vector.as_ref().map(Vec::len),
-            )?)
-        }
-        Some(HybridQuery { semantic_ratio, embedder: _ }) if **semantic_ratio == 0.0 => {
+        // hybrid.semantic_ratio == 0.0 => keyword
+        (_, Some(HybridQuery { semantic_ratio, embedder: _ }), _) if **semantic_ratio == 0.0 => {
             Ok(SearchKind::KeywordOnly)
         }
-        Some(HybridQuery { semantic_ratio, embedder }) => Ok(SearchKind::hybrid(
+        // no query, hybrid, vector => semantic
+        (None, Some(HybridQuery { semantic_ratio: _, embedder }), Some(v)) => {
+            SearchKind::semantic(index_scheduler, index, embedder, Some(v.len()))
+        }
+        // query, no hybrid, no vector => keyword
+        (Some(_), None, None) => Ok(SearchKind::KeywordOnly),
+        // query, hybrid, maybe vector => hybrid
+        (Some(_), Some(HybridQuery { semantic_ratio, embedder }), v) => SearchKind::hybrid(
             index_scheduler,
             index,
-            embedder.as_deref(),
+            embedder,
             **semantic_ratio,
-            query.vector.as_ref().map(Vec::len),
-        )?),
-        None => match (query.q.as_deref(), query.vector.as_deref()) {
-            (_query, None) => Ok(SearchKind::KeywordOnly),
-            (None, Some(_vector)) => Ok(SearchKind::semantic(
-                index_scheduler,
-                index,
-                None,
-                query.vector.as_ref().map(Vec::len),
-            )?),
-            (Some(_), Some(_)) => Err(MeilisearchHttpError::MissingSearchHybrid.into()),
-        },
+            v.map(|v| v.len()),
+        ),
+
+        (_, None, Some(_)) => Err(MeilisearchHttpError::MissingSearchHybrid.into()),
     }
 }
 

meilisearch/src/routes/indexes/similar.rs

@@ -103,7 +103,7 @@ async fn similar(
     let index = index_scheduler.index(&index_uid)?;
 
     let (embedder_name, embedder) =
-        SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
+        SearchKind::embedder(&index_scheduler, &index, &query.embedder, None)?;
 
     tokio::task::spawn_blocking(move || {
         perform_similar(
@@ -139,8 +139,8 @@ pub struct SimilarQueryGet {
     show_ranking_score_details: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSimilarRankingScoreThreshold>, default)]
     pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
-    #[deserr(default, error = DeserrQueryParamError<InvalidEmbedder>)]
-    pub embedder: Option<String>,
+    #[deserr(error = DeserrQueryParamError<InvalidEmbedder>)]
+    pub embedder: String,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]

meilisearch/src/search/mod.rs

@@ -267,8 +267,8 @@ impl fmt::Debug for SearchQuery {
 pub struct HybridQuery {
     #[deserr(default, error = DeserrJsonError<InvalidSearchSemanticRatio>, default)]
     pub semantic_ratio: SemanticRatio,
-    #[deserr(default, error = DeserrJsonError<InvalidEmbedder>, default)]
-    pub embedder: Option<String>,
+    #[deserr(error = DeserrJsonError<InvalidEmbedder>)]
+    pub embedder: String,
 }
 
 #[derive(Clone)]
@@ -282,7 +282,7 @@ impl SearchKind {
     pub(crate) fn semantic(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
-        embedder_name: Option<&str>,
+        embedder_name: &str,
         vector_len: Option<usize>,
     ) -> Result<Self, ResponseError> {
         let (embedder_name, embedder) =
@@ -293,7 +293,7 @@ impl SearchKind {
     pub(crate) fn hybrid(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
-        embedder_name: Option<&str>,
+        embedder_name: &str,
         semantic_ratio: f32,
         vector_len: Option<usize>,
     ) -> Result<Self, ResponseError> {
@@ -305,14 +305,12 @@ impl SearchKind {
     pub(crate) fn embedder(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
-        embedder_name: Option<&str>,
+        embedder_name: &str,
         vector_len: Option<usize>,
     ) -> Result<(String, Arc<Embedder>), ResponseError> {
         let embedder_configs = index.embedding_configs(&index.read_txn()?)?;
         let embedders = index_scheduler.embedders(embedder_configs)?;
 
-        let embedder_name = embedder_name.unwrap_or_else(|| embedders.get_default_embedder_name());
-
         let embedder = embedders.get(embedder_name);
 
         let embedder = embedder
@@ -538,8 +536,8 @@ pub struct SimilarQuery {
     pub limit: usize,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarFilter>)]
     pub filter: Option<Value>,
-    #[deserr(default, error = DeserrJsonError<InvalidEmbedder>, default)]
-    pub embedder: Option<String>,
+    #[deserr(error = DeserrJsonError<InvalidEmbedder>)]
+    pub embedder: String,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarRetrieveVectors>)]