remove accept new fields / add indexed *

meilisearch-core/src/settings.rs

@@ -29,8 +29,6 @@ pub struct Settings {
     #[serde(default, deserialize_with = "deserialize_some")]
     pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
     #[serde(default, deserialize_with = "deserialize_some")]
-    pub accept_new_fields: Option<Option<bool>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
     pub attributes_for_faceting: Option<Option<Vec<String>>>,
 }
 
@@ -60,7 +58,6 @@ impl Settings {
             displayed_attributes: settings.displayed_attributes.into(),
             stop_words: settings.stop_words.into(),
             synonyms: settings.synonyms.into(),
-            accept_new_fields: settings.accept_new_fields.into(),
             attributes_for_faceting: settings.attributes_for_faceting.into(),
         })
     }
@@ -167,7 +164,6 @@ pub struct SettingsUpdate {
     pub displayed_attributes: UpdateState<HashSet<String>>,
     pub stop_words: UpdateState<BTreeSet<String>>,
     pub synonyms: UpdateState<BTreeMap<String, Vec<String>>>,
-    pub accept_new_fields: UpdateState<bool>,
     pub attributes_for_faceting: UpdateState<Vec<String>>,
 }
 
@@ -181,7 +177,6 @@ impl Default for SettingsUpdate {
             displayed_attributes: UpdateState::Nothing,
             stop_words: UpdateState::Nothing,
             synonyms: UpdateState::Nothing,
-            accept_new_fields: UpdateState::Nothing,
             attributes_for_faceting: UpdateState::Nothing,
         }
     }

meilisearch-core/src/update/settings_update.rs

@@ -68,16 +68,6 @@ pub fn apply_settings_update(
         UpdateState::Nothing => (),
     }
 
-    match settings.accept_new_fields {
-        UpdateState::Update(v) => {
-            schema.set_accept_new_fields(v);
-        },
-        UpdateState::Clear => {
-            schema.set_accept_new_fields(true);
-        },
-        UpdateState::Nothing => (),
-    }
-
     match settings.searchable_attributes.clone() {
         UpdateState::Update(v) => {
             if v.len() == 1 && v[0] == "*" {

meilisearch-http/src/routes/setting.rs

@@ -1,6 +1,7 @@
 use actix_web::{web, HttpResponse};
 use actix_web_macros::{delete, get, post};
 use meilisearch_core::settings::{Settings, SettingsUpdate, UpdateState, DEFAULT_RANKING_RULES};
+use meilisearch_schema::Schema;
 use std::collections::{BTreeMap, BTreeSet, HashSet};
 
 use crate::error::{Error, ResponseError};
@@ -24,8 +25,6 @@ pub fn services(cfg: &mut web::ServiceConfig) {
         .service(get_displayed)
         .service(update_displayed)
         .service(delete_displayed)
-        .service(get_accept_new_fields)
-        .service(update_accept_new_fields)
         .service(get_attributes_for_faceting)
         .service(delete_attributes_for_faceting)
         .service(update_attributes_for_faceting);
@@ -108,23 +107,8 @@ async fn get_all(
         _ => vec![],
     };
 
-    println!("{:?}", attributes_for_faceting);
+    let searchable_attributes = schema.as_ref().map(get_indexed_attributes);
-
+    let displayed_attributes = schema.as_ref().map(get_displayed_attributes);
-    let searchable_attributes = schema.as_ref().map(|s| {
-        s.indexed_name()
-            .iter()
-            .map(|s| s.to_string())
-            .collect::<Vec<String>>()
-    });
-
-    let displayed_attributes = schema.as_ref().map(|s| {
-        s.displayed_name()
-            .iter()
-            .map(|s| s.to_string())
-            .collect::<HashSet<String>>()
-    });
-
-    let accept_new_fields = schema.map(|s| s.accept_new_fields());
 
     let settings = Settings {
         ranking_rules: Some(Some(ranking_rules)),
@@ -133,7 +117,6 @@ async fn get_all(
         displayed_attributes: Some(displayed_attributes),
         stop_words: Some(Some(stop_words)),
         synonyms: Some(Some(synonyms)),
-        accept_new_fields: Some(accept_new_fields),
         attributes_for_faceting: Some(Some(attributes_for_faceting)),
     };
 
@@ -158,7 +141,6 @@ async fn delete_all(
         displayed_attributes: UpdateState::Clear,
         stop_words: UpdateState::Clear,
         synonyms: UpdateState::Clear,
-        accept_new_fields: UpdateState::Clear,
         attributes_for_faceting: UpdateState::Clear,
     };
 
@@ -326,7 +308,7 @@ async fn get_searchable(
     let reader = data.db.main_read_txn()?;
     let schema = index.main.schema(&reader)?;
     let searchable_attributes: Option<Vec<String>> =
-        schema.map(|s| s.indexed_name().iter().map(|i| i.to_string()).collect());
+        schema.as_ref().map(get_indexed_attributes);
 
     Ok(HttpResponse::Ok().json(searchable_attributes))
 }
@@ -396,8 +378,7 @@ async fn get_displayed(
 
     let schema = index.main.schema(&reader)?;
 
-    let displayed_attributes: Option<HashSet<String>> =
+    let displayed_attributes = schema.as_ref().map(get_displayed_attributes);
-        schema.map(|s| s.displayed_name().iter().map(|i| i.to_string()).collect());
 
     Ok(HttpResponse::Ok().json(displayed_attributes))
 }
@@ -450,52 +431,6 @@ async fn delete_displayed(
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
 
-#[get(
-    "/indexes/{index_uid}/settings/accept-new-fields",
-    wrap = "Authentication::Private"
-)]
-async fn get_accept_new_fields(
-    data: web::Data<Data>,
-    path: web::Path<IndexParam>,
-) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
-    let reader = data.db.main_read_txn()?;
-
-    let schema = index.main.schema(&reader)?;
-
-    let accept_new_fields = schema.map(|s| s.accept_new_fields());
-
-    Ok(HttpResponse::Ok().json(accept_new_fields))
-}
-
-#[post(
-    "/indexes/{index_uid}/settings/accept-new-fields",
-    wrap = "Authentication::Private"
-)]
-async fn update_accept_new_fields(
-    data: web::Data<Data>,
-    path: web::Path<IndexParam>,
-    body: web::Json<Option<bool>>,
-) -> Result<HttpResponse, ResponseError> {
-    let index = data
-        .db
-        .open_index(&path.index_uid)
-        .ok_or(Error::index_not_found(&path.index_uid))?;
-
-    let settings = Settings {
-        accept_new_fields: Some(body.into_inner()),
-        ..Settings::default()
-    };
-
-    let settings = settings.to_update().map_err(Error::bad_request)?;
-    let update_id = data.db.update_write(|w| index.settings_update(w, settings))?;
-
-    Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
-}
-
 #[get(
     "/indexes/{index_uid}/settings/attributes-for-faceting",
     wrap = "Authentication::Private"
@@ -577,3 +512,25 @@ async fn delete_attributes_for_faceting(
 
     Ok(HttpResponse::Accepted().json(IndexUpdateResponse::with_id(update_id)))
 }
+
+fn get_indexed_attributes(schema: &Schema) -> Vec<String> {
+    if schema.is_indexed_all() {
+        ["*"].iter().map(|s| s.to_string()).collect()
+    } else {
+        schema.indexed_name()
+            .iter()
+            .map(|s| s.to_string())
+            .collect::<Vec<String>>()
+    }
+}
+
+fn get_displayed_attributes(schema: &Schema) -> HashSet<String> {
+    if schema.is_displayed_all() {
+        ["*"].iter().map(|s| s.to_string()).collect()
+    } else {
+        schema.displayed_name()
+            .iter()
+            .map(|s| s.to_string())
+            .collect::<HashSet<String>>()
+    }
+}

meilisearch-schema/src/schema.rs

@@ -15,6 +15,21 @@ impl<T> OptionAll<T> {
     fn take(&mut self) -> OptionAll<T> {
         std::mem::replace(self, OptionAll::None)
     }
+
+    fn map<U, F: FnOnce(T) -> U>(self, f: F) -> OptionAll<U> {
+        match self {
+            OptionAll::Some(x) => OptionAll::Some(f(x)),
+            OptionAll::All => OptionAll::All,
+            OptionAll::None => OptionAll::None,
+        }
+    }
+
+    pub fn is_all(&self) -> bool {
+        match self {
+            OptionAll::All => true,
+            _ => false,
+        }
+    }
 }
 
 impl<T> Default for OptionAll<T> {
@@ -31,18 +46,13 @@ pub struct Schema {
     ranked: HashSet<FieldId>,
     displayed: OptionAll<HashSet<FieldId>>,
 
-    indexed: Vec<FieldId>,
+    indexed: OptionAll<Vec<FieldId>>,
     indexed_map: HashMap<FieldId, IndexedPos>,
-
-    accept_new_fields: bool,
 }
 
 impl Schema {
     pub fn new() -> Schema {
-        Schema {
+        Schema::default()
-            accept_new_fields: true,
-            ..Default::default()
-        }
     }
 
     pub fn with_primary_key(name: &str) -> Schema {
@@ -50,11 +60,9 @@ impl Schema {
         let field_id = fields_map.insert(name).unwrap();
 
         let mut displayed = HashSet::new();
-        let mut indexed = Vec::new();
         let mut indexed_map = HashMap::new();
 
         displayed.insert(field_id);
-        indexed.push(field_id);
         indexed_map.insert(field_id, 0.into());
 
         Schema {
@@ -62,9 +70,8 @@ impl Schema {
             primary_key: Some(field_id),
             ranked: HashSet::new(),
             displayed: OptionAll::All,
-            indexed,
+            indexed: OptionAll::All,
             indexed_map,
-            accept_new_fields: true,
         }
     }
 
@@ -79,10 +86,8 @@ impl Schema {
 
         let id = self.insert(name)?;
         self.primary_key = Some(id);
-        if self.accept_new_fields {
+        self.set_indexed(name)?;
-            self.set_indexed(name)?;
+        self.set_displayed(name)?;
-            self.set_displayed(name)?;
-        }
 
         Ok(id)
     }
@@ -113,12 +118,8 @@ impl Schema {
                 Ok(id)
             }
             None => {
-                if self.accept_new_fields {
+                self.set_indexed(name)?;
-                    self.set_indexed(name)?;
+                self.set_displayed(name)
-                    self.set_displayed(name)
-                } else {
-                    self.fields_map.insert(name)
-                }
             }
         }
     }
@@ -147,34 +148,34 @@ impl Schema {
     }
 
     pub fn is_displayed_all(&self) -> bool {
-        match self.displayed {
+        self.displayed.is_all()
-            OptionAll::All => true,
-            _ => false,
-        }
     }
 
     pub fn displayed_name(&self) -> HashSet<&str> {
         match self.displayed {
-            OptionAll::Some(ref v) => {
+            OptionAll::All => self.fields_map.iter().filter_map(|(_, &v)| self.name(v)).collect(),
-                v.iter().filter_map(|a| self.name(*a)).collect()
+            OptionAll::Some(ref v) => v.iter().filter_map(|a| self.name(*a)).collect(),
-            }
+            OptionAll::None => HashSet::new(),
-            OptionAll::All => {
-                self
-                    .fields_map
-                    .iter()
-                    .filter_map(|(_, &v)| self.name(v))
-                    .collect::<HashSet<_>>()
-            }
-            OptionAll::None => HashSet::new()
         }
     }
 
-    pub fn indexed(&self) -> &Vec<FieldId> {
+    pub fn indexed(&self) -> Cow<[FieldId]> {
-        &self.indexed
+        match self.indexed {
+            OptionAll::Some(ref v) => Cow::Borrowed(v),
+            OptionAll::All => {
+                let fields = self
+                    .fields_map
+                    .iter()
+                    .map(|(_, &f)| f)
+                    .collect();
+                Cow::Owned(fields)
+            },
+            OptionAll::None => Cow::Owned(Vec::new())
+        }
     }
 
     pub fn indexed_name(&self) -> Vec<&str> {
-        self.indexed.iter().filter_map(|a| self.name(*a)).collect()
+        self.indexed().iter().filter_map(|a| self.name(*a)).collect()
     }
 
     pub fn set_ranked(&mut self, name: &str) -> SResult<FieldId> {
@@ -186,7 +187,8 @@ impl Schema {
     pub fn set_displayed(&mut self, name: &str) -> SResult<FieldId> {
         let id = self.fields_map.insert(name)?;
         self.displayed = match self.displayed.take() {
-            OptionAll::All | OptionAll::None => {
+            OptionAll::All => OptionAll::All,
+            OptionAll::None => {
                 let mut displayed = HashSet::new();
                 displayed.insert(id);
                 OptionAll::Some(displayed)
@@ -201,12 +203,16 @@ impl Schema {
 
     pub fn set_indexed(&mut self, name: &str) -> SResult<(FieldId, IndexedPos)> {
         let id = self.fields_map.insert(name)?;
+
         if let Some(indexed_pos) = self.indexed_map.get(&id) {
             return Ok((id, *indexed_pos))
         };
-        let pos = self.indexed.len() as u16;
+        let pos = self.indexed_map.len() as u16;
-        self.indexed.push(id);
         self.indexed_map.insert(id, pos.into());
+        self.indexed = self.indexed.take().map(|mut v| {
+            v.push(id);
+            v
+        });
         Ok((id, pos.into()))
     }
 
@@ -251,7 +257,16 @@ impl Schema {
     pub fn remove_indexed(&mut self, name: &str) {
         if let Some(id) = self.fields_map.id(name) {
             self.indexed_map.remove(&id);
-            self.indexed.retain(|x| *x != id);
+            self.indexed = match self.indexed.take() {
+                // valid because indexed is All and indexed() return the content of
+                // indexed_map that is already updated
+                OptionAll::All => OptionAll::Some(self.indexed().into_owned()),
+                OptionAll::Some(mut v) => {
+                    v.retain(|x| *x != id);
+                    OptionAll::Some(v)
+                }
+                OptionAll::None => OptionAll::None,
+            }
         }
     }
 
@@ -273,13 +288,17 @@ impl Schema {
         self.indexed_map.get(&id)
     }
 
+    pub fn is_indexed_all(&self) -> bool {
+        self.indexed.is_all()
+    }
+
     pub fn indexed_pos_to_field_id<I: Into<IndexedPos>>(&self, pos: I) -> Option<FieldId> {
-        let indexed_pos = pos.into().0 as usize;
+        let indexed_pos = pos.into().0;
-        if indexed_pos < self.indexed.len() {
+        self
-            Some(self.indexed[indexed_pos as usize])
+            .indexed_map
-        } else {
+            .iter()
-            None
+            .find(|(_, &v)| v.0 == indexed_pos)
-        }
+            .map(|(&k, _)| k)
     }
 
     pub fn update_ranked<S: AsRef<str>>(&mut self, data: impl IntoIterator<Item = S>) -> SResult<()> {
@@ -291,9 +310,13 @@ impl Schema {
     }
 
     pub fn update_displayed<S: AsRef<str>>(&mut self, data: impl IntoIterator<Item = S>) -> SResult<()> {
-        if let OptionAll::Some(ref mut v) = self.displayed {
+        self.displayed = match self.displayed.take() {
-            v.clear()
+            OptionAll::Some(mut v) => {
-        }
+                v.clear();
+                OptionAll::Some(v)
+            }
+            _ => OptionAll::Some(HashSet::new())
+        };
         for name in data {
             self.set_displayed(name.as_ref())?;
         }
@@ -301,7 +324,13 @@ impl Schema {
     }
 
     pub fn update_indexed<S: AsRef<str>>(&mut self, data: Vec<S>) -> SResult<()> {
-        self.indexed.clear();
+        self.indexed = match self.indexed.take() {
+            OptionAll::Some(mut v) => {
+                v.clear();
+                OptionAll::Some(v)
+            },
+            _ => OptionAll::Some(Vec::new()),
+        };
         self.indexed_map.clear();
         for name in data {
             self.set_indexed(name.as_ref())?;
@@ -310,12 +339,11 @@ impl Schema {
     }
 
     pub fn set_all_fields_as_indexed(&mut self) {
-        self.indexed.clear();
+        self.indexed = OptionAll::All;
         self.indexed_map.clear();
 
         for (_name, id) in self.fields_map.iter() {
-            let pos = self.indexed.len() as u16;
+            let pos = self.indexed_map.len() as u16;
-            self.indexed.push(*id);
             self.indexed_map.insert(*id, pos.into());
         }
     }
@@ -323,12 +351,4 @@ impl Schema {
     pub fn set_all_fields_as_displayed(&mut self) {
         self.displayed = OptionAll::All
     }
-
-    pub fn accept_new_fields(&self) -> bool {
-        self.accept_new_fields
-    }
-
-    pub fn set_accept_new_fields(&mut self, value: bool) {
-        self.accept_new_fields = value;
-    }
 }