Merge #183

183: Add cropping and update `_formatted` behavior r=curquiza a=MarinPostma

TODO:
- [x] Solves #5 
- [x] Solves #203 
- [x] integrate the new milli highlight (according to the query words)

Co-authored-by: Marin Postma <postma.marin@protonmail.com>
Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

meilisearch-http/src/index/search.rs

@@ -1,13 +1,12 @@
 use std::borrow::Cow;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashSet};
 use std::time::Instant;
 
 use anyhow::bail;
 use either::Either;
 use heed::RoTxn;
 use indexmap::IndexMap;
-use itertools::Itertools;
+use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
-use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
 use milli::{FilterCondition, FieldId, FieldsIdsMap, MatchingWords};
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -17,11 +16,15 @@ use super::Index;
 pub type Document = IndexMap<String, Value>;
 
 pub const DEFAULT_SEARCH_LIMIT: usize = 20;
-
 const fn default_search_limit() -> usize {
     DEFAULT_SEARCH_LIMIT
 }
 
+pub const DEFAULT_CROP_LENGTH: usize = 200;
+const fn default_crop_length() -> usize {
+    DEFAULT_CROP_LENGTH
+}
+
 #[derive(Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct SearchQuery {
@@ -29,9 +32,10 @@ pub struct SearchQuery {
     pub offset: Option<usize>,
     #[serde(default = "default_search_limit")]
     pub limit: usize,
-    pub attributes_to_retrieve: Option<HashSet<String>>,
+    pub attributes_to_retrieve: Option<BTreeSet<String>>,
-    pub attributes_to_crop: Option<HashSet<String>>,
+    pub attributes_to_crop: Option<Vec<String>>,
-    pub crop_length: Option<usize>,
+    #[serde(default = "default_crop_length")]
+    pub crop_length: usize,
     pub attributes_to_highlight: Option<HashSet<String>>,
     pub matches: Option<bool>,
     pub filter: Option<Value>,
@@ -60,6 +64,12 @@ pub struct SearchResult {
     pub facet_distributions: Option<BTreeMap<String, BTreeMap<String, u64>>>,
 }
 
+#[derive(Copy, Clone)]
+struct FormatOptions {
+    highlight: bool,
+    crop: Option<usize>,
+}
+
 impl Index {
     pub fn perform_search(&self, query: SearchQuery) -> anyhow::Result<SearchResult> {
         let before_search = Instant::now();
@@ -75,7 +85,7 @@ impl Index {
         search.offset(query.offset.unwrap_or_default());
 
         if let Some(ref filter) = query.filter {
-            if let Some(facets) = parse_facets(filter, self, &rtxn)? {
+            if let Some(facets) = parse_filter(filter, self, &rtxn)? {
                 search.filter(facets);
             }
         }
@@ -91,11 +101,11 @@ impl Index {
 
         let displayed_ids = self
             .displayed_fields_ids(&rtxn)?
-            .map(|fields| fields.into_iter().collect::<HashSet<_>>())
+            .map(|fields| fields.into_iter().collect::<BTreeSet<_>>())
             .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
 
-        let fids = |attrs: &HashSet<String>| {
+        let fids = |attrs: &BTreeSet<String>| {
-            let mut ids = HashSet::new();
+            let mut ids = BTreeSet::new();
             for attr in attrs {
                 if attr == "*" {
                     ids = displayed_ids.clone();
@@ -109,65 +119,53 @@ impl Index {
             ids
         };
 
-        let to_retrieve_ids = query
+        // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default),
+        // but these attributes must be also be present
+        // - in the fields_ids_map
+        // - in the the displayed attributes
+        let to_retrieve_ids: BTreeSet<_> = query
             .attributes_to_retrieve
             .as_ref()
             .map(fids)
-            .unwrap_or_else(|| displayed_ids.clone());
+            .unwrap_or_else(|| displayed_ids.clone())
-
-        let to_highlight_ids = query
-            .attributes_to_highlight
-            .as_ref()
-            .map(fids)
-            .unwrap_or_default();
-
-        let to_crop_ids = query
-            .attributes_to_crop
-            .as_ref()
-            .map(fids)
-            .unwrap_or_default();
-
-        // The attributes to retrieve are:
-        // - the ones explicitly marked as to retrieve that are also in the displayed attributes
-        let all_attributes: Vec<_> = to_retrieve_ids
             .intersection(&displayed_ids)
             .cloned()
-            .sorted()
             .collect();
 
-        // The formatted attributes are:
+        let attr_to_highlight = query
-        // - The one in either highlighted attributes or cropped attributes if there are attributes
+            .attributes_to_highlight
-        // to retrieve
+            .unwrap_or_default();
-        // - All the attributes to retrieve if there are either highlighted or cropped attributes
+
-        // the request specified that all attributes are to retrieve (i.e attributes to retrieve is
+        let attr_to_crop = query
-        // empty in the query)
+            .attributes_to_crop
-        let all_formatted = if query.attributes_to_retrieve.is_none() {
+            .unwrap_or_default();
-            if query.attributes_to_highlight.is_some() || query.attributes_to_crop.is_some() {
+
-                Cow::Borrowed(&all_attributes)
+        // Attributes in `formatted_options` correspond to the attributes that will be in `_formatted`
-            } else {
+        // These attributes are:
-                Cow::Owned(Vec::new())
+        // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`)
-            }
+        // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped
-        } else {
+        // But these attributes must be also present in displayed attributes
-            let attrs = (&to_crop_ids | &to_highlight_ids)
+        let formatted_options = compute_formatted_options(
-                .intersection(&displayed_ids)
+            &attr_to_highlight,
-                .cloned()
+            &attr_to_crop,
-                .collect::<Vec<_>>();
+            query.crop_length,
-            Cow::Owned(attrs)
+            &to_retrieve_ids,
-        };
+            &fields_ids_map,
+            &displayed_ids,
+        );
 
         let stop_words = fst::Set::default();
-        let highlighter =
+        let formatter =
-            Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
 
         for (_id, obkv) in self.documents(&rtxn, documents_ids)? {
-            let document = make_document(&all_attributes, &fields_ids_map, obkv)?;
+            let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?;
-            let formatted = compute_formatted(
+            let formatted = format_fields(
                 &fields_ids_map,
                 obkv,
-                &highlighter,
+                &formatter,
                 &matching_words,
-                all_formatted.as_ref().as_slice(),
+                &formatted_options,
-                &to_highlight_ids,
             )?;
             let hit = SearchHit {
                 document,
@@ -203,8 +201,129 @@ impl Index {
     }
 }
 
+fn compute_formatted_options(
+    attr_to_highlight: &HashSet<String>,
+    attr_to_crop: &[String],
+    query_crop_length: usize,
+    to_retrieve_ids: &BTreeSet<u8>,
+    fields_ids_map: &FieldsIdsMap,
+    displayed_ids: &BTreeSet<u8>,
+    ) -> BTreeMap<FieldId, FormatOptions> {
+
+    let mut formatted_options = BTreeMap::new();
+
+    add_highlight_to_formatted_options(
+        &mut formatted_options,
+        attr_to_highlight,
+        fields_ids_map,
+        displayed_ids,
+    );
+
+    add_crop_to_formatted_options(
+        &mut formatted_options,
+        attr_to_crop,
+        query_crop_length,
+        fields_ids_map,
+        displayed_ids,
+    );
+
+    // Should not return `_formatted` if no valid attributes to highlight/crop
+    if !formatted_options.is_empty() {
+        add_non_formatted_ids_to_formatted_options(
+            &mut formatted_options,
+            to_retrieve_ids,
+        );
+    }
+
+    formatted_options
+}
+
+fn add_highlight_to_formatted_options(
+    formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
+    attr_to_highlight: &HashSet<String>,
+    fields_ids_map: &FieldsIdsMap,
+    displayed_ids: &BTreeSet<u8>,
+) {
+    for attr in attr_to_highlight {
+        let new_format = FormatOptions {
+            highlight: true,
+            crop: None,
+        };
+
+        if attr == "*" {
+            for id in displayed_ids {
+                formatted_options.insert(*id, new_format);
+            }
+            break;
+        }
+
+        if let Some(id) = fields_ids_map.id(&attr) {
+            if displayed_ids.contains(&id) {
+                formatted_options.insert(id, new_format);
+            }
+        }
+    }
+}
+
+fn add_crop_to_formatted_options(
+    formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
+    attr_to_crop: &[String],
+    crop_length: usize,
+    fields_ids_map: &FieldsIdsMap,
+    displayed_ids: &BTreeSet<u8>,
+) {
+    for attr in attr_to_crop {
+        let mut split = attr.rsplitn(2, ':');
+        let (attr_name, attr_len) = match split.next().zip(split.next()) {
+            Some((len, name)) => {
+                let crop_len = len.parse::<usize>().unwrap_or(crop_length);
+                (name, crop_len)
+            },
+            None => (attr.as_str(), crop_length),
+        };
+
+        if attr_name == "*" {
+            for id in displayed_ids {
+                formatted_options
+                    .entry(*id)
+                    .and_modify(|f| f.crop = Some(attr_len))
+                    .or_insert(FormatOptions {
+                        highlight: false,
+                        crop: Some(attr_len),
+                    });
+            }
+        }
+
+        if let Some(id) = fields_ids_map.id(&attr_name) {
+            if displayed_ids.contains(&id) {
+                formatted_options
+                    .entry(id)
+                    .and_modify(|f| f.crop = Some(attr_len))
+                    .or_insert(FormatOptions {
+                        highlight: false,
+                        crop: Some(attr_len),
+                    });
+            }
+        }
+    }
+}
+
+fn add_non_formatted_ids_to_formatted_options(
+    formatted_options: &mut BTreeMap<FieldId, FormatOptions>,
+    to_retrieve_ids: &BTreeSet<u8>
+) {
+    for id in to_retrieve_ids {
+        formatted_options
+            .entry(*id)
+            .or_insert(FormatOptions {
+                highlight: false,
+                crop: None,
+            });
+    }
+}
+
 fn make_document(
-    attributes_to_retrieve: &[FieldId],
+    attributes_to_retrieve: &BTreeSet<FieldId>,
     field_ids_map: &FieldsIdsMap,
     obkv: obkv::KvReader,
 ) -> anyhow::Result<Document> {
@@ -226,28 +345,29 @@ fn make_document(
     Ok(document)
 }
 
-fn compute_formatted<A: AsRef<[u8]>>(
+fn format_fields<A: AsRef<[u8]>>(
     field_ids_map: &FieldsIdsMap,
     obkv: obkv::KvReader,
-    highlighter: &Highlighter<A>,
+    formatter: &Formatter<A>,
     matching_words: &impl Matcher,
-    all_formatted: &[FieldId],
+    formatted_options: &BTreeMap<FieldId, FormatOptions>,
-    to_highlight_ids: &HashSet<FieldId>,
 ) -> anyhow::Result<Document> {
     let mut document = Document::new();
 
-    for field in all_formatted {
+    for (id, format) in formatted_options {
-        if let Some(value) = obkv.get(*field) {
+        if let Some(value) = obkv.get(*id) {
             let mut value: Value = serde_json::from_slice(value)?;
 
-            if to_highlight_ids.contains(field) {
+            value = formatter.format_value(
-                value = highlighter.highlight_value(value, matching_words);
+                value,
-            }
+                matching_words,
+                *format,
+            );
 
             // This unwrap must be safe since we got the ids from the fields_ids_map just
             // before.
             let key = field_ids_map
-                .name(*field)
+                .name(*id)
                 .expect("Missing field name")
                 .to_string();
 
@@ -258,30 +378,30 @@ fn compute_formatted<A: AsRef<[u8]>>(
     Ok(document)
 }
 
-/// trait to allow unit testing of `compute_formated`
+/// trait to allow unit testing of `format_fields`
 trait Matcher {
-    fn matches(&self, w: &str) -> bool;
+    fn matches(&self, w: &str) -> Option<usize>;
 }
 
 #[cfg(test)]
-impl Matcher for HashSet<String> {
+impl Matcher for BTreeMap<&str, Option<usize>> {
-    fn matches(&self, w: &str) -> bool {
+    fn matches(&self, w: &str) -> Option<usize> {
-        self.contains(w)
+        self.get(w).cloned().flatten()
     }
 }
 
 impl Matcher for MatchingWords {
-    fn matches(&self, w: &str) -> bool {
+    fn matches(&self, w: &str) -> Option<usize> {
-        self.matching_bytes(w).is_some()
+        self.matching_bytes(w)
     }
 }
 
-struct Highlighter<'a, A> {
+struct Formatter<'a, A> {
     analyzer: Analyzer<'a, A>,
     marks: (String, String),
 }
 
-impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
+impl<'a, A: AsRef<[u8]>> Formatter<'a, A> {
     pub fn new(stop_words: &'a fst::Set<A>, marks: (String, String)) -> Self {
         let mut config = AnalyzerConfig::default();
         config.stop_words(stop_words);
@@ -291,59 +411,121 @@ impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
         Self { analyzer, marks }
     }
 
-    fn highlight_value(&self, value: Value, words_to_highlight: &impl Matcher) -> Value {
+    fn format_value(
+        &self,
+        value: Value,
+        matcher: &impl Matcher,
+        format_options: FormatOptions,
+    ) -> Value {
         match value {
-            Value::Null => Value::Null,
-            Value::Bool(boolean) => Value::Bool(boolean),
-            Value::Number(number) => Value::Number(number),
             Value::String(old_string) => {
-                let mut string = String::new();
+                let value =
-                let analyzed = self.analyzer.analyze(&old_string);
+                    self.format_string(old_string, matcher, format_options);
-                for (word, token) in analyzed.reconstruct() {
+                Value::String(value)
-                    if token.is_word() {
-                        let to_highlight = words_to_highlight.matches(token.text());
-                        if to_highlight {
-                            string.push_str(&self.marks.0)
-                        }
-                        string.push_str(word);
-                        if to_highlight {
-                            string.push_str(&self.marks.1)
-                        }
-                    } else {
-                        string.push_str(word);
-                    }
-                }
-                Value::String(string)
             }
             Value::Array(values) => Value::Array(
                 values
                     .into_iter()
-                    .map(|v| self.highlight_value(v, words_to_highlight))
+                    .map(|v| self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None }))
                     .collect(),
             ),
             Value::Object(object) => Value::Object(
                 object
                     .into_iter()
-                    .map(|(k, v)| (k, self.highlight_value(v, words_to_highlight)))
+                    .map(|(k, v)| (k, self.format_value(v, matcher, FormatOptions { highlight: format_options.highlight, crop: None })))
                     .collect(),
             ),
-        }
+            value => value,
         }
     }
 
-fn parse_facets(
+    fn format_string(
+        &self,
+        s: String,
+        matcher: &impl Matcher,
+        format_options: FormatOptions,
+    ) -> String {
+        let analyzed = self.analyzer.analyze(&s);
+
+        let tokens: Box<dyn Iterator<Item = (&str, Token)>> = match format_options.crop {
+            Some(crop_len) => {
+                let mut buffer = Vec::new();
+                let mut tokens = analyzed.reconstruct().peekable();
+
+                while let Some((word, token)) = tokens.next_if(|(_, token)| matcher.matches(token.text()).is_none()) {
+                    buffer.push((word, token));
+                }
+
+                match tokens.next() {
+                    Some(token) => {
+                        let mut total_len: usize = buffer.iter().map(|(word, _)| word.len()).sum();
+                        let before_iter = buffer.into_iter().skip_while(move |(word, _)| {
+                            total_len -= word.len();
+                            total_len >= crop_len
+                        });
+
+                        let mut taken_after = 0;
+                        let after_iter = tokens
+                        .take_while(move |(word, _)| {
+                            let take = taken_after < crop_len;
+                            taken_after += word.chars().count();
+                            take
+                        });
+
+                        let iter = before_iter
+                            .chain(Some(token))
+                            .chain(after_iter);
+
+                        Box::new(iter)
+                    },
+                    // If no word matches in the attribute
+                    None => {
+                        let mut count = 0;
+                        let iter = buffer.into_iter().take_while(move |(word, _)| {
+                            let take = count < crop_len;
+                            count += word.len();
+                            take
+                        });
+
+                        Box::new(iter)
+                    }
+                }
+            }
+            None => Box::new(analyzed.reconstruct()),
+        };
+
+        tokens
+            .map(|(word, token)| {
+                if format_options.highlight && token.is_word() && matcher.matches(token.text()).is_some() {
+                    let mut new_word = String::new();
+                    new_word.push_str(&self.marks.0);
+                    if let Some(match_len) = matcher.matches(token.text()) {
+                        new_word.push_str(&word[..match_len]);
+                        new_word.push_str(&self.marks.1);
+                        new_word.push_str(&word[match_len..]);
+                    }
+                    Cow::Owned(new_word)
+                } else {
+                    Cow::Borrowed(word)
+                }
+            })
+            .collect::<String>()
+    }
+}
+
+fn parse_filter(
     facets: &Value,
     index: &Index,
     txn: &RoTxn,
 ) -> anyhow::Result<Option<FilterCondition>> {
     match facets {
         Value::String(expr) => Ok(Some(FilterCondition::from_str(txn, index, expr)?)),
-        Value::Array(arr) => parse_facets_array(txn, index, arr),
+        Value::Array(arr) => parse_filter_array(txn, index, arr),
         v => bail!("Invalid facet expression, expected Array, found: {:?}", v),
     }
 }
 
-fn parse_facets_array(
+fn parse_filter_array(
     txn: &RoTxn,
     index: &Index,
     arr: &[Value],
@@ -374,15 +556,13 @@ fn parse_facets_array(
 
 #[cfg(test)]
 mod test {
-    use std::iter::FromIterator;
-
     use super::*;
 
     #[test]
-    fn no_formatted() {
+    fn no_ids_no_formatted() {
         let stop_words = fst::Set::default();
-        let highlighter =
+        let formatter =
-            Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
 
         let mut fields = FieldsIdsMap::new();
         let id = fields.insert("test").unwrap();
@@ -395,18 +575,16 @@ mod test {
 
         let obkv = obkv::KvReader::new(&buf);
 
-        let all_formatted = Vec::new();
+        let formatted_options = BTreeMap::new();
-        let to_highlight_ids = HashSet::new();
 
         let matching_words = MatchingWords::default();
 
-        let value = compute_formatted(
+        let value = format_fields(
             &fields,
             obkv,
-            &highlighter,
+            &formatter,
             &matching_words,
-            &all_formatted,
+            &formatted_options,
-            &to_highlight_ids,
         )
         .unwrap();
 
@@ -414,72 +592,296 @@ mod test {
     }
 
     #[test]
-    fn formatted_no_highlight() {
+    fn formatted_with_highlight_in_word() {
         let stop_words = fst::Set::default();
-        let highlighter =
+        let formatter =
-            Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
 
         let mut fields = FieldsIdsMap::new();
-        let id = fields.insert("test").unwrap();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
 
         let mut buf = Vec::new();
         let mut obkv = obkv::KvWriter::new(&mut buf);
-        obkv.insert(id, Value::String("hello".into()).to_string().as_bytes())
+        obkv.insert(title, Value::String("The Hobbit".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. R. R. Tolkien".into()).to_string().as_bytes())
             .unwrap();
         obkv.finish().unwrap();
 
         let obkv = obkv::KvReader::new(&buf);
 
-        let all_formatted = vec![id];
+        let mut formatted_options = BTreeMap::new();
-        let to_highlight_ids = HashSet::new();
+        formatted_options.insert(title, FormatOptions { highlight: true, crop: None });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
 
-        let matching_words = MatchingWords::default();
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("hobbit", Some(3));
 
-        let value = compute_formatted(
+        let value = format_fields(
             &fields,
             obkv,
-            &highlighter,
+            &formatter,
             &matching_words,
-            &all_formatted,
+            &formatted_options,
-            &to_highlight_ids,
         )
         .unwrap();
 
-        assert_eq!(value["test"], "hello");
+        assert_eq!(value["title"], "The <em>Hob</em>bit");
+        assert_eq!(value["author"], "J. R. R. Tolkien");
     }
 
     #[test]
-    fn formatted_with_highlight() {
+    fn formatted_with_crop_2() {
         let stop_words = fst::Set::default();
-        let highlighter =
+        let formatter =
-            Highlighter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
 
         let mut fields = FieldsIdsMap::new();
-        let id = fields.insert("test").unwrap();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
 
         let mut buf = Vec::new();
         let mut obkv = obkv::KvWriter::new(&mut buf);
-        obkv.insert(id, Value::String("hello".into()).to_string().as_bytes())
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
             .unwrap();
         obkv.finish().unwrap();
 
         let obkv = obkv::KvReader::new(&buf);
 
-        let all_formatted = vec![id];
+        let mut formatted_options = BTreeMap::new();
-        let to_highlight_ids = HashSet::from_iter(Some(id));
+        formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(2) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
 
-        let matching_words = HashSet::from_iter(Some(String::from("hello")));
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("potter", Some(6));
 
-        let value = compute_formatted(
+        let value = format_fields(
             &fields,
             obkv,
-            &highlighter,
+            &formatter,
             &matching_words,
-            &all_formatted,
+            &formatted_options,
-            &to_highlight_ids,
         )
         .unwrap();
 
-        assert_eq!(value["test"], "<em>hello</em>");
+        assert_eq!(value["title"], "Harry Potter and");
+        assert_eq!(value["author"], "J. K. Rowling");
+    }
+
+    #[test]
+    fn formatted_with_crop_10() {
+        let stop_words = fst::Set::default();
+        let formatter =
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(10) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("potter", Some(6));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], "Harry Potter and the Half");
+        assert_eq!(value["author"], "J. K. Rowling");
+    }
+
+    #[test]
+    fn formatted_with_crop_0() {
+        let stop_words = fst::Set::default();
+        let formatter =
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(0) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("potter", Some(6));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], "Potter");
+        assert_eq!(value["author"], "J. K. Rowling");
+    }
+
+    #[test]
+    fn formatted_with_crop_and_no_match() {
+        let stop_words = fst::Set::default();
+        let formatter =
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(title, FormatOptions { highlight: false, crop: Some(6) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: Some(20) });
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("rowling", Some(3));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], "Harry ");
+        assert_eq!(value["author"], "J. K. Rowling");
+    }
+
+    #[test]
+    fn formatted_with_crop_and_highlight() {
+        let stop_words = fst::Set::default();
+        let formatter =
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(1) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("and", Some(3));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], " <em>and</em> ");
+        assert_eq!(value["author"], "J. K. Rowling");
+    }
+
+    #[test]
+    fn formatted_with_crop_and_highlight_in_word() {
+        let stop_words = fst::Set::default();
+        let formatter =
+            Formatter::new(&stop_words, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(title, Value::String("Harry Potter and the Half-Blood Prince".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+        obkv.insert(author, Value::String("J. K. Rowling".into()).to_string().as_bytes())
+            .unwrap();
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(title, FormatOptions { highlight: true, crop: Some(9) });
+        formatted_options.insert(author, FormatOptions { highlight: false, crop: None });
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("blood", Some(3));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], "the Half-<em>Blo</em>od Prince");
+        assert_eq!(value["author"], "J. K. Rowling");
     }
 }

meilisearch-http/src/index_controller/index_actor/message.rs

@@ -8,6 +8,7 @@ use crate::index_controller::{Failed, IndexStats, Processed, Processing};
 
 use super::{IndexMeta, IndexResult, IndexSettings};
 
+#[allow(clippy::large_enum_variant)]
 pub enum IndexMsg {
     CreateIndex {
         uuid: Uuid,

meilisearch-http/src/routes/search.rs

@@ -1,4 +1,4 @@
-use std::collections::HashSet;
+use std::collections::{BTreeSet, HashSet};
 use std::convert::{TryFrom, TryInto};
 
 use actix_web::{get, post, web, HttpResponse};
@@ -23,7 +23,7 @@ pub struct SearchQueryGet {
     limit: Option<usize>,
     attributes_to_retrieve: Option<String>,
     attributes_to_crop: Option<String>,
-    crop_length: Option<usize>,
+    crop_length: usize,
     attributes_to_highlight: Option<String>,
     filter: Option<String>,
     matches: Option<bool>,
@@ -36,11 +36,11 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
     fn try_from(other: SearchQueryGet) -> anyhow::Result<Self> {
         let attributes_to_retrieve = other
             .attributes_to_retrieve
-            .map(|attrs| attrs.split(',').map(String::from).collect::<HashSet<_>>());
+            .map(|attrs| attrs.split(',').map(String::from).collect::<BTreeSet<_>>());
 
         let attributes_to_crop = other
             .attributes_to_crop
-            .map(|attrs| attrs.split(',').map(String::from).collect::<HashSet<_>>());
+            .map(|attrs| attrs.split(',').map(String::from).collect::<Vec<_>>());
 
         let attributes_to_highlight = other
             .attributes_to_highlight