restore synonyms tests

meilisearch-core/src/database.rs

@@ -882,7 +882,6 @@ mod tests {
 
         // even try to search for a document
         let reader = db.main_read_txn().unwrap();
-        println!("here");
         let SortResult {documents, .. } = index.query_builder().query(&reader, Some("21 "), 0..20).unwrap();
         assert_matches!(documents.len(), 1);
 

meilisearch-core/src/raw_indexer.rs

@@ -147,17 +147,17 @@ fn process_tokens<'a>(tokens: impl Iterator<Item = Token<'a>>) -> impl Iterator<
                 match token.kind {
                     TokenKind::Word | TokenKind::StopWord | TokenKind::Any => {
                         *offset += match *sepkind {
-                            Some(SeparatorKind::Hard) => 8,
-                            Some(SeparatorKind::Soft) => 1,
+                            Some(TokenKind::Separator(SeparatorKind::Hard)) => 8,
+                            Some(_) => 1,
                             None => 0,
                         };
-                        *sepkind = None;
+                        *sepkind = Some(token.kind)
                     }
                     TokenKind::Separator(SeparatorKind::Hard) => {
-                        *sepkind = Some(SeparatorKind::Hard);
+                        *sepkind = Some(token.kind);
                     }
                     TokenKind::Separator(SeparatorKind::Soft) if sepkind.is_none() => {
-                        *sepkind = Some(SeparatorKind::Soft);
+                        *sepkind = Some(token.kind);
                     }
                     _ => (),
                 }