Introduce a better query and document lexer

src/bin/indexer.rs

@@ -9,6 +9,7 @@ use std::time::Instant;
 
 use anyhow::Context;
 use arc_cache::ArcCache;
+use bstr::ByteSlice as _;
 use cow_utils::CowUtils;
 use fst::IntoStreamer;
 use heed::EnvOpenOptions;
@@ -18,12 +19,11 @@ use memmap::Mmap;
 use oxidized_mtbl::{Reader, Writer, Merger, Sorter, CompressionType};
 use rayon::prelude::*;
 use roaring::RoaringBitmap;
-use slice_group_by::StrGroupBy;
 use structopt::StructOpt;
 
-use milli::{SmallVec32, Index, DocumentId, Position, Attribute};
+use milli::{lexer, SmallVec32, Index, DocumentId, Position, Attribute};
 
-const LMDB_MAX_KEY_LENGTH: usize = 512;
+const LMDB_MAX_KEY_LENGTH: usize = 511;
 const ONE_MILLION: usize = 1_000_000;
 
 const MAX_POSITION: usize = 1000;
@@ -39,11 +39,6 @@ const WORD_ATTRIBUTE_DOCIDS_BYTE: u8 = 3;
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 
-pub fn simple_alphanumeric_tokens(string: &str) -> impl Iterator<Item = &str> {
-    let is_alphanumeric = |s: &&str| s.chars().next().map_or(false, char::is_alphanumeric);
-    string.linear_group_by_key(|c| c.is_alphanumeric()).filter(is_alphanumeric)
-}
-
 #[derive(Debug, StructOpt)]
 #[structopt(name = "milli-indexer", about = "The indexer binary of the milli project.")]
 struct Opt {
@@ -345,7 +340,7 @@ where F: FnMut(&[u8], &[u8]) -> anyhow::Result<()>
     let mut iter = merger.into_merge_iter()?;
     while let Some(result) = iter.next() {
         let (k, v) = result?;
-        (f)(&k, &v)?;
+        (f)(&k, &v).with_context(|| format!("writing {:?} {:?} into LMDB", k.as_bstr(), k.as_bstr()))?;
     }
 
     debug!("MTBL stores merged in {:.02?}!", before.elapsed());
@@ -389,7 +384,7 @@ fn index_csv(
         }
 
         for (attr, content) in document.iter().enumerate().take(MAX_ATTRIBUTES) {
-            for (pos, word) in simple_alphanumeric_tokens(&content).enumerate().take(MAX_POSITION) {
+            for (pos, word) in lexer::break_string(&content).enumerate().take(MAX_POSITION) {
                 let word = word.cow_to_lowercase();
                 let position = (attr * MAX_POSITION + pos) as u32;
                 store.insert_word_position(&word, position)?;

src/lexer.rs

@@ -0,0 +1,44 @@
+use unicode_linebreak::{linebreaks, BreakClass, break_property};
+
+fn can_be_broken(c: char) -> bool {
+    use BreakClass::*;
+
+    match break_property(c as u32) {
+          Ideographic
+        | Alphabetic
+        | Numeric
+        | CombiningMark
+        | WordJoiner
+        | NonBreakingGlue
+        | OpenPunctuation
+        | Symbol
+        | EmojiBase
+        | EmojiModifier
+        | HangulLJamo
+        | HangulVJamo
+        | HangulTJamo
+        | RegionalIndicator
+        | Quotation => false,
+        _ => true,
+    }
+}
+
+fn extract_token(s: &str) -> &str {
+    let end = s.char_indices().rev()
+        .take_while(|(_, c)| can_be_broken(*c))
+        .last()
+        .map(|(i, _)| i)
+        .unwrap_or(s.len());
+
+    &s[..end]
+}
+
+pub fn break_string(s: &str) -> impl Iterator<Item = &str> {
+    let mut prev = 0;
+    linebreaks(&s).map(move |(i, _)| {
+        let s = &s[prev..i];
+        prev = i;
+        extract_token(s)
+    })
+    .filter(|s| !s.is_empty())
+}

src/lib.rs

@@ -4,6 +4,7 @@ mod node;
 mod query_tokens;
 mod search;
 mod transitive_arc;
+pub mod lexer;
 
 use std::collections::HashMap;
 use std::fs::{File, OpenOptions};

src/query_tokens.rs

@@ -1,4 +1,5 @@
 use std::{mem, str};
+use unicode_linebreak::{break_property, BreakClass};
 
 use QueryToken::{Quoted, Free};
 
@@ -8,6 +9,7 @@ pub enum QueryToken<'a> {
     Quoted(&'a str),
 }
 
+#[derive(Debug)]
 enum State {
     Free(usize),
     Quoted(usize),
@@ -67,8 +69,13 @@ impl<'a> Iterator for QueryTokens<'a> {
                     },
                     State::Fused => return None,
                 }
-            }
-            else if !self.state.is_quoted() && !c.is_alphanumeric() {
+            } else if break_property(c as u32) == BreakClass::Ideographic {
+                match self.state.replace_by(State::Free(afteri)) {
+                    State::Quoted(s) => return Some(Quoted(&self.string[s..afteri])),
+                    State::Free(s) => return Some(Free(&self.string[s..afteri])),
+                    _ => self.state = State::Free(afteri),
+                }
+            } else if !self.state.is_quoted() && !c.is_alphanumeric() {
                 match self.state.replace_by(State::Free(afteri)) {
                     State::Free(s) if i > s => return Some(Free(&self.string[s..i])),
                     _ => self.state = State::Free(afteri),
@@ -83,6 +90,15 @@ mod tests {
     use super::*;
     use QueryToken::{Quoted, Free};
 
+    #[test]
+    fn empty() {
+        let mut iter = QueryTokens::new("");
+        assert_eq!(iter.next(), None);
+
+        let mut iter = QueryTokens::new(" ");
+        assert_eq!(iter.next(), None);
+    }
+
     #[test]
     fn one_quoted_string() {
         let mut iter = QueryTokens::new("\"hello\"");
@@ -154,4 +170,14 @@ mod tests {
         assert_eq!(iter.next(), Some(Quoted("monde est beau")));
         assert_eq!(iter.next(), None);
     }
+
+    #[test]
+    fn chinese() {
+        let mut iter = QueryTokens::new("汽车男生");
+        assert_eq!(iter.next(), Some(Free("汽")));
+        assert_eq!(iter.next(), Some(Free("车")));
+        assert_eq!(iter.next(), Some(Free("男")));
+        assert_eq!(iter.next(), Some(Free("生")));
+        assert_eq!(iter.next(), None);
+    }
 }

src/search.rs

@@ -217,6 +217,10 @@ impl<'a> Search<'a> {
             None => return Ok(Default::default()),
         };
 
+        if dfas.is_empty() {
+            return Ok(Default::default());
+        }
+
         let (derived_words, union_positions) = Self::fetch_words_positions(rtxn, index, &fst, dfas)?;
         let candidates = Self::compute_candidates(rtxn, index, &derived_words)?;