mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-23 21:34:27 +01:00
22 lines
554 B
Rust
22 lines
554 B
Rust
|
use slice_group_by::StrGroupBy;
|
||
|
|
|
||
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||
|
|
pub enum TokenType {
|
||
|
|
Word,
|
||
|
|
Space,
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn simple_tokenizer(text: &str) -> impl Iterator<Item=(TokenType, &str)> {
|
||
|
|
text
|
||
|
|
.linear_group_by_key(|c| c.is_alphanumeric())
|
||
|
|
.map(|s| {
|
||
|
|
let first = s.chars().next().unwrap();
|
||
|
|
let type_ = if first.is_alphanumeric() { TokenType::Word } else { TokenType::Space };
|
||
|
|
(type_, s)
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
pub fn only_words((t, _): &(TokenType, &str)) -> bool {
|
||
|
|
*t == TokenType::Word
|
||
|
|
}
|