mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-06-12 07:39:50 +02:00
16 lines
322 B
Rust
16 lines
322 B
Rust
mod dfa;
|
|
|
|
use meilisearch_tokenizer::is_cjk;
|
|
|
|
pub use self::dfa::{build_dfa, build_prefix_dfa, build_exact_dfa};
|
|
|
|
pub fn normalize_str(string: &str) -> String {
|
|
let mut string = string.to_lowercase();
|
|
|
|
if !string.contains(is_cjk) {
|
|
string = deunicode::deunicode_with_tofu(&string, "");
|
|
}
|
|
|
|
string
|
|
}
|