mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-12-23 21:20:24 +01:00
Merge #134
134: fix(http, index): init analyzer with optional stop words r=MarinPostma a=shekhirin Also bump `milli` and `meilisearch-tokenizer` packages versions Co-authored-by: Alexey Shekhirin <a.shekhirin@gmail.com>
This commit is contained in:
commit
dd1a08087b
28
Cargo.lock
generated
28
Cargo.lock
generated
@ -1847,7 +1847,7 @@ dependencies = [
|
||||
"log",
|
||||
"main_error",
|
||||
"meilisearch-error",
|
||||
"meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?branch=main)",
|
||||
"meilisearch-tokenizer",
|
||||
"memmap",
|
||||
"milli",
|
||||
"mime",
|
||||
@ -1893,22 +1893,6 @@ dependencies = [
|
||||
"whatlang",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "meilisearch-tokenizer"
|
||||
version = "0.1.1"
|
||||
source = "git+https://github.com/meilisearch/Tokenizer.git?branch=main#31ba3ff4a15501f12b7d37ac64ddce7c35a9757c"
|
||||
dependencies = [
|
||||
"character_converter",
|
||||
"cow-utils",
|
||||
"deunicode",
|
||||
"fst",
|
||||
"jieba-rs",
|
||||
"once_cell",
|
||||
"slice-group-by",
|
||||
"unicode-segmentation",
|
||||
"whatlang",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.4"
|
||||
@ -1937,7 +1921,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "milli"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/meilisearch/milli.git?rev=2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c"
|
||||
source = "git+https://github.com/meilisearch/milli.git?tag=v0.1.0#2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bstr",
|
||||
@ -1957,7 +1941,7 @@ dependencies = [
|
||||
"linked-hash-map",
|
||||
"log",
|
||||
"logging_timer",
|
||||
"meilisearch-tokenizer 0.1.1 (git+https://github.com/meilisearch/Tokenizer.git?tag=v0.2.0)",
|
||||
"meilisearch-tokenizer",
|
||||
"memmap",
|
||||
"num-traits",
|
||||
"obkv",
|
||||
@ -2252,7 +2236,8 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.1.3"
|
||||
source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
|
||||
dependencies = [
|
||||
"ucd-trie",
|
||||
]
|
||||
@ -2260,8 +2245,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "pest"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
|
||||
source = "git+https://github.com/pest-parser/pest.git?rev=51fd1d49f1041f7839975664ef71fe15c7dcaf67#51fd1d49f1041f7839975664ef71fe15c7dcaf67"
|
||||
dependencies = [
|
||||
"ucd-trie",
|
||||
]
|
||||
|
@ -40,9 +40,9 @@ itertools = "0.10.0"
|
||||
log = "0.4.8"
|
||||
main_error = "0.1.0"
|
||||
meilisearch-error = { path = "../meilisearch-error" }
|
||||
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", branch = "main" }
|
||||
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.0" }
|
||||
memmap = "0.7.0"
|
||||
milli = { git = "https://github.com/meilisearch/milli.git", rev = "2bcdd8844c4ec9f6f8a34617ea0e4321fa633c0c" }
|
||||
milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.1.0" }
|
||||
mime = "0.3.16"
|
||||
once_cell = "1.5.2"
|
||||
parking_lot = "0.11.1"
|
||||
|
@ -155,7 +155,10 @@ pub struct Highlighter<'a, A> {
|
||||
|
||||
impl<'a, A: AsRef<[u8]>> Highlighter<'a, A> {
|
||||
pub fn new(stop_words: &'a fst::Set<A>) -> Self {
|
||||
let analyzer = Analyzer::new(AnalyzerConfig::default_with_stopwords(stop_words));
|
||||
let mut config = AnalyzerConfig::default();
|
||||
config.stop_words(stop_words);
|
||||
|
||||
let analyzer = Analyzer::new(config);
|
||||
|
||||
Self { analyzer }
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user