searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 20:37:15 +02:00
Code Issues Releases Wiki Activity

highlight with new tokenizer

Browse source
This commit is contained in:
mpostma 2020-12-23 20:04:19 +01:00
parent 1ae761311e
commit 4f7f7538f7
No known key found for this signature in database
GPG key ID: CBC8A7C1D7A28C3A
6 changed files with 49 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/query_tokens.rs
View file

@ -54,8 +54,8 @@ mod tests {
match $test {
Quoted(val) => assert_eq!(val.text(), $val),
Free(val) => panic!("expected Quoted(\"{}\"), found Free(\"{}\")", $val, val.text()),
}
};
}
};
($test:expr, Free($val:literal)) => {
match $test {

8
src/update/index_documents/store.rs
View file

@ -8,21 +8,21 @@ use std::{cmp, iter};
use anyhow::{bail, Context};
use bstr::ByteSlice as _;
use fst::Set;
use grenad::{Reader, FileFuse, Writer, Sorter, CompressionType};
use heed::BytesEncode;
use linked_hash_map::LinkedHashMap;
use log::{debug, info};
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig};
use ordered_float::OrderedFloat;
use roaring::RoaringBitmap;
use serde_json::Value;
use tempfile::tempfile;
use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, TokenKind};
use fst::Set;
use crate::facet::FacetType;
use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec};
use crate::heed_codec::facet::{FacetValueStringCodec, FacetLevelValueF64Codec, FacetLevelValueI64Codec};
use crate::heed_codec::facet::{FieldDocIdFacetStringCodec, FieldDocIdFacetF64Codec, FieldDocIdFacetI64Codec};
use crate::heed_codec::{BoRoaringBitmapCodec, CboRoaringBitmapCodec};
use crate::update::UpdateIndexingStep;
use crate::{json_to_string, SmallVec8, SmallVec32, SmallString32, Position, DocumentId, FieldId};
@ -167,7 +167,7 @@ impl<'s, A: AsRef<[u8]>> Store<'s, A> {
// MTBL writers
docid_word_positions_writer,
documents_writer,
//tokenizer
// tokenizer
analyzer,
})
}