mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-05 20:48:58 +01:00
Merge #628
628: Make sure that long words are ignored r=ManyTheFish a=Kerollmops This PR is related to https://github.com/meilisearch/meilisearch/issues/2743 and is fixing it. Co-authored-by: Kerollmops <clement@meilisearch.com>
This commit is contained in:
commit
2907928d93
@ -7,7 +7,9 @@ use charabia::{SeparatorKind, Token, TokenKind, TokenizerBuilder};
|
|||||||
use roaring::RoaringBitmap;
|
use roaring::RoaringBitmap;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
|
||||||
use super::helpers::{concat_u32s_array, create_sorter, sorter_into_reader, GrenadParameters};
|
use super::helpers::{
|
||||||
|
concat_u32s_array, create_sorter, sorter_into_reader, GrenadParameters, MAX_WORD_LENGTH,
|
||||||
|
};
|
||||||
use crate::error::{InternalError, SerializationError};
|
use crate::error::{InternalError, SerializationError};
|
||||||
use crate::{absolute_from_relative_position, FieldId, Result, MAX_POSITION_PER_ATTRIBUTE};
|
use crate::{absolute_from_relative_position, FieldId, Result, MAX_POSITION_PER_ATTRIBUTE};
|
||||||
|
|
||||||
@ -68,7 +70,7 @@ pub fn extract_docid_word_positions<R: io::Read + io::Seek>(
|
|||||||
|
|
||||||
for (index, token) in tokens {
|
for (index, token) in tokens {
|
||||||
let token = token.lemma().trim();
|
let token = token.lemma().trim();
|
||||||
if !token.is_empty() {
|
if !token.is_empty() && token.len() <= MAX_WORD_LENGTH {
|
||||||
key_buffer.truncate(mem::size_of::<u32>());
|
key_buffer.truncate(mem::size_of::<u32>());
|
||||||
key_buffer.extend_from_slice(token.as_bytes());
|
key_buffer.extend_from_slice(token.as_bytes());
|
||||||
|
|
||||||
|
@ -18,8 +18,11 @@ pub use merge_functions::{
|
|||||||
roaring_bitmap_from_u32s_array, serialize_roaring_bitmap, MergeFn,
|
roaring_bitmap_from_u32s_array, serialize_roaring_bitmap, MergeFn,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/// The maximum length a word can be
|
||||||
|
pub const MAX_WORD_LENGTH: usize = 250;
|
||||||
|
|
||||||
pub fn valid_lmdb_key(key: impl AsRef<[u8]>) -> bool {
|
pub fn valid_lmdb_key(key: impl AsRef<[u8]>) -> bool {
|
||||||
key.as_ref().len() <= 511 && !key.as_ref().is_empty()
|
key.as_ref().len() <= MAX_WORD_LENGTH * 2 && !key.as_ref().is_empty()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Divides one slice into two at an index, returns `None` if mid is out of bounds.
|
/// Divides one slice into two at an index, returns `None` if mid is out of bounds.
|
||||||
|
@ -1741,4 +1741,22 @@ mod tests {
|
|||||||
index.add_documents(doc3).unwrap_err();
|
index.add_documents(doc3).unwrap_err();
|
||||||
index.add_documents(doc4).unwrap_err();
|
index.add_documents(doc4).unwrap_err();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn long_words_must_be_skipped() {
|
||||||
|
let index = TempIndex::new();
|
||||||
|
|
||||||
|
// this is obviousy too long
|
||||||
|
let long_word = "lol".repeat(1000);
|
||||||
|
let doc1 = documents! {[{
|
||||||
|
"id": "1",
|
||||||
|
"title": long_word.clone(),
|
||||||
|
}]};
|
||||||
|
|
||||||
|
index.add_documents(doc1).unwrap();
|
||||||
|
|
||||||
|
let rtxn = index.read_txn().unwrap();
|
||||||
|
let words_fst = index.words_fst(&rtxn).unwrap();
|
||||||
|
assert!(!words_fst.contains(&long_word));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user