searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-04 12:27:13 +02:00
Code Issues Releases Wiki Activity

feat: Replace compressed Match fields by uncompressed ones

Browse source
This commit is contained in:
Clément Renault 2019-02-02 14:17:50 +01:00
parent b0b3175641
commit a3a28c56fa
No known key found for this signature in database
GPG key ID: 0151CDAB43460DAE
5 changed files with 56 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/data/doc_indexes.rs
View file

@ -158,18 +158,24 @@ mod tests {
fn builder_serialize_deserialize() -> Result<(), Box<Error>> {
let a = DocIndex {
document_id: DocumentId(0),
attribute: Attribute::new_faillible(3, 11),
word_area: WordArea::new_faillible(30, 4)
attribute: 3,
word_index: 11,
char_index: 30,
char_length: 4,
};
let b = DocIndex {
document_id: DocumentId(1),
attribute: Attribute::new_faillible(4, 21),
word_area: WordArea::new_faillible(35, 6)
attribute: 4,
word_index: 21,
char_index: 35,
char_length: 6,
};
let c = DocIndex {
document_id: DocumentId(2),
attribute: Attribute::new_faillible(8, 2),
word_area: WordArea::new_faillible(89, 6)
attribute: 8,
word_index: 2,
char_index: 89,
char_length: 6,
};
let mut builder = DocIndexesBuilder::memory();
@ -193,18 +199,24 @@ mod tests {
fn serialize_deserialize() -> Result<(), Box<Error>> {
let a = DocIndex {
document_id: DocumentId(0),
attribute: Attribute::new_faillible(3, 11),
word_area: WordArea::new_faillible(30, 4)
attribute: 3,
word_index: 11,
char_index: 30,
char_length: 4,
};
let b = DocIndex {
document_id: DocumentId(1),
attribute: Attribute::new_faillible(4, 21),
word_area: WordArea::new_faillible(35, 6)
attribute: 4,
word_index: 21,
char_index: 35,
char_length: 6,
};
let c = DocIndex {
document_id: DocumentId(2),
attribute: Attribute::new_faillible(8, 2),
word_area: WordArea::new_faillible(89, 6)
attribute: 8,
word_index: 2,
char_index: 89,
char_length: 6,
};
let mut builder = DocIndexesBuilder::memory();

22
src/database/serde/indexer_serializer.rs
View file

@ -54,10 +54,8 @@ where B: TokenizerBuilder
let document_id = self.document_id;
// FIXME must u32::try_from instead
let attribute = match Attribute::new(self.attribute.0, word_index as u32) {
Ok(attribute) => attribute,
Err(_) => return Ok(()),
};
let attribute = self.attribute.0;
let word_index = word_index as u32;
// insert the exact representation
let word_lower = word.to_lowercase();
@ -68,21 +66,17 @@ where B: TokenizerBuilder
// and the unidecoded lowercased version
let word_unidecoded = unidecode::unidecode(word).to_lowercase();
if word_lower != word_unidecoded {
let word_area = match WordArea::new(char_index as u32, length) {
Ok(word_area) => word_area,
Err(_) => return Ok(()),
};
let char_index = char_index as u32;
let char_length = length;
let doc_index = DocIndex { document_id, attribute, word_area };
let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length };
self.update.insert_doc_index(word_unidecoded.into_bytes(), doc_index);
}
let word_area = match WordArea::new(char_index as u32, length) {
Ok(word_area) => word_area,
Err(_) => return Ok(()),
};
let char_index = char_index as u32;
let char_length = length;
let doc_index = DocIndex { document_id, attribute, word_area };
let doc_index = DocIndex { document_id, attribute, word_index, char_index, char_length };
self.update.insert_doc_index(word_lower.into_bytes(), doc_index);
}
Ok(())

26
src/lib.rs
View file

@ -36,14 +36,16 @@ pub struct DocIndex {
/// The attribute in the document where the word was found
/// along with the index in it.
pub attribute: Attribute,
pub attribute: u16,
pub word_index: u32,
/// The position in bytes where the word was found
/// along with the length of it.
///
/// It informs on the original word area in the text indexed
/// without needing to run the tokenizer again.
pub word_area: WordArea,
pub char_index: u32,
pub char_length: u16,
}
/// This structure represent a matching word with informations
@ -68,7 +70,8 @@ pub struct Match {
/// The attribute in the document where the word was found
/// along with the index in it.
pub attribute: Attribute,
pub attribute: u16,
pub word_index: u32,
/// Whether the word that match is an exact match or a prefix.
pub is_exact: bool,
@ -78,7 +81,8 @@ pub struct Match {
///
/// It informs on the original word area in the text indexed
/// without needing to run the tokenizer again.
pub word_area: WordArea,
pub char_index: u32,
pub char_length: u16,
}
impl Match {
@ -86,9 +90,11 @@ impl Match {
Match {
query_index: 0,
distance: 0,
attribute: Attribute::new_faillible(0, 0),
attribute: 0,
word_index: 0,
is_exact: false,
word_area: WordArea::new_faillible(0, 0),
char_index: 0,
char_length: 0,
}
}
@ -96,9 +102,11 @@ impl Match {
Match {
query_index: u32::max_value(),
distance: u8::max_value(),
attribute: Attribute::max_value(),
attribute: u16::max_value(),
word_index: u32::max_value(),
is_exact: true,
word_area: WordArea::max_value(),
char_index: u32::max_value(),
char_length: u16::max_value(),
}
}
}
@ -110,6 +118,6 @@ mod tests {
#[test]
fn docindex_mem_size() {
assert_eq!(mem::size_of::<DocIndex>(), 16);
assert_eq!(mem::size_of::<DocIndex>(), 24);
}
}

4
src/rank/query_builder.rs
View file

@ -111,8 +111,10 @@ where D: Deref<Target=DB>,
query_index: iv.index as u32,
distance: distance,
attribute: doc_index.attribute,
word_index: doc_index.word_index,
is_exact: is_exact,
word_area: doc_index.word_area,
char_index: doc_index.char_index,
char_length: doc_index.char_length,
};
matches.entry(doc_index.document_id).or_insert_with(Vec::new).push(match_);
}