searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-15 13:58:36 +02:00
Code Issues Releases Wiki Activity

Compute the biggest values of the words_level_positions_docids

Browse source
This commit is contained in:
Kerollmops 2021-03-17 16:09:18 +01:00 • committed by many
parent f713828406
commit 8bd4f5d93e
No known key found for this signature in database
GPG key ID: 2CEF23B75189EACA
2 changed files with 20 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

18
infos/src/main.rs
View file

@ -346,6 +346,7 @@ fn biggest_value_sizes(index: &Index, rtxn: &heed::RoTxn, limit: usize) -> anyho
let docid_word_positions_name = "docid_word_positions";
let word_prefix_pair_proximity_docids_name = "word_prefix_pair_proximity_docids";
let word_pair_proximity_docids_name = "word_pair_proximity_docids";
let word_level_position_docids_name = "word_level_position_docids";
let facet_field_id_value_docids_name = "facet_field_id_value_docids";
let documents_name = "documents";
@ -402,6 +403,13 @@ fn biggest_value_sizes(index: &Index, rtxn: &heed::RoTxn, limit: usize) -> anyho
if heap.len() > limit { heap.pop(); }
}
for result in word_level_position_docids.remap_data_type::<ByteSlice>().iter(rtxn)? {
let ((word, level, left, right), value) = result?;
let key = format!("{} {} {:?}", word, level, left..=right);
heap.push(Reverse((value.len(), key, word_level_position_docids_name)));
if heap.len() > limit { heap.pop(); }
}
let faceted_fields = index.faceted_fields_ids(rtxn)?;
let fields_ids_map = index.fields_ids_map(rtxn)?;
for (field_id, field_type) in faceted_fields {
@ -549,7 +557,7 @@ fn words_level_positions_docids(
{
let stdout = io::stdout();
let mut wtr = csv::Writer::from_writer(stdout.lock());
wtr.write_record(&["word", "level", "position_range", "documents_count", "documents_ids"])?;
wtr.write_record(&["word", "level", "positions", "documents_count", "documents_ids"])?;
for word in words.iter().map(AsRef::as_ref) {
let range = {
@ -561,14 +569,18 @@ fn words_level_positions_docids(
let ((w, level, left, right), docids) = result?;
if word != w { break }
let level = level.to_string();
let count = docids.len().to_string();
let docids = if debug {
format!("{:?}", docids)
} else {
format!("{:?}", docids.iter().collect::<Vec<_>>())
};
let position_range = format!("{:?}", left..=right);
let position_range = if level == 0 {
format!("{:?}", left)
} else {
format!("{:?}", left..=right)
};
let level = level.to_string();
wtr.write_record(&[w, &level, &position_range, &count, &docids])?;
}
}