searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-06-13 16:19:50 +02:00
Code Issues Releases Wiki Activity

Use a vecDeque in wpp database

Browse Source
This commit is contained in:
ManyTheFish 2023-09-25 18:55:20 +02:00 committed by Louis Dureuil
parent df9e5c8651
commit 96be85396d
No known key found for this signature in database
1 changed files with 6 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
milli/src/update/index_documents/extract/extract_word_pair_proximity_docids.rs
View File

@ -1,5 +1,5 @@
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::{HashMap, VecDeque};
use std::fs::File;
use std::io::BufReader;
use std::{cmp, io};
@ -37,7 +37,8 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
max_memory.map(|m| m / 2),
);
let mut word_positions: Vec<(String, u16)> = Vec::with_capacity(MAX_DISTANCE as usize);
let mut word_positions: VecDeque<(String, u16)> =
VecDeque::with_capacity(MAX_DISTANCE as usize);
let mut word_pair_proximity = HashMap::new();
let mut current_document_id = None;
@ -80,7 +81,7 @@ pub fn extract_word_pair_proximity_docids<R: io::Read + io::Seek>(
// insert the new word.
let word = std::str::from_utf8(word)?;
word_positions.push((word.to_string(), position));
word_positions.push_back((word.to_string(), position));
}
}
@ -123,10 +124,10 @@ fn document_word_positions_into_sorter(
}
fn word_positions_into_word_pair_proximity(
word_positions: &mut Vec<(String, u16)>,
word_positions: &mut VecDeque<(String, u16)>,
word_pair_proximity: &mut HashMap<(String, String), u8>,
) -> Result<()> {
let (head_word, head_position) = word_positions.remove(0);
let (head_word, head_position) = word_positions.pop_front().unwrap();
for (word, position) in word_positions.iter() {
let prox = positions_proximity(head_position as u32, *position as u32) as u8;
word_pair_proximity
@ -138,29 +139,3 @@ fn word_positions_into_word_pair_proximity(
}
Ok(())
}
struct PeekedWordPosition<I> {
word: String,
position: u32,
iter: I,
}
impl<I> Ord for PeekedWordPosition<I> {
fn cmp(&self, other: &Self) -> Ordering {
self.position.cmp(&other.position).reverse()
}
}
impl<I> PartialOrd for PeekedWordPosition<I> {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl<I> Eq for PeekedWordPosition<I> {}
impl<I> PartialEq for PeekedWordPosition<I> {
fn eq(&self, other: &Self) -> bool {
self.position == other.position
}
}