searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-06-13 08:09:50 +02:00
Code Issues Releases Wiki Activity

Introduce the WordsLevelPositions update

Browse Source
This commit is contained in:
Kerollmops 2021-03-17 14:34:21 +01:00 committed by many
parent 9242f2f1d4
commit c765f277a3
No known key found for this signature in database
GPG Key ID: 2CEF23B75189EACA
1 changed files with 61 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
milli/src/update/words_level_positions.rs
View File

@ -3,12 +3,11 @@ use std::fs::File;
use std::num::NonZeroUsize;
use grenad::{CompressionType, Reader, Writer, FileFuse};
use heed::types::{ByteSlice, DecodeIgnore};
use heed::types::DecodeIgnore;
use heed::{BytesEncode, Error};
use log::debug;
use roaring::RoaringBitmap;
use crate::facet::FacetType;
use crate::heed_codec::{StrLevelPositionCodec, CboRoaringBitmapCodec};
use crate::Index;
use crate::update::index_documents::WriteMethod;
@ -69,12 +68,16 @@ impl<'t, 'u, 'i> WordsLevelPositions<'t, 'u, 'i> {
self.min_level_size,
)?;
// The previously computed entries also defines the level 0 entries
// so we can clear the database and append all of these entries.
self.index.word_level_position_docids.clear(self.wtxn)?;
write_into_lmdb_database(
self.wtxn,
*self.index.facet_field_id_value_docids.as_polymorph(),
entries,
|_, _| anyhow::bail!("invalid facet level merging"),
WriteMethod::GetMergePut,
WriteMethod::Append,
)?;
Ok(())
@ -107,77 +110,79 @@ fn compute_positions_levels(
min_level_size: NonZeroUsize,
) -> anyhow::Result<Reader<FileFuse>>
{
// let first_level_size = db.prefix_iter(rtxn, &[field_id])?
// .remap_types::<DecodeIgnore, DecodeIgnore>()
// .fold(Ok(0usize), |count, result| result.and(count).map(|c| c + 1))?;
// It is forbidden to keep a cursor and write in a database at the same time with LMDB
// therefore we write the facet levels entries into a grenad file before transfering them.
let mut writer = tempfile::tempfile().and_then(|file| {
create_writer(compression_type, compression_level, file)
})?;
// // It is forbidden to keep a cursor and write in a database at the same time with LMDB
// // therefore we write the facet levels entries into a grenad file before transfering them.
// let mut writer = tempfile::tempfile().and_then(|file| {
// create_writer(compression_type, compression_level, file)
// })?;
for result in db.iter(rtxn)? {
let ((word, level, left, right), docids) = result?;
// let level_0_range = {
// let left = (field_id, 0, T::min_value(), T::min_value());
// let right = (field_id, 0, T::max_value(), T::max_value());
// left..=right
// };
let first_level_size = db.remap_data_type::<DecodeIgnore>()
.prefix_iter(rtxn, &(word, level, u32::min_value(), u32::min_value()))?
.fold(Ok(0usize), |count, result| result.and(count).map(|c| c + 1))?;
// // Groups sizes are always a power of the original level_group_size and therefore a group
// // always maps groups of the previous level and never splits previous levels groups in half.
// let group_size_iter = (1u8..)
// .map(|l| (l, level_group_size.get().pow(l as u32)))
// .take_while(|(_, s)| first_level_size / *s >= min_level_size.get());
let level_0_range = {
let left = (word, 0, u32::min_value(), u32::min_value());
let right = (word, 0, u32::max_value(), u32::max_value());
left..=right
};
// for (level, group_size) in group_size_iter {
// let mut left = T::zero();
// let mut right = T::zero();
// let mut group_docids = RoaringBitmap::new();
// Groups sizes are always a power of the original level_group_size and therefore a group
// always maps groups of the previous level and never splits previous levels groups in half.
let group_size_iter = (1u8..)
.map(|l| (l, level_group_size.get().pow(l as u32)))
.take_while(|(_, s)| first_level_size / *s >= min_level_size.get());
// let db = db.remap_key_type::<KC>();
// for (i, result) in db.range(rtxn, &level_0_range)?.enumerate() {
// let ((_field_id, _level, value, _right), docids) = result?;
// As specified in the documentation, we also write the level 0 entries.
write_level_entry(&mut writer, word, level, left, right, &docids)?;
// if i == 0 {
// left = value;
// } else if i % group_size == 0 {
// // we found the first bound of the next group, we must store the left
// // and right bounds associated with the docids.
// write_entry::<T, KC>(&mut writer, field_id, level, left, right, &group_docids)?;
for (level, group_size) in group_size_iter {
let mut left = 0;
let mut right = 0;
let mut group_docids = RoaringBitmap::new();
// // We save the left bound for the new group and also reset the docids.
// group_docids = RoaringBitmap::new();
// left = value;
// }
for (i, result) in db.range(rtxn, &level_0_range)?.enumerate() {
let ((_field_id, _level, value, _right), docids) = result?;
// // The right bound is always the bound we run through.
// group_docids.union_with(&docids);
// right = value;
// }
if i == 0 {
left = value;
} else if i % group_size == 0 {
// we found the first bound of the next group, we must store the left
// and right bounds associated with the docids.
write_level_entry(&mut writer, word, level, left, right, &group_docids)?;
// if !group_docids.is_empty() {
// write_entry::<T, KC>(&mut writer, field_id, level, left, right, &group_docids)?;
// }
// }
// We save the left bound for the new group and also reset the docids.
group_docids = RoaringBitmap::new();
left = value;
}
// writer_into_reader(writer, shrink_size)
// The right bound is always the bound we run through.
group_docids.union_with(&docids);
right = value;
}
todo!()
if !group_docids.is_empty() {
write_level_entry(&mut writer, word, level, left, right, &group_docids)?;
}
}
}
writer_into_reader(writer, shrink_size)
}
fn write_entry<T, KC>(
fn write_level_entry(
writer: &mut Writer<File>,
field_id: u8,
word: &str,
level: u8,
left: T,
right: T,
left: u32,
right: u32,
ids: &RoaringBitmap,
) -> anyhow::Result<()>
where
KC: for<'x> heed::BytesEncode<'x, EItem = (u8, u8, T, T)>,
{
let key = (field_id, level, left, right);
let key = KC::bytes_encode(&key).ok_or(Error::Encoding)?;
let key = (word, level, left, right);
let key = StrLevelPositionCodec::bytes_encode(&key).ok_or(Error::Encoding)?;
let data = CboRoaringBitmapCodec::bytes_encode(&ids).ok_or(Error::Encoding)?;
writer.insert(&key, &data)?;
Ok(())