From 51d1e64b238b73919bd15f564d3f88bfa1901947 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= <clement@meilisearch.com>
Date: Tue, 25 Jan 2022 17:12:12 +0100
Subject: [PATCH] Remove, now useless, the WriteMethod enum
---
milli/src/update/facets.rs | 6 +-
.../index_documents/helpers/grenad_helpers.rs | 74 ++++++-------------
milli/src/update/index_documents/mod.rs | 6 --
milli/src/update/word_prefix_docids.rs | 3 +-
.../word_prefix_pair_proximity_docids.rs | 3 +-
.../update/words_prefix_position_docids.rs | 3 +-
6 files changed, 28 insertions(+), 67 deletions(-)
diff --git a/milli/src/update/facets.rs b/milli/src/update/facets.rs
index a2f17cba3..19684c6ea 100644
--- a/milli/src/update/facets.rs
+++ b/milli/src/update/facets.rs
@@ -15,9 +15,7 @@ use crate::heed_codec::facet::{
FacetStringLevelZeroValueCodec, FacetStringZeroBoundsValueCodec,
};
use crate::heed_codec::CboRoaringBitmapCodec;
-use crate::update::index_documents::{
- create_writer, write_into_lmdb_database, writer_into_reader, WriteMethod,
-};
+use crate::update::index_documents::{create_writer, write_into_lmdb_database, writer_into_reader};
use crate::{FieldId, Index, Result};
pub struct Facets<'t, 'u, 'i> {
@@ -120,7 +118,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> {
*self.index.facet_id_f64_docids.as_polymorph(),
facet_number_levels,
|_, _| Err(InternalError::IndexingMergingKeys { process: "facet number levels" })?,
- WriteMethod::GetMergePut,
)?;
write_into_lmdb_database(
@@ -128,7 +125,6 @@ impl<'t, 'u, 'i> Facets<'t, 'u, 'i> {
*self.index.facet_id_string_docids.as_polymorph(),
facet_string_levels,
|_, _| Err(InternalError::IndexingMergingKeys { process: "facet string levels" })?,
- WriteMethod::GetMergePut,
)?;
}
diff --git a/milli/src/update/index_documents/helpers/grenad_helpers.rs b/milli/src/update/index_documents/helpers/grenad_helpers.rs
index 10662892b..eef067122 100644
--- a/milli/src/update/index_documents/helpers/grenad_helpers.rs
+++ b/milli/src/update/index_documents/helpers/grenad_helpers.rs
@@ -9,7 +9,6 @@ use log::debug;
use super::{ClonableMmap, MergeFn};
use crate::error::InternalError;
-use crate::update::index_documents::WriteMethod;
use crate::Result;
pub type CursorClonableMmap = io::Cursor<ClonableMmap>;
@@ -169,34 +168,22 @@ pub fn write_into_lmdb_database(
database: heed::PolyDatabase,
mut reader: Reader<File>,
merge: MergeFn,
- method: WriteMethod,
) -> Result<()> {
debug!("Writing MTBL stores...");
let before = Instant::now();
- match method {
- WriteMethod::Append => {
- let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
- while let Some((k, v)) = reader.next()? {
+ while let Some((k, v)) = reader.next()? {
+ let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
+ match iter.next().transpose()? {
+ Some((key, old_val)) if key == k => {
+ let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
+ let val = merge(k, &vals)?;
// safety: we don't keep references from inside the LMDB database.
- unsafe { out_iter.append(k, v)? };
+ unsafe { iter.put_current(k, &val)? };
}
- }
- WriteMethod::GetMergePut => {
- while let Some((k, v)) = reader.next()? {
- let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
- match iter.next().transpose()? {
- Some((key, old_val)) if key == k => {
- let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
- let val = merge(k, &vals)?;
- // safety: we don't keep references from inside the LMDB database.
- unsafe { iter.put_current(k, &val)? };
- }
- _ => {
- drop(iter);
- database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
- }
- }
+ _ => {
+ drop(iter);
+ database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
}
}
}
@@ -210,12 +197,11 @@ pub fn sorter_into_lmdb_database(
database: heed::PolyDatabase,
sorter: Sorter<MergeFn>,
merge: MergeFn,
- method: WriteMethod,
) -> Result<()> {
debug!("Writing MTBL sorter...");
let before = Instant::now();
- merger_iter_into_lmdb_database(wtxn, database, sorter.into_merger_iter()?, merge, method)?;
+ merger_iter_into_lmdb_database(wtxn, database, sorter.into_merger_iter()?, merge)?;
debug!("MTBL sorter writen in {:.02?}!", before.elapsed());
Ok(())
@@ -226,34 +212,22 @@ fn merger_iter_into_lmdb_database<R: io::Read>(
database: heed::PolyDatabase,
mut sorter: MergerIter<R, MergeFn>,
merge: MergeFn,
- method: WriteMethod,
) -> Result<()> {
- match method {
- WriteMethod::Append => {
- let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
- while let Some((k, v)) = sorter.next()? {
+ while let Some((k, v)) = sorter.next()? {
+ let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
+ match iter.next().transpose()? {
+ Some((key, old_val)) if key == k => {
+ let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)];
+ let val = merge(k, &vals).map_err(|_| {
+ // TODO just wrap this error?
+ InternalError::IndexingMergingKeys { process: "get-put-merge" }
+ })?;
// safety: we don't keep references from inside the LMDB database.
- unsafe { out_iter.append(k, v)? };
+ unsafe { iter.put_current(k, &val)? };
}
- }
- WriteMethod::GetMergePut => {
- while let Some((k, v)) = sorter.next()? {
- let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
- match iter.next().transpose()? {
- Some((key, old_val)) if key == k => {
- let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)];
- let val = merge(k, &vals).map_err(|_| {
- // TODO just wrap this error?
- InternalError::IndexingMergingKeys { process: "get-put-merge" }
- })?;
- // safety: we don't keep references from inside the LMDB database.
- unsafe { iter.put_current(k, &val)? };
- }
- _ => {
- drop(iter);
- database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
- }
- }
+ _ => {
+ drop(iter);
+ database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
}
}
}
diff --git a/milli/src/update/index_documents/mod.rs b/milli/src/update/index_documents/mod.rs
index 7ea5c3816..ee80d8ada 100644
--- a/milli/src/update/index_documents/mod.rs
+++ b/milli/src/update/index_documents/mod.rs
@@ -59,12 +59,6 @@ impl Default for IndexDocumentsMethod {
}
}
-#[derive(Debug, Copy, Clone)]
-pub enum WriteMethod {
- Append,
- GetMergePut,
-}
-
pub struct IndexDocuments<'t, 'u, 'i, 'a, F> {
wtxn: &'t mut heed::RwTxn<'i, 'u>,
index: &'i Index,
diff --git a/milli/src/update/word_prefix_docids.rs b/milli/src/update/word_prefix_docids.rs
index 1e2996c9b..cf50a5b8a 100644
--- a/milli/src/update/word_prefix_docids.rs
+++ b/milli/src/update/word_prefix_docids.rs
@@ -7,7 +7,7 @@ use slice_group_by::GroupBy;
use crate::update::index_documents::{
create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_roaring_bitmaps,
- sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod,
+ sorter_into_lmdb_database, CursorClonableMmap, MergeFn,
};
use crate::{Index, Result};
@@ -126,7 +126,6 @@ impl<'t, 'u, 'i> WordPrefixDocids<'t, 'u, 'i> {
*self.index.word_prefix_docids.as_polymorph(),
prefix_docids_sorter,
merge_roaring_bitmaps,
- WriteMethod::GetMergePut,
)?;
Ok(())
diff --git a/milli/src/update/word_prefix_pair_proximity_docids.rs b/milli/src/update/word_prefix_pair_proximity_docids.rs
index f846e8d9e..5b025e4fc 100644
--- a/milli/src/update/word_prefix_pair_proximity_docids.rs
+++ b/milli/src/update/word_prefix_pair_proximity_docids.rs
@@ -8,7 +8,7 @@ use slice_group_by::GroupBy;
use crate::update::index_documents::{
create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_cbo_roaring_bitmaps,
- sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod,
+ sorter_into_lmdb_database, CursorClonableMmap, MergeFn,
};
use crate::{Index, Result, StrStrU8Codec};
@@ -192,7 +192,6 @@ impl<'t, 'u, 'i> WordPrefixPairProximityDocids<'t, 'u, 'i> {
*self.index.word_prefix_pair_proximity_docids.as_polymorph(),
word_prefix_pair_proximity_docids_sorter,
merge_cbo_roaring_bitmaps,
- WriteMethod::GetMergePut,
)?;
Ok(())
diff --git a/milli/src/update/words_prefix_position_docids.rs b/milli/src/update/words_prefix_position_docids.rs
index b1b8273ef..178684cf0 100644
--- a/milli/src/update/words_prefix_position_docids.rs
+++ b/milli/src/update/words_prefix_position_docids.rs
@@ -14,7 +14,7 @@ use crate::heed_codec::StrBEU32Codec;
use crate::index::main_key::WORDS_PREFIXES_FST_KEY;
use crate::update::index_documents::{
create_sorter, fst_stream_into_hashset, fst_stream_into_vec, merge_cbo_roaring_bitmaps,
- sorter_into_lmdb_database, CursorClonableMmap, MergeFn, WriteMethod,
+ sorter_into_lmdb_database, CursorClonableMmap, MergeFn,
};
use crate::{Index, Result};
@@ -170,7 +170,6 @@ impl<'t, 'u, 'i> WordPrefixPositionDocids<'t, 'u, 'i> {
*self.index.word_prefix_position_docids.as_polymorph(),
prefix_position_docids_sorter,
merge_cbo_roaring_bitmaps,
- WriteMethod::GetMergePut,
)?;
Ok(())