Reintroduce appending sorted entries when possible

milli/src/update/index_documents/helpers/grenad_helpers.rs

@@ -3,7 +3,7 @@ use std::fs::File;
 use std::io::{self, Seek, SeekFrom};
 use std::time::Instant;
 
-use grenad::{CompressionType, MergerIter, Reader, Sorter};
+use grenad::{CompressionType, Reader, Sorter};
 use heed::types::ByteSlice;
 use log::debug;
 
@@ -209,36 +209,34 @@ pub fn sorter_into_lmdb_database(
     debug!("Writing MTBL sorter...");
     let before = Instant::now();
 
-    merger_iter_into_lmdb_database(wtxn, database, sorter.into_stream_merger_iter()?, merge)?;
+    let mut merger_iter = sorter.into_stream_merger_iter()?;
-
+    if database.is_empty(wtxn)? {
-    debug!("MTBL sorter writen in {:.02?}!", before.elapsed());
+        let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
-    Ok(())
+        while let Some((k, v)) = merger_iter.next()? {
-}
+            // safety: we don't keep references from inside the LMDB database.
-
+            unsafe { out_iter.append(k, v)? };
-fn merger_iter_into_lmdb_database<R: io::Read + io::Seek>(
+        }
-    wtxn: &mut heed::RwTxn,
+    } else {
-    database: heed::PolyDatabase,
+        while let Some((k, v)) = merger_iter.next()? {
-    mut merger_iter: MergerIter<R, MergeFn>,
+            let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
-    merge: MergeFn,
+            match iter.next().transpose()? {
-) -> Result<()> {
+                Some((key, old_val)) if key == k => {
-    while let Some((k, v)) = merger_iter.next()? {
+                    let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)];
-        let mut iter = database.prefix_iter_mut::<_, ByteSlice, ByteSlice>(wtxn, k)?;
+                    let val = merge(k, &vals).map_err(|_| {
-        match iter.next().transpose()? {
+                        // TODO just wrap this error?
-            Some((key, old_val)) if key == k => {
+                        InternalError::IndexingMergingKeys { process: "get-put-merge" }
-                let vals = vec![Cow::Borrowed(old_val), Cow::Borrowed(v)];
+                    })?;
-                let val = merge(k, &vals).map_err(|_| {
+                    // safety: we don't keep references from inside the LMDB database.
-                    // TODO just wrap this error?
+                    unsafe { iter.put_current(k, &val)? };
-                    InternalError::IndexingMergingKeys { process: "get-put-merge" }
+                }
-                })?;
+                _ => {
-                // safety: we don't keep references from inside the LMDB database.
+                    drop(iter);
-                unsafe { iter.put_current(k, &val)? };
+                    database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
-            }
+                }
-            _ => {
-                drop(iter);
-                database.put::<_, ByteSlice, ByteSlice>(wtxn, k, v)?;
             }
         }
     }
 
+    debug!("MTBL sorter writen in {:.02?}!", before.elapsed());
     Ok(())
 }