Merge #266

266: Bump LMDB to the latest version (v0.9.70) r=Kerollmops a=Kerollmops

By bumping to a new version of heed (from git, v0.12.0 unpublished yet), this PR fixes Windows disk reservation problems. This new version of heed changes the `del/put_current`, and `append` iterator methods signature by declaring them unsafe.

This PR also bumps milli itself into v0.7.0 as it is breaking due to the heed/LMDB bump.

This PR must be merged after #264.

Co-authored-by: Clément Renault <clement@meilisearch.com>

helpers/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "helpers"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Clément Renault <clement@meilisearch.com>"]
 edition = "2018"
 
 [dependencies]
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 milli = { path = "../milli" }
 stderrlog = "0.5.1"
 structopt = { version = "0.3.21", default-features = false }

http-ui/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "http-ui"
 description = "The HTTP user interface of the milli search engine"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Clément Renault <clement@meilisearch.com>"]
 edition = "2018"
 
@@ -10,7 +10,7 @@ anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
 crossbeam-channel = "0.5.0"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
 memmap = "0.7.0"
 milli = { path = "../milli" }

infos/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "infos"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Clément Renault <clement@meilisearch.com>"]
 edition = "2018"
 
@@ -8,7 +8,7 @@ edition = "2018"
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
 csv = "1.1.5"
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 milli = { path = "../milli" }
 roaring = "0.6.6"
 serde_json = "1.0.62"

milli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "milli"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Kerollmops <clement@meilisearch.com>"]
 edition = "2018"
 
@@ -14,7 +14,7 @@ flate2 = "1.0.20"
 fst = "0.4.5"
 fxhash = "0.2.1"
 grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
-heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] }
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] }
 human_format = "1.0.3"
 levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
 linked-hash-map = "0.5.4"

milli/src/update/delete_documents.rs

@@ -132,7 +132,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     };
                     external_ids.push(external_id);
                 }
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
             drop(iter);
 
@@ -143,7 +144,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                 let ((_docid, word), _positions) = result?;
                 // This boolean will indicate if we must remove this word from the words FST.
                 words.push((SmallString32::from(word), false));
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             }
         }
 
@@ -194,11 +196,13 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
                     let previous_len = docids.len();
                     docids.difference_with(&self.documents_ids);
                     if docids.is_empty() {
-                        iter.del_current()?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.del_current()? };
                         *must_remove = true;
                     } else if docids.len() != previous_len {
                         let key = key.to_owned();
-                        iter.put_current(&key, &docids)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(&key, &docids)? };
                     }
                 }
             }
@@ -243,10 +247,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
                 prefixes_to_delete.insert(prefix)?;
             } else if docids.len() != previous_len {
-                iter.put_current(&prefix, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&prefix, &docids)? };
             }
         }
 
@@ -281,10 +287,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let key = key.to_owned();
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -300,10 +308,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -317,10 +327,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -334,10 +346,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
                 let bytes = bytes.to_owned();
-                iter.put_current(&bytes, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&bytes, &docids)? };
             }
         }
 
@@ -349,9 +363,11 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
             let previous_len = docids.len();
             docids.difference_with(&self.documents_ids);
             if docids.is_empty() {
-                iter.del_current()?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.del_current()? };
             } else if docids.len() != previous_len {
-                iter.put_current(&key, &docids)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { iter.put_current(&key, &docids)? };
             }
         }
 
@@ -420,7 +436,8 @@ where
     while let Some(result) = iter.next() {
         let (key, ()) = result?;
         if to_remove.contains(convert(key)) {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         }
     }
 
@@ -441,10 +458,12 @@ where
         let previous_len = docids.len();
         docids.difference_with(to_remove);
         if docids.is_empty() {
-            iter.del_current()?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
         } else if docids.len() != previous_len {
             let bytes = bytes.to_owned();
-            iter.put_current(&bytes, &docids)?;
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.put_current(&bytes, &docids)? };
         }
     }
 

milli/src/update/index_documents/mod.rs

@@ -144,7 +144,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = reader.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -154,7 +155,8 @@ where
                     Some((key, old_val)) if key == k => {
                         let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
                         let val = merge(k, &vals)?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);
@@ -203,7 +205,8 @@ where
         WriteMethod::Append => {
             let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
             while let Some((k, v)) = sorter.next()? {
-                out_iter.append(k, v)?;
+                // safety: we don't keep references from inside the LMDB database.
+                unsafe { out_iter.append(k, v)? };
             }
         }
         WriteMethod::GetMergePut => {
@@ -216,7 +219,8 @@ where
                             // TODO just wrap this error?
                             InternalError::IndexingMergingKeys { process: "get-put-merge" }
                         })?;
-                        iter.put_current(k, &val)?;
+                        // safety: we don't keep references from inside the LMDB database.
+                        unsafe { iter.put_current(k, &val)? };
                     }
                     _ => {
                         drop(iter);

search/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "search"
-version = "0.6.0"
+version = "0.7.0"
 authors = ["Clément Renault <clement@meilisearch.com>"]
 edition = "2018"
 
 [dependencies]
 anyhow = "1.0.38"
 byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-heed = "0.10.6"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
 log = "0.4.14"
 milli = { path = "../milli" }
 serde_json = "1.0.62"