266: Bump LMDB to the latest version (v0.9.70) r=Kerollmops a=Kerollmops

By bumping to a new version of heed (from git, v0.12.0 unpublished yet), this PR fixes Windows disk reservation problems. This new version of heed changes the `del/put_current`, and `append` iterator methods signature by declaring them unsafe.

This PR also bumps milli itself into v0.7.0 as it is breaking due to the heed/LMDB bump.

This PR must be merged after #264.

Co-authored-by: Clément Renault <clement@meilisearch.com>
This commit is contained in:
bors[bot] 2021-06-28 17:11:41 +00:00 committed by GitHub
commit 9dbc8b2dd0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 411 additions and 326 deletions

648
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,13 +1,13 @@
[package] [package]
name = "helpers" name = "helpers"
version = "0.6.0" version = "0.7.0"
authors = ["Clément Renault <clement@meilisearch.com>"] authors = ["Clément Renault <clement@meilisearch.com>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
anyhow = "1.0.38" anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
heed = "0.10.6" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
milli = { path = "../milli" } milli = { path = "../milli" }
stderrlog = "0.5.1" stderrlog = "0.5.1"
structopt = { version = "0.3.21", default-features = false } structopt = { version = "0.3.21", default-features = false }

View File

@ -1,7 +1,7 @@
[package] [package]
name = "http-ui" name = "http-ui"
description = "The HTTP user interface of the milli search engine" description = "The HTTP user interface of the milli search engine"
version = "0.6.0" version = "0.7.0"
authors = ["Clément Renault <clement@meilisearch.com>"] authors = ["Clément Renault <clement@meilisearch.com>"]
edition = "2018" edition = "2018"
@ -10,7 +10,7 @@ anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
crossbeam-channel = "0.5.0" crossbeam-channel = "0.5.0"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = "0.10.6" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" } meilisearch-tokenizer = { git = "https://github.com/meilisearch/Tokenizer.git", tag = "v0.2.3" }
memmap = "0.7.0" memmap = "0.7.0"
milli = { path = "../milli" } milli = { path = "../milli" }

View File

@ -1,6 +1,6 @@
[package] [package]
name = "infos" name = "infos"
version = "0.6.0" version = "0.7.0"
authors = ["Clément Renault <clement@meilisearch.com>"] authors = ["Clément Renault <clement@meilisearch.com>"]
edition = "2018" edition = "2018"
@ -8,7 +8,7 @@ edition = "2018"
anyhow = "1.0.38" anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
csv = "1.1.5" csv = "1.1.5"
heed = "0.10.6" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
milli = { path = "../milli" } milli = { path = "../milli" }
roaring = "0.6.6" roaring = "0.6.6"
serde_json = "1.0.62" serde_json = "1.0.62"

View File

@ -1,6 +1,6 @@
[package] [package]
name = "milli" name = "milli"
version = "0.6.0" version = "0.7.0"
authors = ["Kerollmops <clement@meilisearch.com>"] authors = ["Kerollmops <clement@meilisearch.com>"]
edition = "2018" edition = "2018"
@ -14,7 +14,7 @@ flate2 = "1.0.20"
fst = "0.4.5" fst = "0.4.5"
fxhash = "0.2.1" fxhash = "0.2.1"
grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" } grenad = { git = "https://github.com/Kerollmops/grenad.git", rev = "3adcb26" }
heed = { version = "0.10.6", default-features = false, features = ["lmdb", "sync-read-txn"] } heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0", default-features = false, features = ["lmdb", "sync-read-txn"] }
human_format = "1.0.3" human_format = "1.0.3"
levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] } levenshtein_automata = { version = "0.2.0", features = ["fst_automaton"] }
linked-hash-map = "0.5.4" linked-hash-map = "0.5.4"

View File

@ -132,7 +132,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
}; };
external_ids.push(external_id); external_ids.push(external_id);
} }
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} }
drop(iter); drop(iter);
@ -143,7 +144,8 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let ((_docid, word), _positions) = result?; let ((_docid, word), _positions) = result?;
// This boolean will indicate if we must remove this word from the words FST. // This boolean will indicate if we must remove this word from the words FST.
words.push((SmallString32::from(word), false)); words.push((SmallString32::from(word), false));
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} }
} }
@ -194,11 +196,13 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
*must_remove = true; *must_remove = true;
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let key = key.to_owned(); let key = key.to_owned();
iter.put_current(&key, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&key, &docids)? };
} }
} }
} }
@ -243,10 +247,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
prefixes_to_delete.insert(prefix)?; prefixes_to_delete.insert(prefix)?;
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
iter.put_current(&prefix, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&prefix, &docids)? };
} }
} }
@ -281,10 +287,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let key = key.to_owned(); let key = key.to_owned();
iter.put_current(&key, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&key, &docids)? };
} }
} }
@ -300,10 +308,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let bytes = bytes.to_owned(); let bytes = bytes.to_owned();
iter.put_current(&bytes, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&bytes, &docids)? };
} }
} }
@ -317,10 +327,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let bytes = bytes.to_owned(); let bytes = bytes.to_owned();
iter.put_current(&bytes, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&bytes, &docids)? };
} }
} }
@ -334,10 +346,12 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let bytes = bytes.to_owned(); let bytes = bytes.to_owned();
iter.put_current(&bytes, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&bytes, &docids)? };
} }
} }
@ -349,9 +363,11 @@ impl<'t, 'u, 'i> DeleteDocuments<'t, 'u, 'i> {
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(&self.documents_ids); docids.difference_with(&self.documents_ids);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
iter.put_current(&key, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&key, &docids)? };
} }
} }
@ -420,7 +436,8 @@ where
while let Some(result) = iter.next() { while let Some(result) = iter.next() {
let (key, ()) = result?; let (key, ()) = result?;
if to_remove.contains(convert(key)) { if to_remove.contains(convert(key)) {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} }
} }
@ -441,10 +458,12 @@ where
let previous_len = docids.len(); let previous_len = docids.len();
docids.difference_with(to_remove); docids.difference_with(to_remove);
if docids.is_empty() { if docids.is_empty() {
iter.del_current()?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.del_current()? };
} else if docids.len() != previous_len { } else if docids.len() != previous_len {
let bytes = bytes.to_owned(); let bytes = bytes.to_owned();
iter.put_current(&bytes, &docids)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(&bytes, &docids)? };
} }
} }

View File

@ -144,7 +144,8 @@ where
WriteMethod::Append => { WriteMethod::Append => {
let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
while let Some((k, v)) = reader.next()? { while let Some((k, v)) = reader.next()? {
out_iter.append(k, v)?; // safety: we don't keep references from inside the LMDB database.
unsafe { out_iter.append(k, v)? };
} }
} }
WriteMethod::GetMergePut => { WriteMethod::GetMergePut => {
@ -154,7 +155,8 @@ where
Some((key, old_val)) if key == k => { Some((key, old_val)) if key == k => {
let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..]; let vals = &[Cow::Borrowed(old_val), Cow::Borrowed(v)][..];
let val = merge(k, &vals)?; let val = merge(k, &vals)?;
iter.put_current(k, &val)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(k, &val)? };
} }
_ => { _ => {
drop(iter); drop(iter);
@ -203,7 +205,8 @@ where
WriteMethod::Append => { WriteMethod::Append => {
let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?; let mut out_iter = database.iter_mut::<_, ByteSlice, ByteSlice>(wtxn)?;
while let Some((k, v)) = sorter.next()? { while let Some((k, v)) = sorter.next()? {
out_iter.append(k, v)?; // safety: we don't keep references from inside the LMDB database.
unsafe { out_iter.append(k, v)? };
} }
} }
WriteMethod::GetMergePut => { WriteMethod::GetMergePut => {
@ -216,7 +219,8 @@ where
// TODO just wrap this error? // TODO just wrap this error?
InternalError::IndexingMergingKeys { process: "get-put-merge" } InternalError::IndexingMergingKeys { process: "get-put-merge" }
})?; })?;
iter.put_current(k, &val)?; // safety: we don't keep references from inside the LMDB database.
unsafe { iter.put_current(k, &val)? };
} }
_ => { _ => {
drop(iter); drop(iter);

View File

@ -1,13 +1,13 @@
[package] [package]
name = "search" name = "search"
version = "0.6.0" version = "0.7.0"
authors = ["Clément Renault <clement@meilisearch.com>"] authors = ["Clément Renault <clement@meilisearch.com>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
anyhow = "1.0.38" anyhow = "1.0.38"
byte-unit = { version = "4.0.9", default-features = false, features = ["std"] } byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
heed = "0.10.6" heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.0" }
log = "0.4.14" log = "0.4.14"
milli = { path = "../milli" } milli = { path = "../milli" }
serde_json = "1.0.62" serde_json = "1.0.62"