From 6b326a45d75f6131cd7337b73bc4f44fcba85e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 31 Oct 2019 17:27:58 +0100 Subject: [PATCH] Fix the update system to always consume updates even if failing --- meilidb-core/Cargo.toml | 2 +- meilidb-core/src/database.rs | 63 +++++++++++++++++++++++++--------- meilidb-core/src/update/mod.rs | 15 +++----- meilidb-http/Cargo.toml | 2 +- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/meilidb-core/Cargo.toml b/meilidb-core/Cargo.toml index 35f31468d..751c1144b 100644 --- a/meilidb-core/Cargo.toml +++ b/meilidb-core/Cargo.toml @@ -12,7 +12,7 @@ crossbeam-channel = "0.3.9" deunicode = "1.0.0" env_logger = "0.7.0" hashbrown = { version = "0.6.0", features = ["serde"] } -heed = "0.3.0" +heed = "0.3.1" log = "0.4.8" meilidb-schema = { path = "../meilidb-schema", version = "0.6.0" } meilidb-tokenizer = { path = "../meilidb-tokenizer", version = "0.6.0" } diff --git a/meilidb-core/src/database.rs b/meilidb-core/src/database.rs index fa5b6bfde..d2280dd48 100644 --- a/meilidb-core/src/database.rs +++ b/meilidb-core/src/database.rs @@ -28,34 +28,63 @@ fn update_awaiter(receiver: Receiver<()>, env: heed::Env, update_fn: Arc writer, Err(e) => { - error!("LMDB writer transaction begin failed: {}", e); + error!("LMDB write transaction begin failed: {}", e); break; } }; - match update::update_task(&mut writer, index.clone()) { - Ok(Some(status)) => { - match status.result { - Ok(_) => { - if let Err(e) = writer.commit() { - error!("update transaction failed: {}", e) - } - } - Err(_) => writer.abort(), - } - - if let Some(ref callback) = *update_fn.load() { - (callback)(status); - } - } - // no more updates to handle for now + let (update_id, update) = match index.updates.pop_front(&mut writer) { + Ok(Some(value)) => value, Ok(None) => { debug!("no more updates"); writer.abort(); break; } + Err(e) => { + error!("pop front update failed: {}", e); + break; + } + }; + + let mut nested_writer = match unsafe { env.nested_write_txn(&writer) } { + Ok(writer) => writer, + Err(e) => { + error!("LMDB nested write transaction begin failed: {}", e); + break; + } + }; + + match update::update_task(&mut nested_writer, index.clone(), update_id, update) { + Ok(status) => { + match &status.result { + Ok(_) => { + if let Err(e) = nested_writer.commit() { + error!("update nested transaction failed: {}", e); + } + } + Err(_) => nested_writer.abort(), + } + + let result = + index + .updates_results + .put_update_result(&mut writer, update_id, &status); + + if let Err(e) = result { + error!("update result store commit failed: {}", e); + } + + if let Err(e) = writer.commit() { + error!("update parent transaction failed: {}", e); + } + + if let Some(ref callback) = *update_fn.load() { + (callback)(status); + } + } Err(e) => { error!("update task failed: {}", e); + nested_writer.abort(); writer.abort() } } diff --git a/meilidb-core/src/update/mod.rs b/meilidb-core/src/update/mod.rs index 360539fe2..64d85c088 100644 --- a/meilidb-core/src/update/mod.rs +++ b/meilidb-core/src/update/mod.rs @@ -152,12 +152,9 @@ pub fn next_update_id( pub fn update_task( writer: &mut heed::RwTxn, index: store::Index, -) -> MResult> { - let (update_id, update) = match index.updates.pop_front(writer)? { - Some(value) => value, - None => return Ok(None), - }; - + update_id: u64, + update: Update, +) -> MResult { debug!("Processing update number {}", update_id); let (update_type, result, duration) = match update { @@ -308,9 +305,5 @@ pub fn update_task( detailed_duration, }; - index - .updates_results - .put_update_result(writer, update_id, &status)?; - - Ok(Some(status)) + Ok(status) } diff --git a/meilidb-http/Cargo.toml b/meilidb-http/Cargo.toml index 12c1a5fc6..02c8fdcb4 100644 --- a/meilidb-http/Cargo.toml +++ b/meilidb-http/Cargo.toml @@ -13,7 +13,7 @@ chrono = { version = "0.4.9", features = ["serde"] } crossbeam-channel = "0.3.9" envconfig = "0.5.1" envconfig_derive = "0.5.1" -heed = "0.3.0" +heed = "0.3.1" http = "0.1.19" indexmap = { version = "1.3.0", features = ["serde-1"] } jemallocator = "0.3.2"