searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2025-07-03 03:47:02 +02:00
Code Issues Releases Wiki Activity

Separate the update and main databases

Browse source 
We used the heed typed transaction to make it safe (https://github.com/Kerollmops/heed/pull/27).
This commit is contained in:
Clément Renault 2019-11-26 16:12:06 +01:00
parent 86a87d6032
commit d08b76a323
No known key found for this signature in database
GPG key ID: 92ADA4E935E71FA4
41 changed files with 498 additions and 414 deletions
Download patch file Download diff file Expand all files Collapse all files

5
meilisearch-core/src/update/clear_all.rs
View file

@ -1,8 +1,9 @@
use crate::database::{MainT, UpdateT};
use crate::update::{next_update_id, Update};
use crate::{store, MResult, RankedMap};
pub fn apply_clear_all(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,
@ -21,7 +22,7 @@ pub fn apply_clear_all(
}
pub fn push_clear_all(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
) -> MResult<u64> {

10
meilisearch-core/src/update/customs_update.rs
View file

@ -1,9 +1,11 @@
use crate::store;
use crate::update::{next_update_id, Update};
use heed::Result as ZResult;
use crate::database::{MainT, UpdateT};
use crate::store;
use crate::update::{next_update_id, Update};
pub fn apply_customs_update(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
customs: &[u8],
) -> ZResult<()> {
@ -11,7 +13,7 @@ pub fn apply_customs_update(
}
pub fn push_customs_update(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
customs: Vec<u8>,

13
meilisearch-core/src/update/documents_addition.rs
View file

@ -4,6 +4,7 @@ use fst::{set::OpBuilder, SetBuilder};
use sdset::{duo::Union, SetOperation};
use serde::{Deserialize, Serialize};
use crate::database::{MainT, UpdateT};
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::raw_indexer::RawIndexer;
use crate::serde::{extract_document_id, serialize_value, Deserializer, Serializer};
@ -52,7 +53,7 @@ impl<D> DocumentsAddition<D> {
self.documents.push(document);
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64>
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64>
where
D: serde::Serialize,
{
@ -75,7 +76,7 @@ impl<D> Extend<D> for DocumentsAddition<D> {
}
pub fn push_documents_addition<D: serde::Serialize>(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
addition: Vec<D>,
@ -102,7 +103,7 @@ pub fn push_documents_addition<D: serde::Serialize>(
}
pub fn apply_documents_addition<'a, 'b>(
writer: &'a mut heed::RwTxn<'b>,
writer: &'a mut heed::RwTxn<'b, MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,
@ -181,7 +182,7 @@ pub fn apply_documents_addition<'a, 'b>(
}
pub fn apply_documents_partial_addition<'a, 'b>(
writer: &'a mut heed::RwTxn<'b>,
writer: &'a mut heed::RwTxn<'b, MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,
@ -277,7 +278,7 @@ pub fn apply_documents_partial_addition<'a, 'b>(
}
pub fn reindex_all_documents(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,
@ -354,7 +355,7 @@ pub fn reindex_all_documents(
}
pub fn write_documents_addition_index(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
postings_lists_store: store::PostingsLists,
docs_words_store: store::DocsWords,

7
meilisearch-core/src/update/documents_deletion.rs
View file

@ -4,6 +4,7 @@ use fst::{SetBuilder, Streamer};
use meilisearch_schema::Schema;
use sdset::{duo::DifferenceByKey, SetBuf, SetOperation};
use crate::database::{MainT, UpdateT};
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::serde::extract_document_id;
use crate::store;
@ -50,7 +51,7 @@ impl DocumentsDeletion {
Ok(())
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
let update_id = push_documents_deletion(
writer,
@ -69,7 +70,7 @@ impl Extend<DocumentId> for DocumentsDeletion {
}
pub fn push_documents_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
deletion: Vec<DocumentId>,
@ -83,7 +84,7 @@ pub fn push_documents_deletion(
}
pub fn apply_documents_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,

21
meilisearch-core/src/update/mod.rs
View file

@ -30,6 +30,7 @@ use log::debug;
use serde::{Deserialize, Serialize};
use crate::{store, DocumentId, MResult};
use crate::database::{MainT, UpdateT};
use meilisearch_schema::Schema;
#[derive(Debug, Clone, Serialize, Deserialize)]
@ -203,14 +204,14 @@ pub enum UpdateStatus {
}
pub fn update_status(
reader: &heed::RoTxn,
update_reader: &heed::RoTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
update_id: u64,
) -> MResult<Option<UpdateStatus>> {
match updates_results_store.update_result(reader, update_id)? {
match updates_results_store.update_result(update_reader, update_id)? {
Some(result) => Ok(Some(UpdateStatus::Processed { content: result })),
None => match updates_store.get(reader, update_id)? {
None => match updates_store.get(update_reader, update_id)? {
Some(update) => Ok(Some(UpdateStatus::Enqueued {
content: EnqueuedUpdateResult {
update_id,
@ -224,25 +225,25 @@ pub fn update_status(
}
pub fn next_update_id(
writer: &mut heed::RwTxn,
update_writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
) -> ZResult<u64> {
let last_update_id = updates_store.last_update_id(writer)?;
let last_update_id = last_update_id.map(|(n, _)| n);
let last_update = updates_store.last_update(update_writer)?;
let last_update = last_update.map(|(n, _)| n);
let last_update_results_id = updates_results_store.last_update_id(writer)?;
let last_update_results_id = updates_results_store.last_update(update_writer)?;
let last_update_results_id = last_update_results_id.map(|(n, _)| n);
let max_update_id = cmp::max(last_update_id, last_update_results_id);
let max_update_id = cmp::max(last_update, last_update_results_id);
let new_update_id = max_update_id.map_or(0, |n| n + 1);
Ok(new_update_id)
}
pub fn update_task<'a, 'b>(
writer: &'a mut heed::RwTxn<'b>,
index: store::Index,
writer: &'a mut heed::RwTxn<'b, MainT>,
index: &store::Index,
update_id: u64,
update: Update,
) -> MResult<ProcessedUpdateResult> {

5
meilisearch-core/src/update/schema_update.rs
View file

@ -1,11 +1,12 @@
use meilisearch_schema::{Diff, Schema};
use crate::database::{MainT, UpdateT};
use crate::update::documents_addition::reindex_all_documents;
use crate::update::{next_update_id, Update};
use crate::{error::UnsupportedOperation, store, MResult};
pub fn apply_schema_update(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
new_schema: &Schema,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
@ -61,7 +62,7 @@ pub fn apply_schema_update(
}
pub fn push_schema_update(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
schema: Schema,

7
meilisearch-core/src/update/stop_words_addition.rs
View file

@ -2,6 +2,7 @@ use std::collections::BTreeSet;
use fst::{set::OpBuilder, SetBuilder};
use crate::database::{MainT, UpdateT};
use crate::automaton::normalize_str;
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::update::{next_update_id, Update};
@ -33,7 +34,7 @@ impl StopWordsAddition {
self.stop_words.insert(stop_word);
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
let update_id = push_stop_words_addition(
writer,
@ -46,7 +47,7 @@ impl StopWordsAddition {
}
pub fn push_stop_words_addition(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
addition: BTreeSet<String>,
@ -60,7 +61,7 @@ pub fn push_stop_words_addition(
}
pub fn apply_stop_words_addition(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
postings_lists_store: store::PostingsLists,
addition: BTreeSet<String>,

7
meilisearch-core/src/update/stop_words_deletion.rs
View file

@ -2,6 +2,7 @@ use std::collections::BTreeSet;
use fst::{set::OpBuilder, SetBuilder};
use crate::database::{MainT, UpdateT};
use crate::automaton::normalize_str;
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::update::documents_addition::reindex_all_documents;
@ -34,7 +35,7 @@ impl StopWordsDeletion {
self.stop_words.insert(stop_word);
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
let update_id = push_stop_words_deletion(
writer,
@ -47,7 +48,7 @@ impl StopWordsDeletion {
}
pub fn push_stop_words_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
deletion: BTreeSet<String>,
@ -61,7 +62,7 @@ pub fn push_stop_words_deletion(
}
pub fn apply_stop_words_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
documents_fields_store: store::DocumentsFields,
documents_fields_counts_store: store::DocumentsFieldsCounts,

7
meilisearch-core/src/update/synonyms_addition.rs
View file

@ -3,6 +3,7 @@ use std::collections::BTreeMap;
use fst::{set::OpBuilder, SetBuilder};
use sdset::SetBuf;
use crate::database::{MainT, UpdateT};
use crate::automaton::normalize_str;
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::update::{next_update_id, Update};
@ -43,7 +44,7 @@ impl SynonymsAddition {
.extend(alternatives);
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
let update_id = push_synonyms_addition(
writer,
@ -56,7 +57,7 @@ impl SynonymsAddition {
}
pub fn push_synonyms_addition(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
addition: BTreeMap<String, Vec<String>>,
@ -70,7 +71,7 @@ pub fn push_synonyms_addition(
}
pub fn apply_synonyms_addition(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
synonyms_store: store::Synonyms,
addition: BTreeMap<String, Vec<String>>,

7
meilisearch-core/src/update/synonyms_deletion.rs
View file

@ -4,6 +4,7 @@ use std::iter::FromIterator;
use fst::{set::OpBuilder, SetBuilder};
use sdset::SetBuf;
use crate::database::{MainT, UpdateT};
use crate::automaton::normalize_str;
use crate::database::{UpdateEvent, UpdateEventsEmitter};
use crate::update::{next_update_id, Update};
@ -50,7 +51,7 @@ impl SynonymsDeletion {
}
}
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
pub fn finalize(self, writer: &mut heed::RwTxn<UpdateT>) -> MResult<u64> {
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
let update_id = push_synonyms_deletion(
writer,
@ -63,7 +64,7 @@ impl SynonymsDeletion {
}
pub fn push_synonyms_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<UpdateT>,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
deletion: BTreeMap<String, Option<Vec<String>>>,
@ -77,7 +78,7 @@ pub fn push_synonyms_deletion(
}
pub fn apply_synonyms_deletion(
writer: &mut heed::RwTxn,
writer: &mut heed::RwTxn<MainT>,
main_store: store::Main,
synonyms_store: store::Synonyms,
deletion: BTreeMap<String, Option<Vec<String>>>,