mod clear_all; mod customs_update; mod documents_addition; mod documents_deletion; mod schema_update; mod stop_words_addition; mod stop_words_deletion; mod synonyms_addition; mod synonyms_deletion; pub use self::clear_all::{apply_clear_all, push_clear_all}; pub use self::customs_update::{apply_customs_update, push_customs_update}; pub use self::documents_addition::{ apply_documents_addition, apply_documents_partial_addition, DocumentsAddition, }; pub use self::documents_deletion::{apply_documents_deletion, DocumentsDeletion}; pub use self::schema_update::{apply_schema_update, push_schema_update}; pub use self::stop_words_addition::{apply_stop_words_addition, StopWordsAddition}; pub use self::stop_words_deletion::{apply_stop_words_deletion, StopWordsDeletion}; pub use self::synonyms_addition::{apply_synonyms_addition, SynonymsAddition}; pub use self::synonyms_deletion::{apply_synonyms_deletion, SynonymsDeletion}; use std::cmp; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::Instant; use chrono::{DateTime, Utc}; use heed::Result as ZResult; use log::debug; use serde::{Deserialize, Serialize}; use crate::{store, DocumentId, MResult}; use meilisearch_schema::Schema; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Update { data: UpdateData, enqueued_at: DateTime, } impl Update { fn clear_all() -> Update { Update { data: UpdateData::ClearAll, enqueued_at: Utc::now(), } } fn schema(data: Schema) -> Update { Update { data: UpdateData::Schema(data), enqueued_at: Utc::now(), } } fn customs(data: Vec) -> Update { Update { data: UpdateData::Customs(data), enqueued_at: Utc::now(), } } fn documents_addition(data: Vec>) -> Update { Update { data: UpdateData::DocumentsAddition(data), enqueued_at: Utc::now(), } } fn documents_partial(data: Vec>) -> Update { Update { data: UpdateData::DocumentsPartial(data), enqueued_at: Utc::now(), } } fn documents_deletion(data: Vec) -> Update { Update { data: UpdateData::DocumentsDeletion(data), enqueued_at: Utc::now(), } } fn synonyms_addition(data: BTreeMap>) -> Update { Update { data: UpdateData::SynonymsAddition(data), enqueued_at: Utc::now(), } } fn synonyms_deletion(data: BTreeMap>>) -> Update { Update { data: UpdateData::SynonymsDeletion(data), enqueued_at: Utc::now(), } } fn stop_words_addition(data: BTreeSet) -> Update { Update { data: UpdateData::StopWordsAddition(data), enqueued_at: Utc::now(), } } fn stop_words_deletion(data: BTreeSet) -> Update { Update { data: UpdateData::StopWordsDeletion(data), enqueued_at: Utc::now(), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum UpdateData { ClearAll, Schema(Schema), Customs(Vec), DocumentsAddition(Vec>), DocumentsPartial(Vec>), DocumentsDeletion(Vec), SynonymsAddition(BTreeMap>), SynonymsDeletion(BTreeMap>>), StopWordsAddition(BTreeSet), StopWordsDeletion(BTreeSet), } impl UpdateData { pub fn update_type(&self) -> UpdateType { match self { UpdateData::ClearAll => UpdateType::ClearAll, UpdateData::Schema(_) => UpdateType::Schema, UpdateData::Customs(_) => UpdateType::Customs, UpdateData::DocumentsAddition(addition) => UpdateType::DocumentsAddition { number: addition.len(), }, UpdateData::DocumentsPartial(addition) => UpdateType::DocumentsPartial { number: addition.len(), }, UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion { number: deletion.len(), }, UpdateData::SynonymsAddition(addition) => UpdateType::SynonymsAddition { number: addition.len(), }, UpdateData::SynonymsDeletion(deletion) => UpdateType::SynonymsDeletion { number: deletion.len(), }, UpdateData::StopWordsAddition(addition) => UpdateType::StopWordsAddition { number: addition.len(), }, UpdateData::StopWordsDeletion(deletion) => UpdateType::StopWordsDeletion { number: deletion.len(), }, } } } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(tag = "name")] pub enum UpdateType { ClearAll, Schema, Customs, DocumentsAddition { number: usize }, DocumentsPartial { number: usize }, DocumentsDeletion { number: usize }, SynonymsAddition { number: usize }, SynonymsDeletion { number: usize }, StopWordsAddition { number: usize }, StopWordsDeletion { number: usize }, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ProcessedUpdateResult { pub update_id: u64, #[serde(rename = "type")] pub update_type: UpdateType, #[serde(skip_serializing_if = "Option::is_none")] pub error: Option, pub duration: f64, // in seconds pub enqueued_at: DateTime, pub processed_at: DateTime, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EnqueuedUpdateResult { pub update_id: u64, pub update_type: UpdateType, pub enqueued_at: DateTime, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase", tag = "status")] pub enum UpdateStatus { Enqueued { #[serde(flatten)] content: EnqueuedUpdateResult, }, Processed { #[serde(flatten)] content: ProcessedUpdateResult, }, } pub fn update_status( reader: &heed::RoTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, update_id: u64, ) -> MResult> { match updates_results_store.update_result(reader, update_id)? { Some(result) => Ok(Some(UpdateStatus::Processed { content: result })), None => match updates_store.get(reader, update_id)? { Some(update) => Ok(Some(UpdateStatus::Enqueued { content: EnqueuedUpdateResult { update_id, update_type: update.data.update_type(), enqueued_at: update.enqueued_at, }, })), None => Ok(None), }, } } pub fn next_update_id( writer: &mut heed::RwTxn, updates_store: store::Updates, updates_results_store: store::UpdatesResults, ) -> ZResult { let last_update_id = updates_store.last_update_id(writer)?; let last_update_id = last_update_id.map(|(n, _)| n); let last_update_results_id = updates_results_store.last_update_id(writer)?; let last_update_results_id = last_update_results_id.map(|(n, _)| n); let max_update_id = cmp::max(last_update_id, last_update_results_id); let new_update_id = max_update_id.map_or(0, |n| n + 1); Ok(new_update_id) } pub fn update_task<'a, 'b>( writer: &'a mut heed::RwTxn<'b>, index: store::Index, update_id: u64, update: Update, ) -> MResult { debug!("Processing update number {}", update_id); let Update { enqueued_at, data } = update; let (update_type, result, duration) = match data { UpdateData::ClearAll => { let start = Instant::now(); let update_type = UpdateType::ClearAll; let result = apply_clear_all( writer, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, ); (update_type, result, start.elapsed()) } UpdateData::Schema(schema) => { let start = Instant::now(); let update_type = UpdateType::Schema; let result = apply_schema_update( writer, &schema, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, ); (update_type, result, start.elapsed()) } UpdateData::Customs(customs) => { let start = Instant::now(); let update_type = UpdateType::Customs; let result = apply_customs_update(writer, index.main, &customs).map_err(Into::into); (update_type, result, start.elapsed()) } UpdateData::DocumentsAddition(documents) => { let start = Instant::now(); let update_type = UpdateType::DocumentsAddition { number: documents.len(), }; let result = apply_documents_addition( writer, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, documents, ); (update_type, result, start.elapsed()) } UpdateData::DocumentsPartial(documents) => { let start = Instant::now(); let update_type = UpdateType::DocumentsPartial { number: documents.len(), }; let result = apply_documents_partial_addition( writer, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, documents, ); (update_type, result, start.elapsed()) } UpdateData::DocumentsDeletion(documents) => { let start = Instant::now(); let update_type = UpdateType::DocumentsDeletion { number: documents.len(), }; let result = apply_documents_deletion( writer, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, documents, ); (update_type, result, start.elapsed()) } UpdateData::SynonymsAddition(synonyms) => { let start = Instant::now(); let update_type = UpdateType::SynonymsAddition { number: synonyms.len(), }; let result = apply_synonyms_addition(writer, index.main, index.synonyms, synonyms); (update_type, result, start.elapsed()) } UpdateData::SynonymsDeletion(synonyms) => { let start = Instant::now(); let update_type = UpdateType::SynonymsDeletion { number: synonyms.len(), }; let result = apply_synonyms_deletion(writer, index.main, index.synonyms, synonyms); (update_type, result, start.elapsed()) } UpdateData::StopWordsAddition(stop_words) => { let start = Instant::now(); let update_type = UpdateType::StopWordsAddition { number: stop_words.len(), }; let result = apply_stop_words_addition(writer, index.main, index.postings_lists, stop_words); (update_type, result, start.elapsed()) } UpdateData::StopWordsDeletion(stop_words) => { let start = Instant::now(); let update_type = UpdateType::StopWordsDeletion { number: stop_words.len(), }; let result = apply_stop_words_deletion( writer, index.main, index.documents_fields, index.documents_fields_counts, index.postings_lists, index.docs_words, stop_words, ); (update_type, result, start.elapsed()) } }; debug!( "Processed update number {} {:?} {:?}", update_id, update_type, result ); let status = ProcessedUpdateResult { update_id, update_type, error: result.map_err(|e| e.to_string()).err(), duration: duration.as_secs_f64(), enqueued_at, processed_at: Utc::now(), }; Ok(status) }