diff --git a/Cargo.lock b/Cargo.lock index dc7454ea6..ea62dfe64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1698,7 +1698,6 @@ dependencies = [ "crossbeam-channel", "env_logger", "futures", - "heed", "http 0.1.21", "http-service", "http-service-mock", diff --git a/meilisearch-core/src/database.rs b/meilisearch-core/src/database.rs index 09fc2b326..7e44c7f7d 100644 --- a/meilisearch-core/src/database.rs +++ b/meilisearch-core/src/database.rs @@ -8,10 +8,10 @@ use chrono::{DateTime, Utc}; use crossbeam_channel::{Receiver, Sender}; use heed::types::{Str, Unit, SerdeBincode}; use heed::{CompactionOption, Result as ZResult}; -use log::debug; +use log::{debug, error}; use meilisearch_schema::Schema; -use crate::{store, update, Index, MResult, Error}; +use crate::{store, update, Index, MResult, Error, UpdateReader, MainWriter}; pub type BoxUpdateFn = Box; type ArcSwapFn = arc_swap::ArcSwapOption; @@ -247,6 +247,13 @@ impl Database { } } + pub fn is_indexing(&self, reader: &UpdateReader, index: &str) -> MResult> { + match self.open_index(&index) { + Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())), + None => Ok(None), + } + } + pub fn create_index(&self, name: impl AsRef) -> MResult { let name = name.as_ref(); let mut indexes_lock = self.indexes.write().unwrap(); @@ -449,6 +456,44 @@ impl Database { let common_store = self.common_store(); Ok(common_store.get::<_, Str, Unit>(&reader, UNHEALTHY_KEY)?) } + + pub fn compute_stats(&self, writer: &mut MainWriter, index_uid: &str) -> MResult<()> { + let index = match self.open_index(&index_uid) { + Some(index) => index, + None => { + error!("Impossible to retrieve index {}", index_uid); + return Ok(()); + } + }; + + let schema = match index.main.schema(&writer)? { + Some(schema) => schema, + None => return Ok(()), + }; + + let all_documents_fields = index + .documents_fields_counts + .all_documents_fields_counts(&writer)?; + + // count fields frequencies + let mut fields_frequency = HashMap::<_, usize>::new(); + for result in all_documents_fields { + let (_, attr, _) = result?; + if let Some(field_id) = schema.indexed_pos_to_field_id(attr) { + *fields_frequency.entry(field_id).or_default() += 1; + } + } + + // convert attributes to their names + let frequency: HashMap<_, _> = fields_frequency + .into_iter() + .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c))) + .collect(); + + index + .main + .put_fields_frequency(writer, &frequency) + } } #[cfg(test)] diff --git a/meilisearch-core/src/lib.rs b/meilisearch-core/src/lib.rs index a9938bb73..d837bae62 100644 --- a/meilisearch-core/src/lib.rs +++ b/meilisearch-core/src/lib.rs @@ -52,6 +52,12 @@ use crate::reordered_attrs::ReorderedAttrs; type FstSetCow<'a> = fst::Set>; type FstMapCow<'a> = fst::Map>; +pub type MainWriter<'a> = heed::RwTxn<'a, MainT>; +pub type MainReader = heed::RoTxn; + +pub type UpdateWriter<'a> = heed::RwTxn<'a, UpdateT>; +pub type UpdateReader = heed::RoTxn; + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct Document { pub id: DocumentId, diff --git a/meilisearch-http/Cargo.toml b/meilisearch-http/Cargo.toml index 20f4d4f4e..abcbba9d2 100644 --- a/meilisearch-http/Cargo.toml +++ b/meilisearch-http/Cargo.toml @@ -29,7 +29,6 @@ chrono = { version = "0.4.11", features = ["serde"] } crossbeam-channel = "0.4.2" env_logger = "0.7.1" futures = "0.3.4" -heed = "0.8.0" http = "0.1.19" indexmap = { version = "1.3.2", features = ["serde-1"] } log = "0.4.8" diff --git a/meilisearch-http/src/data.rs b/meilisearch-http/src/data.rs index e278f0ffc..15c2ff1af 100644 --- a/meilisearch-http/src/data.rs +++ b/meilisearch-http/src/data.rs @@ -1,9 +1,7 @@ -use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; -use log::error; -use meilisearch_core::{Database, DatabaseOptions, MResult, MainT, UpdateT}; +use meilisearch_core::{Database, DatabaseOptions}; use sha2::Digest; use sysinfo::Pid; @@ -56,53 +54,6 @@ impl ApiKeys { } } -impl DataInner { - pub fn is_indexing(&self, reader: &heed::RoTxn, index: &str) -> MResult> { - match self.db.open_index(&index) { - Some(index) => index.current_update_id(&reader).map(|u| Some(u.is_some())), - None => Ok(None), - } - } - - pub fn compute_stats(&self, writer: &mut heed::RwTxn, index_uid: &str) -> MResult<()> { - let index = match self.db.open_index(&index_uid) { - Some(index) => index, - None => { - error!("Impossible to retrieve index {}", index_uid); - return Ok(()); - } - }; - - let schema = match index.main.schema(&writer)? { - Some(schema) => schema, - None => return Ok(()), - }; - - let all_documents_fields = index - .documents_fields_counts - .all_documents_fields_counts(&writer)?; - - // count fields frequencies - let mut fields_distribution = HashMap::<_, usize>::new(); - for result in all_documents_fields { - let (_, attr, _) = result?; - if let Some(field_id) = schema.indexed_pos_to_field_id(attr) { - *fields_distribution.entry(field_id).or_default() += 1; - } - } - - // convert attributes to their names - let distribution: HashMap<_, _> = fields_distribution - .into_iter() - .filter_map(|(a, c)| schema.name(a).map(|name| (name.to_string(), c))) - .collect(); - - index - .main - .put_fields_distribution(writer, &distribution) - } -} - impl Data { pub fn new(opt: Opt) -> Data { let db_path = opt.db_path.clone(); diff --git a/meilisearch-http/src/helpers/meilisearch.rs b/meilisearch-http/src/helpers/meilisearch.rs index f760bb1ac..65b6da08f 100644 --- a/meilisearch-http/src/helpers/meilisearch.rs +++ b/meilisearch-http/src/helpers/meilisearch.rs @@ -5,11 +5,11 @@ use std::time::Instant; use indexmap::IndexMap; use log::error; -use meilisearch_core::Filter; +use meilisearch_core::{Filter, MainReader}; use meilisearch_core::facets::FacetFilter; use meilisearch_core::criterion::*; use meilisearch_core::settings::RankingRule; -use meilisearch_core::{Highlight, Index, MainT, RankedMap}; +use meilisearch_core::{Highlight, Index, RankedMap}; use meilisearch_schema::{FieldId, Schema}; use meilisearch_tokenizer::is_cjk; use serde::{Deserialize, Serialize}; @@ -107,7 +107,7 @@ impl<'a> SearchBuilder<'a> { self } - pub fn search(self, reader: &heed::RoTxn) -> Result { + pub fn search(self, reader: &MainReader) -> Result { let schema = self .index .main @@ -257,7 +257,7 @@ impl<'a> SearchBuilder<'a> { pub fn get_criteria( &self, - reader: &heed::RoTxn, + reader: &MainReader, ranked_map: &'a RankedMap, schema: &Schema, ) -> Result>, ResponseError> { diff --git a/meilisearch-http/src/lib.rs b/meilisearch-http/src/lib.rs index 5a44d5761..8e81d9221 100644 --- a/meilisearch-http/src/lib.rs +++ b/meilisearch-http/src/lib.rs @@ -59,7 +59,7 @@ pub fn index_update_callback(index_uid: &str, data: &Data, status: ProcessedUpda if let Some(index) = data.db.open_index(&index_uid) { let db = &data.db; let res = db.main_write::<_, _, ResponseError>(|mut writer| { - if let Err(e) = data.compute_stats(&mut writer, &index_uid) { + if let Err(e) = data.db.compute_stats(&mut writer, &index_uid) { error!("Impossible to compute stats; {}", e) } diff --git a/meilisearch-http/src/routes/stats.rs b/meilisearch-http/src/routes/stats.rs index 5d5988518..5f3e09dd5 100644 --- a/meilisearch-http/src/routes/stats.rs +++ b/meilisearch-http/src/routes/stats.rs @@ -50,7 +50,7 @@ async fn index_stats( let update_reader = data.db.update_read_txn()?; let is_indexing = - data.is_indexing(&update_reader, &path.index_uid)? + data.db.is_indexing(&update_reader, &path.index_uid)? .ok_or(Error::internal( "Impossible to know if the database is indexing", ))?; @@ -86,7 +86,7 @@ async fn get_stats(data: web::Data) -> Result let fields_distribution = index.main.fields_distribution(&reader)?.unwrap_or_default(); - let is_indexing = data.is_indexing(&update_reader, &index_uid)?.ok_or( + let is_indexing = data.db.is_indexing(&update_reader, &index_uid)?.ok_or( Error::internal("Impossible to know if the database is indexing"), )?;