use std::borrow::Cow; use std::collections::HashMap; use chrono::{DateTime, Utc}; use heed::types::{ByteSlice, OwnedType, SerdeBincode, Str}; use meilisearch_schema::{FieldId, Schema}; use meilisearch_types::DocumentId; use sdset::Set; use crate::database::MainT; use crate::{RankedMap, MResult}; use crate::settings::RankingRule; use crate::{FstSetCow, FstMapCow}; use super::{CowSet, DocumentsIds}; const ATTRIBUTES_FOR_FACETING_KEY: &str = "attributes-for-faceting"; const CREATED_AT_KEY: &str = "created-at"; const CUSTOMS_KEY: &str = "customs"; const DISTINCT_ATTRIBUTE_KEY: &str = "distinct-attribute"; const EXTERNAL_DOCIDS_KEY: &str = "external-docids"; const FIELDS_DISTRIBUTION_KEY: &str = "fields-distribution"; const INTERNAL_DOCIDS_KEY: &str = "internal-docids"; const NAME_KEY: &str = "name"; const NUMBER_OF_DOCUMENTS_KEY: &str = "number-of-documents"; const RANKED_MAP_KEY: &str = "ranked-map"; const RANKING_RULES_KEY: &str = "ranking-rules"; const SCHEMA_KEY: &str = "schema"; const STOP_WORDS_KEY: &str = "stop-words"; const SYNONYMS_KEY: &str = "synonyms"; const UPDATED_AT_KEY: &str = "updated-at"; const WORDS_KEY: &str = "words"; pub type FreqsMap = HashMap; type SerdeFreqsMap = SerdeBincode; type SerdeDatetime = SerdeBincode>; #[derive(Copy, Clone)] pub struct Main { pub(crate) main: heed::PolyDatabase, } impl Main { pub fn clear(self, writer: &mut heed::RwTxn) -> MResult<()> { Ok(self.main.clear(writer)?) } pub fn put_name(self, writer: &mut heed::RwTxn, name: &str) -> MResult<()> { Ok(self.main.put::<_, Str, Str>(writer, NAME_KEY, name)?) } pub fn name(self, reader: &heed::RoTxn) -> MResult> { Ok(self .main .get::<_, Str, Str>(reader, NAME_KEY)? .map(|name| name.to_owned())) } pub fn put_created_at(self, writer: &mut heed::RwTxn) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeDatetime>(writer, CREATED_AT_KEY, &Utc::now())?) } pub fn created_at(self, reader: &heed::RoTxn) -> MResult>> { Ok(self.main.get::<_, Str, SerdeDatetime>(reader, CREATED_AT_KEY)?) } pub fn put_updated_at(self, writer: &mut heed::RwTxn) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeDatetime>(writer, UPDATED_AT_KEY, &Utc::now())?) } pub fn updated_at(self, reader: &heed::RoTxn) -> MResult>> { Ok(self.main.get::<_, Str, SerdeDatetime>(reader, UPDATED_AT_KEY)?) } pub fn put_internal_docids(self, writer: &mut heed::RwTxn, ids: &sdset::Set) -> MResult<()> { Ok(self.main.put::<_, Str, DocumentsIds>(writer, INTERNAL_DOCIDS_KEY, ids)?) } pub fn internal_docids<'txn>(self, reader: &'txn heed::RoTxn) -> MResult>> { match self.main.get::<_, Str, DocumentsIds>(reader, INTERNAL_DOCIDS_KEY)? { Some(ids) => Ok(ids), None => Ok(Cow::default()), } } pub fn merge_internal_docids(self, writer: &mut heed::RwTxn, new_ids: &sdset::Set) -> MResult<()> { use sdset::SetOperation; // We do an union of the old and new internal ids. let internal_docids = self.internal_docids(writer)?; let internal_docids = sdset::duo::Union::new(&internal_docids, new_ids).into_set_buf(); Ok(self.put_internal_docids(writer, &internal_docids)?) } pub fn remove_internal_docids(self, writer: &mut heed::RwTxn, ids: &sdset::Set) -> MResult<()> { use sdset::SetOperation; // We do a difference of the old and new internal ids. let internal_docids = self.internal_docids(writer)?; let internal_docids = sdset::duo::Difference::new(&internal_docids, ids).into_set_buf(); Ok(self.put_internal_docids(writer, &internal_docids)?) } pub fn put_external_docids(self, writer: &mut heed::RwTxn, ids: &fst::Map) -> MResult<()> where A: AsRef<[u8]>, { Ok(self.main.put::<_, Str, ByteSlice>(writer, EXTERNAL_DOCIDS_KEY, ids.as_fst().as_bytes())?) } pub fn merge_external_docids(self, writer: &mut heed::RwTxn, new_docids: &fst::Map) -> MResult<()> where A: AsRef<[u8]>, { use fst::{Streamer, IntoStreamer}; // Do an union of the old and the new set of external docids. let external_docids = self.external_docids(writer)?; let mut op = external_docids.op().add(new_docids.into_stream()).r#union(); let mut build = fst::MapBuilder::memory(); while let Some((docid, values)) = op.next() { build.insert(docid, values[0].value).unwrap(); } drop(op); let external_docids = build.into_map(); Ok(self.put_external_docids(writer, &external_docids)?) } pub fn remove_external_docids(self, writer: &mut heed::RwTxn, ids: &fst::Map) -> MResult<()> where A: AsRef<[u8]>, { use fst::{Streamer, IntoStreamer}; // Do an union of the old and the new set of external docids. let external_docids = self.external_docids(writer)?; let mut op = external_docids.op().add(ids.into_stream()).difference(); let mut build = fst::MapBuilder::memory(); while let Some((docid, values)) = op.next() { build.insert(docid, values[0].value).unwrap(); } drop(op); let external_docids = build.into_map(); self.put_external_docids(writer, &external_docids) } pub fn external_docids(self, reader: &heed::RoTxn) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, EXTERNAL_DOCIDS_KEY)? { Some(bytes) => Ok(fst::Map::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Map::default().map_data(Cow::Owned).unwrap()), } } pub fn external_to_internal_docid(self, reader: &heed::RoTxn, external_docid: &str) -> MResult> { let external_ids = self.external_docids(reader)?; Ok(external_ids.get(external_docid).map(|id| DocumentId(id as u32))) } pub fn words_fst(self, reader: &heed::RoTxn) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, WORDS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), } } pub fn put_words_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { Ok(self.main.put::<_, Str, ByteSlice>(writer, WORDS_KEY, fst.as_fst().as_bytes())?) } pub fn put_schema(self, writer: &mut heed::RwTxn, schema: &Schema) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeBincode>(writer, SCHEMA_KEY, schema)?) } pub fn schema(self, reader: &heed::RoTxn) -> MResult> { Ok(self.main.get::<_, Str, SerdeBincode>(reader, SCHEMA_KEY)?) } pub fn delete_schema(self, writer: &mut heed::RwTxn) -> MResult { Ok(self.main.delete::<_, Str>(writer, SCHEMA_KEY)?) } pub fn put_ranked_map(self, writer: &mut heed::RwTxn, ranked_map: &RankedMap) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeBincode>(writer, RANKED_MAP_KEY, &ranked_map)?) } pub fn ranked_map(self, reader: &heed::RoTxn) -> MResult> { Ok(self.main.get::<_, Str, SerdeBincode>(reader, RANKED_MAP_KEY)?) } pub fn put_synonyms_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { let bytes = fst.as_fst().as_bytes(); Ok(self.main.put::<_, Str, ByteSlice>(writer, SYNONYMS_KEY, bytes)?) } pub(crate) fn synonyms_fst(self, reader: &heed::RoTxn) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, SYNONYMS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), } } pub fn synonyms(self, reader: &heed::RoTxn) -> MResult> { let synonyms = self .synonyms_fst(&reader)? .stream() .into_strs()?; Ok(synonyms) } pub fn put_stop_words_fst>(self, writer: &mut heed::RwTxn, fst: &fst::Set) -> MResult<()> { let bytes = fst.as_fst().as_bytes(); Ok(self.main.put::<_, Str, ByteSlice>(writer, STOP_WORDS_KEY, bytes)?) } pub(crate) fn stop_words_fst(self, reader: &heed::RoTxn) -> MResult { match self.main.get::<_, Str, ByteSlice>(reader, STOP_WORDS_KEY)? { Some(bytes) => Ok(fst::Set::new(bytes).unwrap().map_data(Cow::Borrowed).unwrap()), None => Ok(fst::Set::default().map_data(Cow::Owned).unwrap()), } } pub fn stop_words(self, reader: &heed::RoTxn) -> MResult> { let stop_word_list = self .stop_words_fst(reader)? .stream() .into_strs()?; Ok(stop_word_list) } pub fn put_number_of_documents(self, writer: &mut heed::RwTxn, f: F) -> MResult where F: Fn(u64) -> u64, { let new = self.number_of_documents(&*writer).map(f)?; self.main .put::<_, Str, OwnedType>(writer, NUMBER_OF_DOCUMENTS_KEY, &new)?; Ok(new) } pub fn number_of_documents(self, reader: &heed::RoTxn) -> MResult { match self .main .get::<_, Str, OwnedType>(reader, NUMBER_OF_DOCUMENTS_KEY)? { Some(value) => Ok(value), None => Ok(0), } } pub fn put_fields_distribution( self, writer: &mut heed::RwTxn, fields_frequency: &FreqsMap, ) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeFreqsMap>(writer, FIELDS_DISTRIBUTION_KEY, fields_frequency)?) } pub fn fields_distribution(&self, reader: &heed::RoTxn) -> MResult> { match self .main .get::<_, Str, SerdeFreqsMap>(reader, FIELDS_DISTRIBUTION_KEY)? { Some(freqs) => Ok(Some(freqs)), None => Ok(None), } } pub fn attributes_for_faceting<'txn>(&self, reader: &'txn heed::RoTxn) -> MResult>>> { Ok(self.main.get::<_, Str, CowSet>(reader, ATTRIBUTES_FOR_FACETING_KEY)?) } pub fn put_attributes_for_faceting(self, writer: &mut heed::RwTxn, attributes: &Set) -> MResult<()> { Ok(self.main.put::<_, Str, CowSet>(writer, ATTRIBUTES_FOR_FACETING_KEY, attributes)?) } pub fn delete_attributes_for_faceting(self, writer: &mut heed::RwTxn) -> MResult { Ok(self.main.delete::<_, Str>(writer, ATTRIBUTES_FOR_FACETING_KEY)?) } pub fn ranking_rules(&self, reader: &heed::RoTxn) -> MResult>> { Ok(self.main.get::<_, Str, SerdeBincode>>(reader, RANKING_RULES_KEY)?) } pub fn put_ranking_rules(self, writer: &mut heed::RwTxn, value: &[RankingRule]) -> MResult<()> { Ok(self.main.put::<_, Str, SerdeBincode>>(writer, RANKING_RULES_KEY, &value.to_vec())?) } pub fn delete_ranking_rules(self, writer: &mut heed::RwTxn) -> MResult { Ok(self.main.delete::<_, Str>(writer, RANKING_RULES_KEY)?) } pub fn distinct_attribute(&self, reader: &heed::RoTxn) -> MResult> { if let Some(value) = self.main.get::<_, Str, Str>(reader, DISTINCT_ATTRIBUTE_KEY)? { return Ok(Some(value.to_owned())) } return Ok(None) } pub fn put_distinct_attribute(self, writer: &mut heed::RwTxn, value: &str) -> MResult<()> { Ok(self.main.put::<_, Str, Str>(writer, DISTINCT_ATTRIBUTE_KEY, value)?) } pub fn delete_distinct_attribute(self, writer: &mut heed::RwTxn) -> MResult { Ok(self.main.delete::<_, Str>(writer, DISTINCT_ATTRIBUTE_KEY)?) } pub fn put_customs(self, writer: &mut heed::RwTxn, customs: &[u8]) -> MResult<()> { Ok(self.main.put::<_, Str, ByteSlice>(writer, CUSTOMS_KEY, customs)?) } pub fn customs<'txn>(self, reader: &'txn heed::RoTxn) -> MResult> { Ok(self.main.get::<_, Str, ByteSlice>(reader, CUSTOMS_KEY)?) } }