MeiliSearch/meilidb-core/src/update/synonyms_deletion.rs

158 lines
5.2 KiB
Rust
Raw Normal View History

2019-10-08 17:16:48 +02:00
use std::collections::BTreeMap;
use std::iter::FromIterator;
2019-10-18 13:05:28 +02:00
use fst::{set::OpBuilder, SetBuilder};
2019-10-08 17:16:48 +02:00
use sdset::SetBuf;
use crate::automaton::normalize_str;
2019-11-06 10:49:13 +01:00
use crate::database::{UpdateEvent, UpdateEventsEmitter};
2019-10-18 13:05:28 +02:00
use crate::update::{next_update_id, Update};
2019-10-08 17:31:07 +02:00
use crate::{store, MResult};
2019-10-08 17:16:48 +02:00
pub struct SynonymsDeletion {
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
2019-11-06 10:49:13 +01:00
updates_notifier: UpdateEventsEmitter,
2019-10-08 17:16:48 +02:00
synonyms: BTreeMap<String, Option<Vec<String>>>,
}
impl SynonymsDeletion {
pub fn new(
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
2019-11-06 10:49:13 +01:00
updates_notifier: UpdateEventsEmitter,
2019-10-18 13:05:28 +02:00
) -> SynonymsDeletion {
2019-10-08 17:16:48 +02:00
SynonymsDeletion {
updates_store,
updates_results_store,
updates_notifier,
synonyms: BTreeMap::new(),
}
}
pub fn delete_all_alternatives_of<S: AsRef<str>>(&mut self, synonym: S) {
let synonym = normalize_str(synonym.as_ref());
self.synonyms.insert(synonym, None);
}
pub fn delete_specific_alternatives_of<S, T, I>(&mut self, synonym: S, alternatives: I)
2019-10-18 13:05:28 +02:00
where
S: AsRef<str>,
T: AsRef<str>,
I: Iterator<Item = T>,
2019-10-08 17:16:48 +02:00
{
let synonym = normalize_str(synonym.as_ref());
let value = self.synonyms.entry(synonym).or_insert(None);
let alternatives = alternatives.map(|s| s.as_ref().to_lowercase());
match value {
Some(v) => v.extend(alternatives),
None => *value = Some(Vec::from_iter(alternatives)),
}
}
2019-10-21 12:05:53 +02:00
pub fn finalize(self, writer: &mut heed::RwTxn) -> MResult<u64> {
2019-11-06 10:49:13 +01:00
let _ = self.updates_notifier.send(UpdateEvent::NewUpdate);
2019-10-08 17:16:48 +02:00
let update_id = push_synonyms_deletion(
2019-10-11 11:29:47 +02:00
writer,
2019-10-08 17:16:48 +02:00
self.updates_store,
self.updates_results_store,
self.synonyms,
)?;
Ok(update_id)
}
}
pub fn push_synonyms_deletion(
2019-10-21 12:05:53 +02:00
writer: &mut heed::RwTxn,
updates_store: store::Updates,
updates_results_store: store::UpdatesResults,
deletion: BTreeMap<String, Option<Vec<String>>>,
2019-10-18 13:05:28 +02:00
) -> MResult<u64> {
let last_update_id = next_update_id(writer, updates_store, updates_results_store)?;
let update = Update::SynonymsDeletion(deletion);
2019-10-08 17:31:07 +02:00
updates_store.put_update(writer, last_update_id, &update)?;
Ok(last_update_id)
}
2019-10-08 17:16:48 +02:00
pub fn apply_synonyms_deletion(
2019-10-21 12:05:53 +02:00
writer: &mut heed::RwTxn,
2019-10-08 17:16:48 +02:00
main_store: store::Main,
synonyms_store: store::Synonyms,
deletion: BTreeMap<String, Option<Vec<String>>>,
2019-10-18 13:05:28 +02:00
) -> MResult<()> {
2019-10-08 17:16:48 +02:00
let mut delete_whole_synonym_builder = SetBuilder::memory();
for (synonym, alternatives) in deletion {
match alternatives {
Some(alternatives) => {
let prev_alternatives = synonyms_store.synonyms(writer, synonym.as_bytes())?;
let prev_alternatives = match prev_alternatives {
Some(alternatives) => alternatives,
None => continue,
};
let delta_alternatives = {
let alternatives = SetBuf::from_dirty(alternatives);
let mut builder = SetBuilder::memory();
builder.extend_iter(alternatives).unwrap();
2019-10-18 13:05:28 +02:00
builder.into_inner().and_then(fst::Set::from_bytes).unwrap()
2019-10-08 17:16:48 +02:00
};
let op = OpBuilder::new()
.add(prev_alternatives.stream())
.add(delta_alternatives.stream())
.difference();
let (alternatives, empty_alternatives) = {
let mut builder = SetBuilder::memory();
let len = builder.get_ref().len();
builder.extend_stream(op).unwrap();
let is_empty = len == builder.get_ref().len();
let bytes = builder.into_inner().unwrap();
let alternatives = fst::Set::from_bytes(bytes).unwrap();
(alternatives, is_empty)
};
if empty_alternatives {
delete_whole_synonym_builder.insert(synonym.as_bytes())?;
} else {
synonyms_store.put_synonyms(writer, synonym.as_bytes(), &alternatives)?;
}
2019-10-18 13:05:28 +02:00
}
2019-10-08 17:16:48 +02:00
None => {
delete_whole_synonym_builder.insert(&synonym).unwrap();
synonyms_store.del_synonyms(writer, synonym.as_bytes())?;
}
}
}
let delta_synonyms = delete_whole_synonym_builder
.into_inner()
.and_then(fst::Set::from_bytes)
.unwrap();
let synonyms = match main_store.synonyms_fst(writer)? {
Some(synonyms) => {
let op = OpBuilder::new()
.add(synonyms.stream())
.add(delta_synonyms.stream())
.difference();
let mut synonyms_builder = SetBuilder::memory();
synonyms_builder.extend_stream(op).unwrap();
synonyms_builder
.into_inner()
.and_then(fst::Set::from_bytes)
.unwrap()
2019-10-18 13:05:28 +02:00
}
2019-10-08 17:16:48 +02:00
None => fst::Set::default(),
};
main_store.put_synonyms_fst(writer, &synonyms)?;
Ok(())
}