mirror of
https://github.com/meilisearch/MeiliSearch
synced 2024-11-26 23:04:26 +01:00
feat: Make the update update serialization be based on message pack
This commit is contained in:
parent
91b44a2759
commit
cd864c40bc
@ -25,6 +25,11 @@ zerocopy = "0.2.8"
|
||||
git = "https://github.com/3Hren/msgpack-rust.git"
|
||||
rev = "40b3d48"
|
||||
|
||||
[dependencies.rmpv]
|
||||
git = "https://github.com/3Hren/msgpack-rust.git"
|
||||
rev = "40b3d48"
|
||||
features = ["with-serde"]
|
||||
|
||||
[dependencies.fst]
|
||||
git = "https://github.com/Kerollmops/fst.git"
|
||||
branch = "arc-byte-slice"
|
||||
|
@ -9,6 +9,8 @@ pub enum Error {
|
||||
MissingDocumentId,
|
||||
SledError(sled::Error),
|
||||
FstError(fst::Error),
|
||||
RmpDecodeError(rmp_serde::decode::Error),
|
||||
RmpEncodeError(rmp_serde::encode::Error),
|
||||
BincodeError(bincode::Error),
|
||||
SerializerError(SerializerError),
|
||||
}
|
||||
@ -25,6 +27,18 @@ impl From<fst::Error> for Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp_serde::decode::Error> for Error {
|
||||
fn from(error: rmp_serde::decode::Error) -> Error {
|
||||
Error::RmpDecodeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<rmp_serde::encode::Error> for Error {
|
||||
fn from(error: rmp_serde::encode::Error) -> Error {
|
||||
Error::RmpEncodeError(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bincode::Error> for Error {
|
||||
fn from(error: bincode::Error) -> Error {
|
||||
Error::BincodeError(error)
|
||||
@ -47,6 +61,8 @@ impl fmt::Display for Error {
|
||||
MissingDocumentId => write!(f, "document id is missing"),
|
||||
SledError(e) => write!(f, "Sled error; {}", e),
|
||||
FstError(e) => write!(f, "fst error; {}", e),
|
||||
RmpDecodeError(e) => write!(f, "rmp decode error; {}", e),
|
||||
RmpEncodeError(e) => write!(f, "rmp encode error; {}", e),
|
||||
BincodeError(e) => write!(f, "bincode error; {}", e),
|
||||
SerializerError(e) => write!(f, "serializer error; {}", e),
|
||||
}
|
||||
|
@ -44,15 +44,15 @@ fn event_is_set(event: &sled::Event) -> bool {
|
||||
|
||||
#[derive(Deserialize)]
|
||||
enum UpdateOwned {
|
||||
DocumentsAddition(Vec<serde_json::Value>),
|
||||
DocumentsAddition(Vec<rmpv::Value>),
|
||||
DocumentsDeletion(Vec<DocumentId>),
|
||||
SynonymsAddition(BTreeMap<String, Vec<String>>),
|
||||
SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
enum Update<D: serde::Serialize> {
|
||||
DocumentsAddition(Vec<D>),
|
||||
enum Update {
|
||||
DocumentsAddition(Vec<rmpv::Value>),
|
||||
DocumentsDeletion(Vec<DocumentId>),
|
||||
SynonymsAddition(BTreeMap<String, Vec<String>>),
|
||||
SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
|
||||
@ -72,7 +72,7 @@ fn spawn_update_system(index: Index) -> thread::JoinHandle<()> {
|
||||
|
||||
// this is an emulation of the try block (#31436)
|
||||
let result: Result<(), Error> = (|| {
|
||||
match bincode::deserialize(&update)? {
|
||||
match rmp_serde::from_read_ref(&update)? {
|
||||
UpdateOwned::DocumentsAddition(documents) => {
|
||||
let ranked_map = index.cache.load().ranked_map.clone();
|
||||
apply_documents_addition(&index, ranked_map, documents)?;
|
||||
@ -293,8 +293,15 @@ impl Index {
|
||||
pub(crate) fn push_documents_addition<D>(&self, addition: Vec<D>) -> Result<u64, Error>
|
||||
where D: serde::Serialize
|
||||
{
|
||||
let addition = Update::DocumentsAddition(addition);
|
||||
let update = bincode::serialize(&addition)?;
|
||||
let mut values = Vec::with_capacity(addition.len());
|
||||
for add in addition {
|
||||
let vec = rmp_serde::to_vec_named(&add)?;
|
||||
let add = rmp_serde::from_read(&vec[..])?;
|
||||
values.push(add);
|
||||
}
|
||||
|
||||
let addition = Update::DocumentsAddition(values);
|
||||
let update = rmp_serde::to_vec_named(&addition)?;
|
||||
self.raw_push_update(update)
|
||||
}
|
||||
|
||||
@ -303,8 +310,8 @@ impl Index {
|
||||
deletion: Vec<DocumentId>,
|
||||
) -> Result<u64, Error>
|
||||
{
|
||||
let deletion = Update::<()>::DocumentsDeletion(deletion);
|
||||
let update = bincode::serialize(&deletion)?;
|
||||
let deletion = Update::DocumentsDeletion(deletion);
|
||||
let update = rmp_serde::to_vec_named(&deletion)?;
|
||||
self.raw_push_update(update)
|
||||
}
|
||||
|
||||
@ -313,8 +320,8 @@ impl Index {
|
||||
addition: BTreeMap<String, Vec<String>>,
|
||||
) -> Result<u64, Error>
|
||||
{
|
||||
let addition = Update::<()>::SynonymsAddition(addition);
|
||||
let update = bincode::serialize(&addition)?;
|
||||
let addition = Update::SynonymsAddition(addition);
|
||||
let update = rmp_serde::to_vec_named(&addition)?;
|
||||
self.raw_push_update(update)
|
||||
}
|
||||
|
||||
@ -323,8 +330,8 @@ impl Index {
|
||||
deletion: BTreeMap<String, Option<Vec<String>>>,
|
||||
) -> Result<u64, Error>
|
||||
{
|
||||
let deletion = Update::<()>::SynonymsDeletion(deletion);
|
||||
let update = bincode::serialize(&deletion)?;
|
||||
let deletion = Update::SynonymsDeletion(deletion);
|
||||
let update = rmp_serde::to_vec_named(&deletion)?;
|
||||
self.raw_push_update(update)
|
||||
}
|
||||
|
||||
|
@ -35,7 +35,7 @@ impl<'a, D> DocumentsAddition<'a, D> {
|
||||
pub fn apply_documents_addition(
|
||||
index: &Index,
|
||||
mut ranked_map: RankedMap,
|
||||
addition: Vec<serde_json::Value>,
|
||||
addition: Vec<rmpv::Value>,
|
||||
) -> Result<(), Error>
|
||||
{
|
||||
let mut document_ids = HashSet::new();
|
||||
|
Loading…
Reference in New Issue
Block a user