feat: Make the update update serialization be based on message pack

meilidb-data/Cargo.toml

@@ -25,6 +25,11 @@ zerocopy = "0.2.8"
 git = "https://github.com/3Hren/msgpack-rust.git"
 rev = "40b3d48"
 
+[dependencies.rmpv]
+git = "https://github.com/3Hren/msgpack-rust.git"
+rev = "40b3d48"
+features = ["with-serde"]
+
 [dependencies.fst]
 git = "https://github.com/Kerollmops/fst.git"
 branch = "arc-byte-slice"

meilidb-data/src/database/error.rs

@@ -9,6 +9,8 @@ pub enum Error {
     MissingDocumentId,
     SledError(sled::Error),
     FstError(fst::Error),
+    RmpDecodeError(rmp_serde::decode::Error),
+    RmpEncodeError(rmp_serde::encode::Error),
     BincodeError(bincode::Error),
     SerializerError(SerializerError),
 }
@@ -25,6 +27,18 @@ impl From<fst::Error> for Error {
     }
 }
 
+impl From<rmp_serde::decode::Error> for Error {
+    fn from(error: rmp_serde::decode::Error) -> Error {
+        Error::RmpDecodeError(error)
+    }
+}
+
+impl From<rmp_serde::encode::Error> for Error {
+    fn from(error: rmp_serde::encode::Error) -> Error {
+        Error::RmpEncodeError(error)
+    }
+}
+
 impl From<bincode::Error> for Error {
     fn from(error: bincode::Error) -> Error {
         Error::BincodeError(error)
@@ -47,6 +61,8 @@ impl fmt::Display for Error {
             MissingDocumentId => write!(f, "document id is missing"),
             SledError(e) => write!(f, "Sled error; {}", e),
             FstError(e) => write!(f, "fst error; {}", e),
+            RmpDecodeError(e) => write!(f, "rmp decode error; {}", e),
+            RmpEncodeError(e) => write!(f, "rmp encode error; {}", e),
             BincodeError(e) => write!(f, "bincode error; {}", e),
             SerializerError(e) => write!(f, "serializer error; {}", e),
         }

meilidb-data/src/database/index/mod.rs

@@ -44,15 +44,15 @@ fn event_is_set(event: &sled::Event) -> bool {
 
 #[derive(Deserialize)]
 enum UpdateOwned {
-    DocumentsAddition(Vec<serde_json::Value>),
+    DocumentsAddition(Vec<rmpv::Value>),
     DocumentsDeletion(Vec<DocumentId>),
     SynonymsAddition(BTreeMap<String, Vec<String>>),
     SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
 }
 
 #[derive(Serialize)]
-enum Update<D: serde::Serialize> {
+enum Update {
-    DocumentsAddition(Vec<D>),
+    DocumentsAddition(Vec<rmpv::Value>),
     DocumentsDeletion(Vec<DocumentId>),
     SynonymsAddition(BTreeMap<String, Vec<String>>),
     SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),
@@ -72,7 +72,7 @@ fn spawn_update_system(index: Index) -> thread::JoinHandle<()> {
 
                     // this is an emulation of the try block (#31436)
                     let result: Result<(), Error> = (|| {
-                        match bincode::deserialize(&update)? {
+                        match rmp_serde::from_read_ref(&update)? {
                             UpdateOwned::DocumentsAddition(documents) => {
                                 let ranked_map = index.cache.load().ranked_map.clone();
                                 apply_documents_addition(&index, ranked_map, documents)?;
@@ -293,8 +293,15 @@ impl Index {
     pub(crate) fn push_documents_addition<D>(&self, addition: Vec<D>) -> Result<u64, Error>
     where D: serde::Serialize
     {
-        let addition = Update::DocumentsAddition(addition);
+        let mut values = Vec::with_capacity(addition.len());
-        let update = bincode::serialize(&addition)?;
+        for add in addition {
+            let vec = rmp_serde::to_vec_named(&add)?;
+            let add = rmp_serde::from_read(&vec[..])?;
+            values.push(add);
+        }
+
+        let addition = Update::DocumentsAddition(values);
+        let update = rmp_serde::to_vec_named(&addition)?;
         self.raw_push_update(update)
     }
 
@@ -303,8 +310,8 @@ impl Index {
         deletion: Vec<DocumentId>,
     ) -> Result<u64, Error>
     {
-        let deletion = Update::<()>::DocumentsDeletion(deletion);
+        let deletion = Update::DocumentsDeletion(deletion);
-        let update = bincode::serialize(&deletion)?;
+        let update = rmp_serde::to_vec_named(&deletion)?;
         self.raw_push_update(update)
     }
 
@@ -313,8 +320,8 @@ impl Index {
         addition: BTreeMap<String, Vec<String>>,
     ) -> Result<u64, Error>
     {
-        let addition = Update::<()>::SynonymsAddition(addition);
+        let addition = Update::SynonymsAddition(addition);
-        let update = bincode::serialize(&addition)?;
+        let update = rmp_serde::to_vec_named(&addition)?;
         self.raw_push_update(update)
     }
 
@@ -323,8 +330,8 @@ impl Index {
         deletion: BTreeMap<String, Option<Vec<String>>>,
     ) -> Result<u64, Error>
     {
-        let deletion = Update::<()>::SynonymsDeletion(deletion);
+        let deletion = Update::SynonymsDeletion(deletion);
-        let update = bincode::serialize(&deletion)?;
+        let update = rmp_serde::to_vec_named(&deletion)?;
         self.raw_push_update(update)
     }
 

meilidb-data/src/database/update/documents_addition.rs

@@ -35,7 +35,7 @@ impl<'a, D> DocumentsAddition<'a, D> {
 pub fn apply_documents_addition(
     index: &Index,
     mut ranked_map: RankedMap,
-    addition: Vec<serde_json::Value>,
+    addition: Vec<rmpv::Value>,
 ) -> Result<(), Error>
 {
     let mut document_ids = HashSet::new();