Merge pull request #216 from meilisearch/get-ride-of-messagepack

Get ride of rust messagepack (rmp)

meilidb-core/Cargo.toml

@@ -20,21 +20,12 @@ once_cell = "1.2.0"
 ordered-float = { version = "1.0.2", features = ["serde"] }
 rkv = "0.10.2"
 sdset = "0.3.2"
-serde = { version = "1.0.99", features = ["derive"] }
-serde_json = "1.0.40"
+serde = { version = "1.0.101", features = ["derive"] }
+serde_json = "1.0.41"
 siphasher = "0.3.0"
 slice-group-by = "0.2.6"
 zerocopy = "0.2.8"
 
-[dependencies.rmp-serde]
-git = "https://github.com/3Hren/msgpack-rust.git"
-rev = "40b3d48"
-
-[dependencies.rmpv]
-git = "https://github.com/3Hren/msgpack-rust.git"
-rev = "40b3d48"
-features = ["with-serde"]
-
 [dependencies.levenshtein_automata]
 git = "https://github.com/Kerollmops/levenshtein-automata.git"
 branch = "arc-byte-slice"

meilidb-core/src/error.rs

@@ -1,4 +1,5 @@
 use std::{error, fmt, io};
+use serde_json::Error as SerdeJsonError;
 use crate::serde::{SerializerError, DeserializerError};
 
 pub type MResult<T> = Result<T, Error>;
@@ -13,8 +14,7 @@ pub enum Error {
     MissingDocumentId,
     Rkv(rkv::StoreError),
     Fst(fst::Error),
-    RmpDecode(rmp_serde::decode::Error),
-    RmpEncode(rmp_serde::encode::Error),
+    SerdeJson(SerdeJsonError),
     Bincode(bincode::Error),
     Serializer(SerializerError),
     Deserializer(DeserializerError),
@@ -39,15 +39,9 @@ impl From<fst::Error> for Error {
     }
 }
 
-impl From<rmp_serde::decode::Error> for Error {
-    fn from(error: rmp_serde::decode::Error) -> Error {
-        Error::RmpDecode(error)
-    }
-}
-
-impl From<rmp_serde::encode::Error> for Error {
-    fn from(error: rmp_serde::encode::Error) -> Error {
-        Error::RmpEncode(error)
+impl From<SerdeJsonError> for Error {
+    fn from(error: SerdeJsonError) -> Error {
+        Error::SerdeJson(error)
     }
 }
 
@@ -87,8 +81,7 @@ impl fmt::Display for Error {
             MissingDocumentId => write!(f, "document id is missing"),
             Rkv(e) => write!(f, "rkv error; {}", e),
             Fst(e) => write!(f, "fst error; {}", e),
-            RmpDecode(e) => write!(f, "rmp decode error; {}", e),
-            RmpEncode(e) => write!(f, "rmp encode error; {}", e),
+            SerdeJson(e) => write!(f, "serde json error; {}", e),
             Bincode(e) => write!(f, "bincode error; {}", e),
             Serializer(e) => write!(f, "serializer error; {}", e),
             Deserializer(e) => write!(f, "deserializer error; {}", e),

meilidb-core/src/serde/deserializer.rs

@@ -3,8 +3,9 @@ use std::io::Cursor;
 use std::{fmt, error::Error};
 
 use meilidb_schema::{Schema, SchemaAttr};
-use rmp_serde::decode::{Deserializer as RmpDeserializer, ReadReader};
-use rmp_serde::decode::{Error as RmpError};
+use serde_json::Error as SerdeJsonError;
+use serde_json::Deserializer as SerdeJsonDeserializer;
+use serde_json::de::IoRead as SerdeJsonIoRead;
 use serde::{de, forward_to_deserialize_any};
 
 use crate::store::DocumentsFields;
@@ -12,8 +13,8 @@ use crate::DocumentId;
 
 #[derive(Debug)]
 pub enum DeserializerError {
-    RmpError(RmpError),
-    RkvError(rkv::StoreError),
+    SerdeJson(SerdeJsonError),
+    Rkv(rkv::StoreError),
     Custom(String),
 }
 
@@ -26,8 +27,8 @@ impl de::Error for DeserializerError {
 impl fmt::Display for DeserializerError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            DeserializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e),
-            DeserializerError::RkvError(e) => write!(f, "rkv related error: {}", e),
+            DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e),
+            DeserializerError::Rkv(e) => write!(f, "rkv related error: {}", e),
             DeserializerError::Custom(s) => f.write_str(s),
         }
     }
@@ -35,15 +36,15 @@ impl fmt::Display for DeserializerError {
 
 impl Error for DeserializerError {}
 
-impl From<RmpError> for DeserializerError {
-    fn from(error: RmpError) -> DeserializerError {
-        DeserializerError::RmpError(error)
+impl From<SerdeJsonError> for DeserializerError {
+    fn from(error: SerdeJsonError) -> DeserializerError {
+        DeserializerError::SerdeJson(error)
     }
 }
 
 impl From<rkv::StoreError> for DeserializerError {
     fn from(error: rkv::StoreError) -> DeserializerError {
-        DeserializerError::RkvError(error)
+        DeserializerError::Rkv(error)
     }
 }
 
@@ -88,7 +89,12 @@ where R: rkv::Readable,
                 let is_displayed = self.schema.props(attr).is_displayed();
                 if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) {
                     let attribute_name = self.schema.attribute_name(attr);
-                    Some((attribute_name, Value::new(value)))
+
+                    let cursor = Cursor::new(value.to_owned());
+                    let ioread = SerdeJsonIoRead::new(cursor);
+                    let value = Value(SerdeJsonDeserializer::new(ioread));
+
+                    Some((attribute_name, value))
                 } else {
                     None
                 }
@@ -104,18 +110,9 @@ where R: rkv::Readable,
     }
 }
 
-struct Value<A>(RmpDeserializer<ReadReader<Cursor<A>>>) where A: AsRef<[u8]>;
+struct Value(SerdeJsonDeserializer<SerdeJsonIoRead<Cursor<Vec<u8>>>>);
 
-impl<A> Value<A> where A: AsRef<[u8]>
-{
-    fn new(value: A) -> Value<A> {
-        Value(RmpDeserializer::new(Cursor::new(value)))
-    }
-}
-
-impl<'de, A> de::IntoDeserializer<'de, RmpError> for Value<A>
-where A: AsRef<[u8]>,
-{
+impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value {
     type Deserializer = Self;
 
     fn into_deserializer(self) -> Self::Deserializer {
@@ -123,10 +120,8 @@ where A: AsRef<[u8]>,
     }
 }
 
-impl<'de, 'a, A> de::Deserializer<'de> for Value<A>
-where A: AsRef<[u8]>,
-{
-    type Error = RmpError;
+impl<'de> de::Deserializer<'de> for Value {
+    type Error = SerdeJsonError;
 
     fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
     where V: de::Visitor<'de>

meilidb-core/src/serde/mod.rs

@@ -26,7 +26,6 @@ use std::collections::BTreeMap;
 use std::{fmt, error::Error};
 
 use meilidb_schema::SchemaAttr;
-use rmp_serde::encode::Error as RmpError;
 use serde_json::Error as SerdeJsonError;
 use serde::ser;
 
@@ -36,10 +35,9 @@ use crate::{DocumentId, ParseNumberError};
 pub enum SerializerError {
     DocumentIdNotFound,
     InvalidDocumentIdType,
-    RmpError(RmpError),
     RkvError(rkv::StoreError),
-    SerdeJsonError(SerdeJsonError),
-    ParseNumberError(ParseNumberError),
+    SerdeJson(SerdeJsonError),
+    ParseNumber(ParseNumberError),
     UnserializableType { type_name: &'static str },
     UnindexableType { type_name: &'static str },
     UnrankableType { type_name: &'static str },
@@ -56,15 +54,14 @@ impl fmt::Display for SerializerError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
             SerializerError::DocumentIdNotFound => {
-                write!(f, "serialized document does not have an id according to the schema")
+                f.write_str("serialized document does not have an id according to the schema")
             },
             SerializerError::InvalidDocumentIdType => {
-                write!(f, "document identifier can only be of type string or number")
+                f.write_str("document identifier can only be of type string or number")
             },
-            SerializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e),
             SerializerError::RkvError(e) => write!(f, "rkv related error: {}", e),
-            SerializerError::SerdeJsonError(e) => write!(f, "serde json error: {}", e),
-            SerializerError::ParseNumberError(e) => {
+            SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
+            SerializerError::ParseNumber(e) => {
                 write!(f, "error while trying to parse a number: {}", e)
             },
             SerializerError::UnserializableType { type_name } => {
@@ -89,15 +86,9 @@ impl From<String> for SerializerError {
     }
 }
 
-impl From<RmpError> for SerializerError {
-    fn from(error: RmpError) -> SerializerError {
-        SerializerError::RmpError(error)
-    }
-}
-
 impl From<SerdeJsonError> for SerializerError {
     fn from(error: SerdeJsonError) -> SerializerError {
-        SerializerError::SerdeJsonError(error)
+        SerializerError::SerdeJson(error)
     }
 }
 
@@ -109,7 +100,7 @@ impl From<rkv::StoreError> for SerializerError {
 
 impl From<ParseNumberError> for SerializerError {
     fn from(error: ParseNumberError) -> SerializerError {
-        SerializerError::ParseNumberError(error)
+        SerializerError::ParseNumber(error)
     }
 }
 

meilidb-core/src/serde/serializer.rs

@@ -270,7 +270,7 @@ where T: ser::Serialize,
     if let Some(attribute) = schema.attribute(key) {
         let props = schema.props(attribute);
 
-        let serialized = rmp_serde::to_vec_named(value)?;
+        let serialized = serde_json::to_vec(value)?;
         document_store.set_document_field(document_id, attribute, serialized);
 
         if props.is_indexed() {

meilidb-core/src/store/mod.rs

@@ -104,7 +104,7 @@ impl Index {
     {
         let bytes = self.documents_fields.document_attribute(reader, document_id, attribute)?;
         match bytes {
-            Some(bytes) => Ok(Some(rmp_serde::from_read_ref(bytes)?)),
+            Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
             None => Ok(None),
         }
     }

meilidb-core/src/store/updates.rs

@@ -68,7 +68,7 @@ impl Updates {
     ) -> MResult<()>
     {
         let update_id_bytes = update_id.to_be_bytes();
-        let update = rmp_serde::to_vec_named(&update)?;
+        let update = serde_json::to_vec(&update)?;
         let blob = Value::Blob(&update);
         self.updates.put(writer, update_id_bytes, &blob)?;
         Ok(())
@@ -86,8 +86,7 @@ impl Updates {
 
         match first_data {
             Some(Value::Blob(bytes)) => {
-                let update = rmp_serde::from_read_ref(&bytes)?;
-
+                let update = serde_json::from_slice(&bytes)?;
                 // remove it from the database now
                 let first_id_bytes = first_id.to_be_bytes();
                 self.updates.delete(writer, first_id_bytes)?;

meilidb-core/src/update/documents_addition.rs

@@ -65,8 +65,8 @@ pub fn push_documents_addition<D: serde::Serialize>(
 {
     let mut values = Vec::with_capacity(addition.len());
     for add in addition {
-        let vec = rmp_serde::to_vec_named(&add)?;
-        let add = rmp_serde::from_read(&vec[..])?;
+        let vec = serde_json::to_vec(&add)?;
+        let add = serde_json::from_slice(&vec)?;
         values.push(add);
     }
 
@@ -85,7 +85,7 @@ pub fn apply_documents_addition(
     postings_lists_store: store::PostingsLists,
     docs_words_store: store::DocsWords,
     mut ranked_map: RankedMap,
-    addition: Vec<rmpv::Value>,
+    addition: Vec<serde_json::Value>,
 ) -> MResult<()>
 {
     let mut document_ids = HashSet::new();

meilidb-core/src/update/mod.rs

@@ -26,7 +26,7 @@ use meilidb_schema::Schema;
 pub enum Update {
     Schema(Schema),
     Customs(Vec<u8>),
-    DocumentsAddition(Vec<rmpv::Value>),
+    DocumentsAddition(Vec<serde_json::Value>),
     DocumentsDeletion(Vec<DocumentId>),
     SynonymsAddition(BTreeMap<String, Vec<String>>),
     SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),