From 710ab2386c17950f8e75529efde76f7d7815fe36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Fri, 11 Oct 2019 16:16:21 +0200 Subject: [PATCH] Get ride of rust messagepack (rmp) --- meilidb-core/Cargo.toml | 13 +---- meilidb-core/src/error.rs | 19 +++----- meilidb-core/src/serde/deserializer.rs | 47 +++++++++---------- meilidb-core/src/serde/mod.rs | 25 ++++------ meilidb-core/src/serde/serializer.rs | 2 +- meilidb-core/src/store/mod.rs | 2 +- meilidb-core/src/store/updates.rs | 5 +- meilidb-core/src/update/documents_addition.rs | 6 +-- meilidb-core/src/update/mod.rs | 2 +- 9 files changed, 45 insertions(+), 76 deletions(-) diff --git a/meilidb-core/Cargo.toml b/meilidb-core/Cargo.toml index f9dcb6d74..b73d1390a 100644 --- a/meilidb-core/Cargo.toml +++ b/meilidb-core/Cargo.toml @@ -20,21 +20,12 @@ once_cell = "1.2.0" ordered-float = { version = "1.0.2", features = ["serde"] } rkv = "0.10.2" sdset = "0.3.2" -serde = { version = "1.0.99", features = ["derive"] } -serde_json = "1.0.40" +serde = { version = "1.0.101", features = ["derive"] } +serde_json = "1.0.41" siphasher = "0.3.0" slice-group-by = "0.2.6" zerocopy = "0.2.8" -[dependencies.rmp-serde] -git = "https://github.com/3Hren/msgpack-rust.git" -rev = "40b3d48" - -[dependencies.rmpv] -git = "https://github.com/3Hren/msgpack-rust.git" -rev = "40b3d48" -features = ["with-serde"] - [dependencies.levenshtein_automata] git = "https://github.com/Kerollmops/levenshtein-automata.git" branch = "arc-byte-slice" diff --git a/meilidb-core/src/error.rs b/meilidb-core/src/error.rs index cc600b388..767337a80 100644 --- a/meilidb-core/src/error.rs +++ b/meilidb-core/src/error.rs @@ -1,4 +1,5 @@ use std::{error, fmt, io}; +use serde_json::Error as SerdeJsonError; use crate::serde::{SerializerError, DeserializerError}; pub type MResult = Result; @@ -13,8 +14,7 @@ pub enum Error { MissingDocumentId, Rkv(rkv::StoreError), Fst(fst::Error), - RmpDecode(rmp_serde::decode::Error), - RmpEncode(rmp_serde::encode::Error), + SerdeJson(SerdeJsonError), Bincode(bincode::Error), Serializer(SerializerError), Deserializer(DeserializerError), @@ -39,15 +39,9 @@ impl From for Error { } } -impl From for Error { - fn from(error: rmp_serde::decode::Error) -> Error { - Error::RmpDecode(error) - } -} - -impl From for Error { - fn from(error: rmp_serde::encode::Error) -> Error { - Error::RmpEncode(error) +impl From for Error { + fn from(error: SerdeJsonError) -> Error { + Error::SerdeJson(error) } } @@ -87,8 +81,7 @@ impl fmt::Display for Error { MissingDocumentId => write!(f, "document id is missing"), Rkv(e) => write!(f, "rkv error; {}", e), Fst(e) => write!(f, "fst error; {}", e), - RmpDecode(e) => write!(f, "rmp decode error; {}", e), - RmpEncode(e) => write!(f, "rmp encode error; {}", e), + SerdeJson(e) => write!(f, "serde json error; {}", e), Bincode(e) => write!(f, "bincode error; {}", e), Serializer(e) => write!(f, "serializer error; {}", e), Deserializer(e) => write!(f, "deserializer error; {}", e), diff --git a/meilidb-core/src/serde/deserializer.rs b/meilidb-core/src/serde/deserializer.rs index ebf008eb7..b9cd3a1d5 100644 --- a/meilidb-core/src/serde/deserializer.rs +++ b/meilidb-core/src/serde/deserializer.rs @@ -3,8 +3,9 @@ use std::io::Cursor; use std::{fmt, error::Error}; use meilidb_schema::{Schema, SchemaAttr}; -use rmp_serde::decode::{Deserializer as RmpDeserializer, ReadReader}; -use rmp_serde::decode::{Error as RmpError}; +use serde_json::Error as SerdeJsonError; +use serde_json::Deserializer as SerdeJsonDeserializer; +use serde_json::de::IoRead as SerdeJsonIoRead; use serde::{de, forward_to_deserialize_any}; use crate::store::DocumentsFields; @@ -12,8 +13,8 @@ use crate::DocumentId; #[derive(Debug)] pub enum DeserializerError { - RmpError(RmpError), - RkvError(rkv::StoreError), + SerdeJson(SerdeJsonError), + Rkv(rkv::StoreError), Custom(String), } @@ -26,8 +27,8 @@ impl de::Error for DeserializerError { impl fmt::Display for DeserializerError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - DeserializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e), - DeserializerError::RkvError(e) => write!(f, "rkv related error: {}", e), + DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e), + DeserializerError::Rkv(e) => write!(f, "rkv related error: {}", e), DeserializerError::Custom(s) => f.write_str(s), } } @@ -35,15 +36,15 @@ impl fmt::Display for DeserializerError { impl Error for DeserializerError {} -impl From for DeserializerError { - fn from(error: RmpError) -> DeserializerError { - DeserializerError::RmpError(error) +impl From for DeserializerError { + fn from(error: SerdeJsonError) -> DeserializerError { + DeserializerError::SerdeJson(error) } } impl From for DeserializerError { fn from(error: rkv::StoreError) -> DeserializerError { - DeserializerError::RkvError(error) + DeserializerError::Rkv(error) } } @@ -88,7 +89,12 @@ where R: rkv::Readable, let is_displayed = self.schema.props(attr).is_displayed(); if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) { let attribute_name = self.schema.attribute_name(attr); - Some((attribute_name, Value::new(value))) + + let cursor = Cursor::new(value.to_owned()); + let ioread = SerdeJsonIoRead::new(cursor); + let value = Value(SerdeJsonDeserializer::new(ioread)); + + Some((attribute_name, value)) } else { None } @@ -104,18 +110,9 @@ where R: rkv::Readable, } } -struct Value(RmpDeserializer>>) where A: AsRef<[u8]>; +struct Value(SerdeJsonDeserializer>>>); -impl Value where A: AsRef<[u8]> -{ - fn new(value: A) -> Value { - Value(RmpDeserializer::new(Cursor::new(value))) - } -} - -impl<'de, A> de::IntoDeserializer<'de, RmpError> for Value -where A: AsRef<[u8]>, -{ +impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value { type Deserializer = Self; fn into_deserializer(self) -> Self::Deserializer { @@ -123,10 +120,8 @@ where A: AsRef<[u8]>, } } -impl<'de, 'a, A> de::Deserializer<'de> for Value -where A: AsRef<[u8]>, -{ - type Error = RmpError; +impl<'de> de::Deserializer<'de> for Value { + type Error = SerdeJsonError; fn deserialize_any(mut self, visitor: V) -> Result where V: de::Visitor<'de> diff --git a/meilidb-core/src/serde/mod.rs b/meilidb-core/src/serde/mod.rs index e3af21f89..6846f14d2 100644 --- a/meilidb-core/src/serde/mod.rs +++ b/meilidb-core/src/serde/mod.rs @@ -26,7 +26,6 @@ use std::collections::BTreeMap; use std::{fmt, error::Error}; use meilidb_schema::SchemaAttr; -use rmp_serde::encode::Error as RmpError; use serde_json::Error as SerdeJsonError; use serde::ser; @@ -36,10 +35,9 @@ use crate::{DocumentId, ParseNumberError}; pub enum SerializerError { DocumentIdNotFound, InvalidDocumentIdType, - RmpError(RmpError), RkvError(rkv::StoreError), - SerdeJsonError(SerdeJsonError), - ParseNumberError(ParseNumberError), + SerdeJson(SerdeJsonError), + ParseNumber(ParseNumberError), UnserializableType { type_name: &'static str }, UnindexableType { type_name: &'static str }, UnrankableType { type_name: &'static str }, @@ -56,15 +54,14 @@ impl fmt::Display for SerializerError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { SerializerError::DocumentIdNotFound => { - write!(f, "serialized document does not have an id according to the schema") + f.write_str("serialized document does not have an id according to the schema") }, SerializerError::InvalidDocumentIdType => { - write!(f, "document identifier can only be of type string or number") + f.write_str("document identifier can only be of type string or number") }, - SerializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e), SerializerError::RkvError(e) => write!(f, "rkv related error: {}", e), - SerializerError::SerdeJsonError(e) => write!(f, "serde json error: {}", e), - SerializerError::ParseNumberError(e) => { + SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e), + SerializerError::ParseNumber(e) => { write!(f, "error while trying to parse a number: {}", e) }, SerializerError::UnserializableType { type_name } => { @@ -89,15 +86,9 @@ impl From for SerializerError { } } -impl From for SerializerError { - fn from(error: RmpError) -> SerializerError { - SerializerError::RmpError(error) - } -} - impl From for SerializerError { fn from(error: SerdeJsonError) -> SerializerError { - SerializerError::SerdeJsonError(error) + SerializerError::SerdeJson(error) } } @@ -109,7 +100,7 @@ impl From for SerializerError { impl From for SerializerError { fn from(error: ParseNumberError) -> SerializerError { - SerializerError::ParseNumberError(error) + SerializerError::ParseNumber(error) } } diff --git a/meilidb-core/src/serde/serializer.rs b/meilidb-core/src/serde/serializer.rs index 8764ce526..57e436c5a 100644 --- a/meilidb-core/src/serde/serializer.rs +++ b/meilidb-core/src/serde/serializer.rs @@ -270,7 +270,7 @@ where T: ser::Serialize, if let Some(attribute) = schema.attribute(key) { let props = schema.props(attribute); - let serialized = rmp_serde::to_vec_named(value)?; + let serialized = serde_json::to_vec(value)?; document_store.set_document_field(document_id, attribute, serialized); if props.is_indexed() { diff --git a/meilidb-core/src/store/mod.rs b/meilidb-core/src/store/mod.rs index 957ee8520..183ec7b0b 100644 --- a/meilidb-core/src/store/mod.rs +++ b/meilidb-core/src/store/mod.rs @@ -104,7 +104,7 @@ impl Index { { let bytes = self.documents_fields.document_attribute(reader, document_id, attribute)?; match bytes { - Some(bytes) => Ok(Some(rmp_serde::from_read_ref(bytes)?)), + Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)), None => Ok(None), } } diff --git a/meilidb-core/src/store/updates.rs b/meilidb-core/src/store/updates.rs index 941abba6b..0f5ce2629 100644 --- a/meilidb-core/src/store/updates.rs +++ b/meilidb-core/src/store/updates.rs @@ -68,7 +68,7 @@ impl Updates { ) -> MResult<()> { let update_id_bytes = update_id.to_be_bytes(); - let update = rmp_serde::to_vec_named(&update)?; + let update = serde_json::to_vec(&update)?; let blob = Value::Blob(&update); self.updates.put(writer, update_id_bytes, &blob)?; Ok(()) @@ -86,8 +86,7 @@ impl Updates { match first_data { Some(Value::Blob(bytes)) => { - let update = rmp_serde::from_read_ref(&bytes)?; - + let update = serde_json::from_slice(&bytes)?; // remove it from the database now let first_id_bytes = first_id.to_be_bytes(); self.updates.delete(writer, first_id_bytes)?; diff --git a/meilidb-core/src/update/documents_addition.rs b/meilidb-core/src/update/documents_addition.rs index 31107665c..a9420a528 100644 --- a/meilidb-core/src/update/documents_addition.rs +++ b/meilidb-core/src/update/documents_addition.rs @@ -65,8 +65,8 @@ pub fn push_documents_addition( { let mut values = Vec::with_capacity(addition.len()); for add in addition { - let vec = rmp_serde::to_vec_named(&add)?; - let add = rmp_serde::from_read(&vec[..])?; + let vec = serde_json::to_vec(&add)?; + let add = serde_json::from_slice(&vec)?; values.push(add); } @@ -85,7 +85,7 @@ pub fn apply_documents_addition( postings_lists_store: store::PostingsLists, docs_words_store: store::DocsWords, mut ranked_map: RankedMap, - addition: Vec, + addition: Vec, ) -> MResult<()> { let mut document_ids = HashSet::new(); diff --git a/meilidb-core/src/update/mod.rs b/meilidb-core/src/update/mod.rs index 794c2f8ec..2493a1c94 100644 --- a/meilidb-core/src/update/mod.rs +++ b/meilidb-core/src/update/mod.rs @@ -26,7 +26,7 @@ use meilidb_schema::Schema; pub enum Update { Schema(Schema), Customs(Vec), - DocumentsAddition(Vec), + DocumentsAddition(Vec), DocumentsDeletion(Vec), SynonymsAddition(BTreeMap>), SynonymsDeletion(BTreeMap>>),