Merge pull request #216 from meilisearch/get-ride-of-messagepack

Get ride of rust messagepack (rmp)
This commit is contained in:
Clément Renault 2019-10-11 16:41:37 +02:00 committed by GitHub
commit b7ea812dcc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 45 additions and 76 deletions

View File

@ -20,21 +20,12 @@ once_cell = "1.2.0"
ordered-float = { version = "1.0.2", features = ["serde"] }
rkv = "0.10.2"
sdset = "0.3.2"
serde = { version = "1.0.99", features = ["derive"] }
serde_json = "1.0.40"
serde = { version = "1.0.101", features = ["derive"] }
serde_json = "1.0.41"
siphasher = "0.3.0"
slice-group-by = "0.2.6"
zerocopy = "0.2.8"
[dependencies.rmp-serde]
git = "https://github.com/3Hren/msgpack-rust.git"
rev = "40b3d48"
[dependencies.rmpv]
git = "https://github.com/3Hren/msgpack-rust.git"
rev = "40b3d48"
features = ["with-serde"]
[dependencies.levenshtein_automata]
git = "https://github.com/Kerollmops/levenshtein-automata.git"
branch = "arc-byte-slice"

View File

@ -1,4 +1,5 @@
use std::{error, fmt, io};
use serde_json::Error as SerdeJsonError;
use crate::serde::{SerializerError, DeserializerError};
pub type MResult<T> = Result<T, Error>;
@ -13,8 +14,7 @@ pub enum Error {
MissingDocumentId,
Rkv(rkv::StoreError),
Fst(fst::Error),
RmpDecode(rmp_serde::decode::Error),
RmpEncode(rmp_serde::encode::Error),
SerdeJson(SerdeJsonError),
Bincode(bincode::Error),
Serializer(SerializerError),
Deserializer(DeserializerError),
@ -39,15 +39,9 @@ impl From<fst::Error> for Error {
}
}
impl From<rmp_serde::decode::Error> for Error {
fn from(error: rmp_serde::decode::Error) -> Error {
Error::RmpDecode(error)
}
}
impl From<rmp_serde::encode::Error> for Error {
fn from(error: rmp_serde::encode::Error) -> Error {
Error::RmpEncode(error)
impl From<SerdeJsonError> for Error {
fn from(error: SerdeJsonError) -> Error {
Error::SerdeJson(error)
}
}
@ -87,8 +81,7 @@ impl fmt::Display for Error {
MissingDocumentId => write!(f, "document id is missing"),
Rkv(e) => write!(f, "rkv error; {}", e),
Fst(e) => write!(f, "fst error; {}", e),
RmpDecode(e) => write!(f, "rmp decode error; {}", e),
RmpEncode(e) => write!(f, "rmp encode error; {}", e),
SerdeJson(e) => write!(f, "serde json error; {}", e),
Bincode(e) => write!(f, "bincode error; {}", e),
Serializer(e) => write!(f, "serializer error; {}", e),
Deserializer(e) => write!(f, "deserializer error; {}", e),

View File

@ -3,8 +3,9 @@ use std::io::Cursor;
use std::{fmt, error::Error};
use meilidb_schema::{Schema, SchemaAttr};
use rmp_serde::decode::{Deserializer as RmpDeserializer, ReadReader};
use rmp_serde::decode::{Error as RmpError};
use serde_json::Error as SerdeJsonError;
use serde_json::Deserializer as SerdeJsonDeserializer;
use serde_json::de::IoRead as SerdeJsonIoRead;
use serde::{de, forward_to_deserialize_any};
use crate::store::DocumentsFields;
@ -12,8 +13,8 @@ use crate::DocumentId;
#[derive(Debug)]
pub enum DeserializerError {
RmpError(RmpError),
RkvError(rkv::StoreError),
SerdeJson(SerdeJsonError),
Rkv(rkv::StoreError),
Custom(String),
}
@ -26,8 +27,8 @@ impl de::Error for DeserializerError {
impl fmt::Display for DeserializerError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
DeserializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e),
DeserializerError::RkvError(e) => write!(f, "rkv related error: {}", e),
DeserializerError::SerdeJson(e) => write!(f, "serde json related error: {}", e),
DeserializerError::Rkv(e) => write!(f, "rkv related error: {}", e),
DeserializerError::Custom(s) => f.write_str(s),
}
}
@ -35,15 +36,15 @@ impl fmt::Display for DeserializerError {
impl Error for DeserializerError {}
impl From<RmpError> for DeserializerError {
fn from(error: RmpError) -> DeserializerError {
DeserializerError::RmpError(error)
impl From<SerdeJsonError> for DeserializerError {
fn from(error: SerdeJsonError) -> DeserializerError {
DeserializerError::SerdeJson(error)
}
}
impl From<rkv::StoreError> for DeserializerError {
fn from(error: rkv::StoreError) -> DeserializerError {
DeserializerError::RkvError(error)
DeserializerError::Rkv(error)
}
}
@ -88,7 +89,12 @@ where R: rkv::Readable,
let is_displayed = self.schema.props(attr).is_displayed();
if is_displayed && self.attributes.map_or(true, |f| f.contains(&attr)) {
let attribute_name = self.schema.attribute_name(attr);
Some((attribute_name, Value::new(value)))
let cursor = Cursor::new(value.to_owned());
let ioread = SerdeJsonIoRead::new(cursor);
let value = Value(SerdeJsonDeserializer::new(ioread));
Some((attribute_name, value))
} else {
None
}
@ -104,18 +110,9 @@ where R: rkv::Readable,
}
}
struct Value<A>(RmpDeserializer<ReadReader<Cursor<A>>>) where A: AsRef<[u8]>;
struct Value(SerdeJsonDeserializer<SerdeJsonIoRead<Cursor<Vec<u8>>>>);
impl<A> Value<A> where A: AsRef<[u8]>
{
fn new(value: A) -> Value<A> {
Value(RmpDeserializer::new(Cursor::new(value)))
}
}
impl<'de, A> de::IntoDeserializer<'de, RmpError> for Value<A>
where A: AsRef<[u8]>,
{
impl<'de> de::IntoDeserializer<'de, SerdeJsonError> for Value {
type Deserializer = Self;
fn into_deserializer(self) -> Self::Deserializer {
@ -123,10 +120,8 @@ where A: AsRef<[u8]>,
}
}
impl<'de, 'a, A> de::Deserializer<'de> for Value<A>
where A: AsRef<[u8]>,
{
type Error = RmpError;
impl<'de> de::Deserializer<'de> for Value {
type Error = SerdeJsonError;
fn deserialize_any<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>
where V: de::Visitor<'de>

View File

@ -26,7 +26,6 @@ use std::collections::BTreeMap;
use std::{fmt, error::Error};
use meilidb_schema::SchemaAttr;
use rmp_serde::encode::Error as RmpError;
use serde_json::Error as SerdeJsonError;
use serde::ser;
@ -36,10 +35,9 @@ use crate::{DocumentId, ParseNumberError};
pub enum SerializerError {
DocumentIdNotFound,
InvalidDocumentIdType,
RmpError(RmpError),
RkvError(rkv::StoreError),
SerdeJsonError(SerdeJsonError),
ParseNumberError(ParseNumberError),
SerdeJson(SerdeJsonError),
ParseNumber(ParseNumberError),
UnserializableType { type_name: &'static str },
UnindexableType { type_name: &'static str },
UnrankableType { type_name: &'static str },
@ -56,15 +54,14 @@ impl fmt::Display for SerializerError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
SerializerError::DocumentIdNotFound => {
write!(f, "serialized document does not have an id according to the schema")
f.write_str("serialized document does not have an id according to the schema")
},
SerializerError::InvalidDocumentIdType => {
write!(f, "document identifier can only be of type string or number")
f.write_str("document identifier can only be of type string or number")
},
SerializerError::RmpError(e) => write!(f, "rmp serde related error: {}", e),
SerializerError::RkvError(e) => write!(f, "rkv related error: {}", e),
SerializerError::SerdeJsonError(e) => write!(f, "serde json error: {}", e),
SerializerError::ParseNumberError(e) => {
SerializerError::SerdeJson(e) => write!(f, "serde json error: {}", e),
SerializerError::ParseNumber(e) => {
write!(f, "error while trying to parse a number: {}", e)
},
SerializerError::UnserializableType { type_name } => {
@ -89,15 +86,9 @@ impl From<String> for SerializerError {
}
}
impl From<RmpError> for SerializerError {
fn from(error: RmpError) -> SerializerError {
SerializerError::RmpError(error)
}
}
impl From<SerdeJsonError> for SerializerError {
fn from(error: SerdeJsonError) -> SerializerError {
SerializerError::SerdeJsonError(error)
SerializerError::SerdeJson(error)
}
}
@ -109,7 +100,7 @@ impl From<rkv::StoreError> for SerializerError {
impl From<ParseNumberError> for SerializerError {
fn from(error: ParseNumberError) -> SerializerError {
SerializerError::ParseNumberError(error)
SerializerError::ParseNumber(error)
}
}

View File

@ -270,7 +270,7 @@ where T: ser::Serialize,
if let Some(attribute) = schema.attribute(key) {
let props = schema.props(attribute);
let serialized = rmp_serde::to_vec_named(value)?;
let serialized = serde_json::to_vec(value)?;
document_store.set_document_field(document_id, attribute, serialized);
if props.is_indexed() {

View File

@ -104,7 +104,7 @@ impl Index {
{
let bytes = self.documents_fields.document_attribute(reader, document_id, attribute)?;
match bytes {
Some(bytes) => Ok(Some(rmp_serde::from_read_ref(bytes)?)),
Some(bytes) => Ok(Some(serde_json::from_slice(bytes)?)),
None => Ok(None),
}
}

View File

@ -68,7 +68,7 @@ impl Updates {
) -> MResult<()>
{
let update_id_bytes = update_id.to_be_bytes();
let update = rmp_serde::to_vec_named(&update)?;
let update = serde_json::to_vec(&update)?;
let blob = Value::Blob(&update);
self.updates.put(writer, update_id_bytes, &blob)?;
Ok(())
@ -86,8 +86,7 @@ impl Updates {
match first_data {
Some(Value::Blob(bytes)) => {
let update = rmp_serde::from_read_ref(&bytes)?;
let update = serde_json::from_slice(&bytes)?;
// remove it from the database now
let first_id_bytes = first_id.to_be_bytes();
self.updates.delete(writer, first_id_bytes)?;

View File

@ -65,8 +65,8 @@ pub fn push_documents_addition<D: serde::Serialize>(
{
let mut values = Vec::with_capacity(addition.len());
for add in addition {
let vec = rmp_serde::to_vec_named(&add)?;
let add = rmp_serde::from_read(&vec[..])?;
let vec = serde_json::to_vec(&add)?;
let add = serde_json::from_slice(&vec)?;
values.push(add);
}
@ -85,7 +85,7 @@ pub fn apply_documents_addition(
postings_lists_store: store::PostingsLists,
docs_words_store: store::DocsWords,
mut ranked_map: RankedMap,
addition: Vec<rmpv::Value>,
addition: Vec<serde_json::Value>,
) -> MResult<()>
{
let mut document_ids = HashSet::new();

View File

@ -26,7 +26,7 @@ use meilidb_schema::Schema;
pub enum Update {
Schema(Schema),
Customs(Vec<u8>),
DocumentsAddition(Vec<rmpv::Value>),
DocumentsAddition(Vec<serde_json::Value>),
DocumentsDeletion(Vec<DocumentId>),
SynonymsAddition(BTreeMap<String, Vec<String>>),
SynonymsDeletion(BTreeMap<String, Option<Vec<String>>>),